From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/arrow/csharp/.editorconfig | 169 ++++ src/arrow/csharp/.gitattributes | 36 + src/arrow/csharp/.gitignore | 267 ++++++ src/arrow/csharp/Apache.Arrow.sln | 67 ++ src/arrow/csharp/ApacheArrow.snk | Bin 0 -> 596 bytes src/arrow/csharp/Directory.Build.props | 59 ++ src/arrow/csharp/Directory.Build.targets | 29 + src/arrow/csharp/README.md | 184 ++++ src/arrow/csharp/examples/Examples.sln | 31 + .../FluentBuilderExample.csproj | 12 + .../examples/FluentBuilderExample/Program.cs | 61 ++ .../Apache.Arrow.Flight.AspNetCore.csproj | 15 + .../FlightIEndpointRouteBuilderExtensions.cs | 28 + .../FlightIGrpcServerBuilderExtensions.cs | 30 + .../Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 21 + .../src/Apache.Arrow.Flight/Client/FlightClient.cs | 120 +++ .../Client/FlightClientRecordBatchStreamReader.cs | 28 + .../Client/FlightClientRecordBatchStreamWriter.cs | 56 ++ .../Client/FlightRecordBatchDuplexStreamingCall.cs | 93 ++ .../Client/FlightRecordBatchStreamingCall.cs | 83 ++ .../csharp/src/Apache.Arrow.Flight/FlightAction.cs | 75 ++ .../src/Apache.Arrow.Flight/FlightActionType.cs | 61 ++ .../src/Apache.Arrow.Flight/FlightCriteria.cs | 70 ++ .../src/Apache.Arrow.Flight/FlightDescriptor.cs | 102 +++ .../Apache.Arrow.Flight/FlightDescriptorType.cs | 23 + .../src/Apache.Arrow.Flight/FlightEndpoint.cs | 73 ++ .../csharp/src/Apache.Arrow.Flight/FlightInfo.cs | 78 ++ .../src/Apache.Arrow.Flight/FlightLocation.cs | 59 ++ .../src/Apache.Arrow.Flight/FlightPutResult.cs | 64 ++ .../FlightRecordBatchStreamReader.cs | 104 +++ .../FlightRecordBatchStreamWriter.cs | 77 ++ .../csharp/src/Apache.Arrow.Flight/FlightResult.cs | 71 ++ .../csharp/src/Apache.Arrow.Flight/FlightTicket.cs | 70 ++ .../Internal/FlightDataStream.cs | 109 +++ .../Internal/FlightMessageSerializer.cs | 61 ++ .../Internal/RecordBatcReaderImplementation.cs | 131 +++ .../Apache.Arrow.Flight/Internal/SchemaWriter.cs | 55 ++ .../Apache.Arrow.Flight/Internal/StreamReader.cs | 54 ++ .../Apache.Arrow.Flight/Internal/StreamWriter.cs | 51 ++ .../Apache.Arrow.Flight/Properties/AssemblyInfo.cs | 18 + .../src/Apache.Arrow.Flight/Server/FlightServer.cs | 61 ++ .../Server/FlightServerRecordBatchStreamReader.cs | 31 + .../Server/FlightServerRecordBatchStreamWriter.cs | 31 + .../Server/Internal/FlightServerImplementation.cs | 100 +++ .../csharp/src/Apache.Arrow/Apache.Arrow.csproj | 42 + src/arrow/csharp/src/Apache.Arrow/Arrays/Array.cs | 91 ++ .../csharp/src/Apache.Arrow/Arrays/ArrayData.cs | 115 +++ .../Apache.Arrow/Arrays/ArrayDataConcatenator.cs | 240 +++++ .../Arrays/ArrowArrayBuilderFactory.cs | 79 ++ .../Apache.Arrow/Arrays/ArrowArrayConcatenator.cs | 45 + .../src/Apache.Arrow/Arrays/ArrowArrayFactory.cs | 84 ++ .../src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs | 22 + .../csharp/src/Apache.Arrow/Arrays/BinaryArray.cs | 358 ++++++++ .../csharp/src/Apache.Arrow/Arrays/BooleanArray.cs | 194 ++++ .../csharp/src/Apache.Arrow/Arrays/Date32Array.cs | 112 +++ .../csharp/src/Apache.Arrow/Arrays/Date64Array.cs | 117 +++ .../src/Apache.Arrow/Arrays/DateArrayBuilder.cs | 209 +++++ .../src/Apache.Arrow/Arrays/Decimal128Array.cs | 95 ++ .../src/Apache.Arrow/Arrays/Decimal256Array.cs | 96 ++ .../Apache.Arrow/Arrays/DelegatingArrayBuilder.cs | 102 +++ .../src/Apache.Arrow/Arrays/DictionaryArray.cs | 61 ++ .../csharp/src/Apache.Arrow/Arrays/DoubleArray.cs | 45 + .../Apache.Arrow/Arrays/FixedSizeBinaryArray.cs | 226 +++++ .../csharp/src/Apache.Arrow/Arrays/FloatArray.cs | 45 + .../csharp/src/Apache.Arrow/Arrays/Int16Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/Int32Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/Int64Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/Int8Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/ListArray.cs | 200 +++++ .../src/Apache.Arrow/Arrays/PrimitiveArray.cs | 70 ++ .../Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs | 201 +++++ .../csharp/src/Apache.Arrow/Arrays/StringArray.cs | 95 ++ .../csharp/src/Apache.Arrow/Arrays/StructArray.cs | 59 ++ .../src/Apache.Arrow/Arrays/TimestampArray.cs | 149 ++++ .../csharp/src/Apache.Arrow/Arrays/UInt16Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/UInt32Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/UInt64Array.cs | 46 + .../csharp/src/Apache.Arrow/Arrays/UInt8Array.cs | 45 + .../csharp/src/Apache.Arrow/Arrays/UnionArray.cs | 51 ++ .../src/Apache.Arrow/ArrowBuffer.BitmapBuilder.cs | 280 ++++++ .../csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs | 255 ++++++ src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.cs | 76 ++ src/arrow/csharp/src/Apache.Arrow/BitUtility.cs | 204 +++++ src/arrow/csharp/src/Apache.Arrow/ChunkedArray.cs | 91 ++ src/arrow/csharp/src/Apache.Arrow/Column.cs | 73 ++ .../csharp/src/Apache.Arrow/DecimalUtility.cs | 162 ++++ .../Apache.Arrow/Extensions/ArrayDataExtensions.cs | 45 + .../Apache.Arrow/Extensions/ArrayPoolExtensions.cs | 63 ++ .../Apache.Arrow/Extensions/ArrowTypeExtensions.cs | 42 + .../Apache.Arrow/Extensions/FlatbufExtensions.cs | 85 ++ .../src/Apache.Arrow/Extensions/SpanExtensions.cs | 31 + .../Apache.Arrow/Extensions/StreamExtensions.cs | 70 ++ .../Extensions/StreamExtensions.netcoreapp2.1.cs | 34 + .../Extensions/StreamExtensions.netstandard.cs | 124 +++ .../Apache.Arrow/Extensions/TimeSpanExtensions.cs | 35 + .../Extensions/TupleExtensions.netstandard.cs | 29 + src/arrow/csharp/src/Apache.Arrow/Field.Builder.cs | 93 ++ src/arrow/csharp/src/Apache.Arrow/Field.cs | 65 ++ src/arrow/csharp/src/Apache.Arrow/Flatbuf/Block.cs | 37 + .../src/Apache.Arrow/Flatbuf/BodyCompression.cs | 47 + .../csharp/src/Apache.Arrow/Flatbuf/Buffer.cs | 36 + .../src/Apache.Arrow/Flatbuf/DictionaryBatch.cs | 54 ++ .../src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs | 57 ++ .../Flatbuf/Enums/BodyCompressionMethod.cs | 24 + .../Apache.Arrow/Flatbuf/Enums/CompressionType.cs | 15 + .../src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs | 15 + .../src/Apache.Arrow/Flatbuf/Enums/Endianness.cs | 17 + .../src/Apache.Arrow/Flatbuf/Enums/Feature.cs | 39 + .../src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs | 15 + .../Apache.Arrow/Flatbuf/Enums/MessageHeader.cs | 26 + .../Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs | 29 + .../src/Apache.Arrow/Flatbuf/Enums/Precision.cs | 16 + .../src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs | 17 + .../csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs | 38 + .../src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs | 15 + src/arrow/csharp/src/Apache.Arrow/Flatbuf/Field.cs | 83 ++ .../csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs | 44 + .../src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs | 39 + .../src/Apache.Arrow/Flatbuf/FixedSizeList.cs | 39 + .../Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs | 891 +++++++++++++++++++ .../Flatbuf/FlatBuffers/ByteBufferUtil.cs | 39 + .../Flatbuf/FlatBuffers/FlatBufferBuilder.cs | 812 +++++++++++++++++ .../Flatbuf/FlatBuffers/FlatBufferConstants.cs | 29 + .../Flatbuf/FlatBuffers/IFlatbufferObject.cs | 28 + .../src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs | 48 + .../src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs | 27 + .../src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs | 195 ++++ .../csharp/src/Apache.Arrow/Flatbuf/Footer.cs | 68 ++ .../csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs | 57 ++ src/arrow/csharp/src/Apache.Arrow/Flatbuf/Map.cs | 63 ++ .../csharp/src/Apache.Arrow/Flatbuf/Message.cs | 60 ++ .../csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs | 67 ++ .../csharp/src/Apache.Arrow/Flatbuf/Schema.cs | 76 ++ .../csharp/src/Apache.Arrow/Flatbuf/Tensor.cs | 60 ++ .../csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs | 53 ++ .../src/Apache.Arrow/Flatbuf/Types/Binary.cs | 29 + .../csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs | 29 + .../csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs | 44 + .../src/Apache.Arrow/Flatbuf/Types/Decimal.cs | 54 ++ .../src/Apache.Arrow/Flatbuf/Types/Duration.cs | 38 + .../Apache.Arrow/Flatbuf/Types/FloatingPoint.cs | 38 + .../csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs | 42 + .../src/Apache.Arrow/Flatbuf/Types/Interval.cs | 38 + .../src/Apache.Arrow/Flatbuf/Types/LargeBinary.cs | 31 + .../src/Apache.Arrow/Flatbuf/Types/LargeList.cs | 31 + .../src/Apache.Arrow/Flatbuf/Types/LargeUtf8.cs | 31 + .../csharp/src/Apache.Arrow/Flatbuf/Types/List.cs | 29 + .../csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs | 30 + .../src/Apache.Arrow/Flatbuf/Types/Struct_.cs | 32 + .../csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs | 45 + .../src/Apache.Arrow/Flatbuf/Types/Timestamp.cs | 74 ++ .../csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs | 56 ++ .../csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs | 30 + .../src/Apache.Arrow/Interfaces/IArrowArray.cs | 40 + .../Apache.Arrow/Interfaces/IArrowArrayBuilder.cs | 54 ++ .../Apache.Arrow/Interfaces/IArrowArrayVisitor.cs | 30 + .../src/Apache.Arrow/Ipc/ArrowFileConstants.cs | 24 + .../csharp/src/Apache.Arrow/Ipc/ArrowFileReader.cs | 69 ++ .../Ipc/ArrowFileReaderImplementation.cs | 308 +++++++ .../csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs | 273 ++++++ .../csharp/src/Apache.Arrow/Ipc/ArrowFooter.cs | 96 ++ .../Ipc/ArrowMemoryReaderImplementation.cs | 118 +++ .../Apache.Arrow/Ipc/ArrowReaderImplementation.cs | 357 ++++++++ .../src/Apache.Arrow/Ipc/ArrowStreamReader.cs | 90 ++ .../Ipc/ArrowStreamReaderImplementation.cs | 268 ++++++ .../src/Apache.Arrow/Ipc/ArrowStreamWriter.cs | 982 +++++++++++++++++++++ .../Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs | 266 ++++++ src/arrow/csharp/src/Apache.Arrow/Ipc/Block.cs | 40 + .../csharp/src/Apache.Arrow/Ipc/DictionaryMemo.cs | 111 +++ .../csharp/src/Apache.Arrow/Ipc/IArrowReader.cs | 26 + .../csharp/src/Apache.Arrow/Ipc/IpcOptions.cs | 37 + .../src/Apache.Arrow/Ipc/MessageSerializer.cs | 201 +++++ .../Ipc/ReadOnlyMemoryBufferAllocator.cs | 39 + .../src/Apache.Arrow/Memory/MemoryAllocator.cs | 81 ++ .../Apache.Arrow/Memory/NativeMemoryAllocator.cs | 51 ++ .../src/Apache.Arrow/Memory/NativeMemoryManager.cs | 83 ++ .../src/Apache.Arrow/Memory/NullMemoryOwner.cs | 29 + .../src/Apache.Arrow/Properties/AssembyInfo.cs | 18 + .../Apache.Arrow/Properties/Resources.Designer.cs | 73 ++ .../src/Apache.Arrow/Properties/Resources.resx | 123 +++ .../csharp/src/Apache.Arrow/RecordBatch.Builder.cs | 167 ++++ src/arrow/csharp/src/Apache.Arrow/RecordBatch.cs | 90 ++ .../csharp/src/Apache.Arrow/Schema.Builder.cs | 92 ++ src/arrow/csharp/src/Apache.Arrow/Schema.cs | 125 +++ src/arrow/csharp/src/Apache.Arrow/Table.cs | 113 +++ .../csharp/src/Apache.Arrow/Types/ArrowType.cs | 43 + .../csharp/src/Apache.Arrow/Types/BinaryType.cs | 28 + .../csharp/src/Apache.Arrow/Types/BooleanType.cs | 30 + .../csharp/src/Apache.Arrow/Types/Date32Type.cs | 30 + .../csharp/src/Apache.Arrow/Types/Date64Type.cs | 30 + .../csharp/src/Apache.Arrow/Types/DateType.cs | 29 + .../src/Apache.Arrow/Types/Decimal128Type.cs | 35 + .../src/Apache.Arrow/Types/Decimal256Type.cs | 35 + .../src/Apache.Arrow/Types/DictionaryType.cs | 46 + .../csharp/src/Apache.Arrow/Types/DoubleType.cs | 31 + .../src/Apache.Arrow/Types/FixedSizeBinaryType.cs | 38 + .../src/Apache.Arrow/Types/FixedWidthType.cs | 25 + .../csharp/src/Apache.Arrow/Types/FloatType.cs | 31 + .../src/Apache.Arrow/Types/FloatingPointType.cs | 30 + .../csharp/src/Apache.Arrow/Types/HalfFloatType.cs | 31 + .../csharp/src/Apache.Arrow/Types/IArrowType.cs | 63 ++ .../src/Apache.Arrow/Types/IArrowTypeVisitor.cs | 29 + .../csharp/src/Apache.Arrow/Types/Int16Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/Int32Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/Int64Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/Int8Type.cs | 30 + .../csharp/src/Apache.Arrow/Types/IntegerType.cs | 22 + .../csharp/src/Apache.Arrow/Types/IntervalUnit.cs | 40 + .../csharp/src/Apache.Arrow/Types/ListType.cs | 37 + .../csharp/src/Apache.Arrow/Types/NestedType.cs | 46 + .../csharp/src/Apache.Arrow/Types/NullType.cs | 28 + .../csharp/src/Apache.Arrow/Types/NumberType.cs | 23 + .../csharp/src/Apache.Arrow/Types/StringType.cs | 28 + .../csharp/src/Apache.Arrow/Types/StructType.cs | 61 ++ .../csharp/src/Apache.Arrow/Types/Time32Type.cs | 32 + .../csharp/src/Apache.Arrow/Types/Time64Type.cs | 32 + .../csharp/src/Apache.Arrow/Types/TimeType.cs | 36 + .../csharp/src/Apache.Arrow/Types/TimestampType.cs | 52 ++ .../csharp/src/Apache.Arrow/Types/UInt16Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/UInt32Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/UInt64Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/UInt8Type.cs | 29 + .../csharp/src/Apache.Arrow/Types/UnionType.cs | 46 + src/arrow/csharp/src/Apache.Arrow/Utility.cs | 87 ++ .../Apache.Arrow.Benchmarks.csproj | 18 + .../ArrowReaderBenchmark.cs | 160 ++++ .../ArrowWriterBenchmark.cs | 58 ++ .../csharp/test/Apache.Arrow.Benchmarks/Program.cs | 29 + .../Apache.Arrow.Flight.TestWeb.csproj | 15 + .../Extensions/AsyncStreamExtensions.cs | 39 + .../Apache.Arrow.Flight.TestWeb/FlightHolder.cs | 62 ++ .../Apache.Arrow.Flight.TestWeb/FlightStore.cs | 27 + .../test/Apache.Arrow.Flight.TestWeb/Program.cs | 52 ++ .../Properties/launchSettings.json | 12 + .../RecordBatchWithMetadata.cs | 31 + .../test/Apache.Arrow.Flight.TestWeb/Startup.cs | 61 ++ .../TestFlightServer.cs | 116 +++ .../appsettings.Development.json | 10 + .../Apache.Arrow.Flight.TestWeb/appsettings.json | 15 + .../Apache.Arrow.Flight.Tests.csproj | 21 + .../FlightInfoComparer.cs | 39 + .../test/Apache.Arrow.Flight.Tests/FlightTests.cs | 316 +++++++ .../Apache.Arrow.Flight.Tests/TestWebFactory.cs | 79 ++ .../Apache.Arrow.IntegrationTest.csproj | 16 + .../IntegrationCommand.cs | 609 +++++++++++++ .../test/Apache.Arrow.IntegrationTest/JsonFile.cs | 184 ++++ .../test/Apache.Arrow.IntegrationTest/Program.cs | 54 ++ .../Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 22 + .../test/Apache.Arrow.Tests/ArrayBuilderTests.cs | 226 +++++ .../ArrayDataConcatenatorTests.cs | 52 ++ .../test/Apache.Arrow.Tests/ArrayTypeComparer.cs | 121 +++ .../ArrowArrayBuilderFactoryReflector.cs | 32 + .../ArrowArrayConcatenatorTests.cs | 396 +++++++++ .../test/Apache.Arrow.Tests/ArrowArrayTests.cs | 274 ++++++ .../ArrowBufferBitmapBuilderTests.cs | 493 +++++++++++ .../Apache.Arrow.Tests/ArrowBufferBuilderTests.cs | 216 +++++ .../test/Apache.Arrow.Tests/ArrowBufferTests.cs | 114 +++ .../Apache.Arrow.Tests/ArrowFileReaderTests.cs | 159 ++++ .../Apache.Arrow.Tests/ArrowFileWriterTests.cs | 168 ++++ .../test/Apache.Arrow.Tests/ArrowReaderVerifier.cs | 302 +++++++ .../Apache.Arrow.Tests/ArrowStreamReaderTests.cs | 248 ++++++ .../Apache.Arrow.Tests/ArrowStreamWriterTests.cs | 682 ++++++++++++++ .../Apache.Arrow.Tests/BinaryArrayBuilderTests.cs | 489 ++++++++++ .../test/Apache.Arrow.Tests/BitUtilityTests.cs | 171 ++++ .../test/Apache.Arrow.Tests/BooleanArrayTests.cs | 222 +++++ .../csharp/test/Apache.Arrow.Tests/ColumnTests.cs | 58 ++ .../test/Apache.Arrow.Tests/Date32ArrayTests.cs | 125 +++ .../test/Apache.Arrow.Tests/Date64ArrayTests.cs | 133 +++ .../Apache.Arrow.Tests/Decimal128ArrayTests.cs | 241 +++++ .../Apache.Arrow.Tests/Decimal256ArrayTests.cs | 241 +++++ .../test/Apache.Arrow.Tests/DecimalUtilityTests.cs | 51 ++ .../Apache.Arrow.Tests/DictionaryArrayTests.cs | 67 ++ .../Extensions/DateTimeOffsetExtensions.cs | 40 + .../test/Apache.Arrow.Tests/FieldComparer.cs | 44 + .../Fixtures/DefaultMemoryAllocatorFixture.cs | 31 + .../test/Apache.Arrow.Tests/SchemaBuilderTests.cs | 156 ++++ .../test/Apache.Arrow.Tests/SchemaComparer.cs | 46 + .../test/Apache.Arrow.Tests/StructArrayTests.cs | 144 +++ .../csharp/test/Apache.Arrow.Tests/TableTests.cs | 83 ++ .../csharp/test/Apache.Arrow.Tests/TestData.cs | 321 +++++++ .../test/Apache.Arrow.Tests/TestDateAndTimeData.cs | 83 ++ .../test/Apache.Arrow.Tests/TestMemoryAllocator.cs | 29 + .../csharp/test/Apache.Arrow.Tests/TypeTests.cs | 131 +++ src/arrow/csharp/test/Directory.Build.props | 26 + 284 files changed, 26742 insertions(+) create mode 100644 src/arrow/csharp/.editorconfig create mode 100644 src/arrow/csharp/.gitattributes create mode 100644 src/arrow/csharp/.gitignore create mode 100644 src/arrow/csharp/Apache.Arrow.sln create mode 100644 src/arrow/csharp/ApacheArrow.snk create mode 100644 src/arrow/csharp/Directory.Build.props create mode 100644 src/arrow/csharp/Directory.Build.targets create mode 100644 src/arrow/csharp/README.md create mode 100644 src/arrow/csharp/examples/Examples.sln create mode 100644 src/arrow/csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj create mode 100644 src/arrow/csharp/examples/FluentBuilderExample/Program.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIEndpointRouteBuilderExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIGrpcServerBuilderExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClient.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchDuplexStreamingCall.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchStreamingCall.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightAction.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightActionType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightCriteria.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptor.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptorType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightEndpoint.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightInfo.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightLocation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightPutResult.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightResult.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/FlightTicket.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/RecordBatcReaderImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/SchemaWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Properties/AssemblyInfo.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow.Flight/Server/Internal/FlightServerImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Apache.Arrow.csproj create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayData.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayConcatenator.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Date32Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Date64Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal128Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal256Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/DictionaryArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/FixedSizeBinaryArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/FloatArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Int16Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Int32Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Int64Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/Int8Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/ListArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/StringArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/StructArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Arrays/UnionArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.BitmapBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/BitUtility.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/ChunkedArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Column.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/DecimalUtility.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/SpanExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/TimeSpanExtensions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Field.Builder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Field.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Block.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/BodyCompression.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/BodyCompressionMethod.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/CompressionType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Feature.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Field.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Footer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Map.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Message.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Schema.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Tensor.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Duration.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/LargeBinary.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/LargeList.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/LargeUtf8.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/List.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Interfaces/IArrowArray.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Interfaces/IArrowArrayBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Interfaces/IArrowArrayVisitor.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowFileConstants.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowFileReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowFileReaderImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowFileWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowFooter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowMemoryReaderImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowStreamReaderImplementation.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/Block.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/DictionaryMemo.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/IArrowReader.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/IpcOptions.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Ipc/ReadOnlyMemoryBufferAllocator.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Memory/MemoryAllocator.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Memory/NativeMemoryAllocator.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Memory/NativeMemoryManager.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Memory/NullMemoryOwner.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Properties/AssembyInfo.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Properties/Resources.Designer.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Properties/Resources.resx create mode 100644 src/arrow/csharp/src/Apache.Arrow/RecordBatch.Builder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/RecordBatch.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Schema.Builder.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Schema.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Table.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/ArrowType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/BinaryType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/BooleanType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Date32Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Date64Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/DateType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Decimal128Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Decimal256Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/DictionaryType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/DoubleType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/FixedSizeBinaryType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/FixedWidthType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/FloatType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/FloatingPointType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/HalfFloatType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/IArrowType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/IArrowTypeVisitor.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Int16Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Int32Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Int64Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Int8Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/IntegerType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/IntervalUnit.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/ListType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/NestedType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/NullType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/NumberType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/StringType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/StructType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Time32Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/Time64Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/TimeType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/TimestampType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/UInt16Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/UInt32Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/UInt64Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/UInt8Type.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Types/UnionType.cs create mode 100644 src/arrow/csharp/src/Apache.Arrow/Utility.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj create mode 100644 src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Extensions/AsyncStreamExtensions.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightHolder.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightStore.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Program.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Properties/launchSettings.json create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/RecordBatchWithMetadata.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.Development.json create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.json create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightInfoComparer.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Flight.Tests/TestWebFactory.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj create mode 100644 src/arrow/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Program.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrayDataConcatenatorTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBitmapBuilderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBuilderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileReaderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/BinaryArrayBuilderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/BitUtilityTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/BooleanArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/ColumnTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Decimal128ArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Decimal256ArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/DecimalUtilityTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/DictionaryArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Extensions/DateTimeOffsetExtensions.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/FieldComparer.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/Fixtures/DefaultMemoryAllocatorFixture.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/TableTests.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/TestData.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/TestMemoryAllocator.cs create mode 100644 src/arrow/csharp/test/Apache.Arrow.Tests/TypeTests.cs create mode 100644 src/arrow/csharp/test/Directory.Build.props (limited to 'src/arrow/csharp') diff --git a/src/arrow/csharp/.editorconfig b/src/arrow/csharp/.editorconfig new file mode 100644 index 000000000..01506a0e2 --- /dev/null +++ b/src/arrow/csharp/.editorconfig @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +root = true + +# Default settings: +# A newline ending every file +# Use 4 spaces as indentation +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# C# files +[*.cs] +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion + +# avoid this. unless absolutely necessary +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Types: use keywords instead of BCL types, and permit var only when the type is clear +csharp_style_var_for_built_in_types = false:suggestion +csharp_style_var_when_type_is_apparent = false:none +csharp_style_var_elsewhere = false:suggestion +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# name all constant fields using PascalCase +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Code style defaults +csharp_using_directive_placement = outside_namespace:suggestion +dotnet_sort_system_directives_first = true +csharp_prefer_braces = true:refactoring +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = false:none +csharp_style_prefer_switch_expression = true:suggestion + +# Code quality +dotnet_style_readonly_field = true:suggestion +dotnet_code_quality_unused_parameters = non_public:suggestion + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:refactoring +dotnet_style_prefer_conditional_expression_over_return = true:refactoring +csharp_prefer_simple_default_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:refactoring +csharp_style_expression_bodied_constructors = true:refactoring +csharp_style_expression_bodied_operators = true:refactoring +csharp_style_expression_bodied_properties = true:refactoring +csharp_style_expression_bodied_indexers = true:refactoring +csharp_style_expression_bodied_accessors = true:refactoring +csharp_style_expression_bodied_lambdas = true:refactoring +csharp_style_expression_bodied_local_functions = true:refactoring + +# Pattern matching +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = false:none +csharp_style_prefer_range_operator = false:none +csharp_style_pattern_local_over_anonymous_function = false:none + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_around_binary_operators = before_and_after +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_between_square_brackets = false + +# Xml project files +[*.{csproj,props,targets}] +indent_size = 2 +charset = utf-8 diff --git a/src/arrow/csharp/.gitattributes b/src/arrow/csharp/.gitattributes new file mode 100644 index 000000000..d2ff52b12 --- /dev/null +++ b/src/arrow/csharp/.gitattributes @@ -0,0 +1,36 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary \ No newline at end of file diff --git a/src/arrow/csharp/.gitignore b/src/arrow/csharp/.gitignore new file mode 100644 index 000000000..a9fbd5882 --- /dev/null +++ b/src/arrow/csharp/.gitignore @@ -0,0 +1,267 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Project-specific +artifacts/ + +# add .sln files back because they are ignored by the root .gitignore file +!*.sln diff --git a/src/arrow/csharp/Apache.Arrow.sln b/src/arrow/csharp/Apache.Arrow.sln new file mode 100644 index 000000000..873a7f5f1 --- /dev/null +++ b/src/arrow/csharp/Apache.Arrow.sln @@ -0,0 +1,67 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29926.136 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow", "src\Apache.Arrow\Apache.Arrow.csproj", "{BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Tests", "test\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj", "{9CCEC01B-E67A-4726-BE72-7B514F76163F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Benchmarks", "test\Apache.Arrow.Benchmarks\Apache.Arrow.Benchmarks.csproj", "{742DF47D-77C5-4B84-9E0C-69645F1161EA}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight.Tests", "test\Apache.Arrow.Flight.Tests\Apache.Arrow.Flight.Tests.csproj", "{D6443535-3740-4F6C-8001-F90EDAF4CF0C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight.TestWeb", "test\Apache.Arrow.Flight.TestWeb\Apache.Arrow.Flight.TestWeb.csproj", "{058F9CFA-2A13-43B8-87D9-E69F63F9EFF0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight", "src\Apache.Arrow.Flight\Apache.Arrow.Flight.csproj", "{2490AA1E-DDA4-4069-B065-79A4897B0582}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight.AspNetCore", "src\Apache.Arrow.Flight.AspNetCore\Apache.Arrow.Flight.AspNetCore.csproj", "{E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.IntegrationTest", "test\Apache.Arrow.IntegrationTest\Apache.Arrow.IntegrationTest.csproj", "{E8264B7F-B680-4A55-939B-85DB628164BB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Release|Any CPU.Build.0 = Release|Any CPU + {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Release|Any CPU.Build.0 = Release|Any CPU + {742DF47D-77C5-4B84-9E0C-69645F1161EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {742DF47D-77C5-4B84-9E0C-69645F1161EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {742DF47D-77C5-4B84-9E0C-69645F1161EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {742DF47D-77C5-4B84-9E0C-69645F1161EA}.Release|Any CPU.Build.0 = Release|Any CPU + {D6443535-3740-4F6C-8001-F90EDAF4CF0C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D6443535-3740-4F6C-8001-F90EDAF4CF0C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6443535-3740-4F6C-8001-F90EDAF4CF0C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D6443535-3740-4F6C-8001-F90EDAF4CF0C}.Release|Any CPU.Build.0 = Release|Any CPU + {058F9CFA-2A13-43B8-87D9-E69F63F9EFF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {058F9CFA-2A13-43B8-87D9-E69F63F9EFF0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {058F9CFA-2A13-43B8-87D9-E69F63F9EFF0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {058F9CFA-2A13-43B8-87D9-E69F63F9EFF0}.Release|Any CPU.Build.0 = Release|Any CPU + {2490AA1E-DDA4-4069-B065-79A4897B0582}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2490AA1E-DDA4-4069-B065-79A4897B0582}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2490AA1E-DDA4-4069-B065-79A4897B0582}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2490AA1E-DDA4-4069-B065-79A4897B0582}.Release|Any CPU.Build.0 = Release|Any CPU + {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF}.Release|Any CPU.Build.0 = Release|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8264B7F-B680-4A55-939B-85DB628164BB}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572} + EndGlobalSection +EndGlobal diff --git a/src/arrow/csharp/ApacheArrow.snk b/src/arrow/csharp/ApacheArrow.snk new file mode 100644 index 000000000..68df43972 Binary files /dev/null and b/src/arrow/csharp/ApacheArrow.snk differ diff --git a/src/arrow/csharp/Directory.Build.props b/src/arrow/csharp/Directory.Build.props new file mode 100644 index 000000000..affee1814 --- /dev/null +++ b/src/arrow/csharp/Directory.Build.props @@ -0,0 +1,59 @@ + + + + + + + $(MSBuildThisFileDirectory)../ + $(MSBuildThisFileDirectory) + $(CSharpDir)/artifacts/$(MSBuildProjectName) + + + + + Apache Arrow library + Copyright 2016-2019 The Apache Software Foundation + The Apache Software Foundation + 6.0.1 + + + + true + 8.0 + true + $(CSharpDir)ApacheArrow.snk + + + + + The Apache Software Foundation + https://www.apache.org/images/feather.png + LICENSE.txt + https://arrow.apache.org/ + apache arrow + git + https://github.com/apache/arrow + true + snupkg + + + + + + + diff --git a/src/arrow/csharp/Directory.Build.targets b/src/arrow/csharp/Directory.Build.targets new file mode 100644 index 000000000..498c752f2 --- /dev/null +++ b/src/arrow/csharp/Directory.Build.targets @@ -0,0 +1,29 @@ + + + + + + + $([System.IO.Path]::Combine('$(IntermediateOutputPath)','$(TargetFrameworkMoniker).AssemblyAttributes$(DefaultLanguageSourceExtension)')) + + + + + + + diff --git a/src/arrow/csharp/README.md b/src/arrow/csharp/README.md new file mode 100644 index 000000000..2a60cd27c --- /dev/null +++ b/src/arrow/csharp/README.md @@ -0,0 +1,184 @@ + + +# Apache Arrow + +An implementation of Arrow targeting .NET Standard. + +This implementation is under development and may not be suitable for use in production environments. + +# Implementation + +- Arrow 0.11 (specification) +- C# 7.2 +- .NET Standard 1.3 +- Asynchronous I/O +- Uses modern .NET runtime features such as **Span<T>**, **Memory<T>**, **MemoryManager<T>**, and **System.Buffers** primitives for memory allocation, memory storage, and fast serialization. +- Uses **Acyclic Visitor Pattern** for array types and arrays to facilitate serialization, record batch traversal, and format growth. + +# Known Issues + +- Can not read Arrow files containing dictionary batches, tensors, or tables. +- Can not easily modify allocation strategy without implementing a custom memory pool. All allocations are currently 64-byte aligned and padded to 8-bytes. +- Default memory allocation strategy uses an over-allocation strategy with pointer fixing, which results in significant memory overhead for small buffers. A buffer that requires a single byte for storage may be backed by an allocation of up to 64-bytes to satisfy alignment requirements. +- There are currently few builder APIs available for specific array types. Arrays must be built manually with an arrow buffer builder abstraction. +- FlatBuffer code generation is not included in the build process. +- Serialization implementation does not perform exhaustive validation checks during deserialization in every scenario. +- Throws exceptions with vague, inconsistent, or non-localized messages in many situations +- Throws exceptions that are non-specific to the Arrow implementation in some circumstances where it probably should (eg. does not throw ArrowException exceptions) +- Lack of code documentation +- Lack of usage examples +- Lack of comprehensive unit tests +- Lack of comprehensive benchmarks + +# Usage + + using System.Diagnostics; + using System.IO; + using System.Threading.Tasks; + using Apache.Arrow; + using Apache.Arrow.Ipc; + + public static async Task ReadArrowAsync(string filename) + { + using (var stream = File.OpenRead("test.arrow")) + using (var reader = new ArrowFileReader(stream)) + { + var recordBatch = await reader.ReadNextRecordBatchAsync(); + Debug.WriteLine("Read record batch with {0} column(s)", recordBatch.ColumnCount); + return recordBatch; + } + } + + +# Status + +## Memory Management + +- Allocations are 64-byte aligned and padded to 8-bytes. +- Allocations are automatically garbage collected + +## Arrays + +### Primitive Types + +- Int8, Int16, Int32, Int64 +- UInt8, UInt16, UInt32, UInt64 +- Float, Double +- Binary (variable-length) +- String (utf-8) +- Null + +### Parametric Types + +- Timestamp +- Date32 +- Date64 +- Decimal +- Time32 +- Time64 +- Binary (fixed-length) +- List +- Struct + +### Type Metadata + +- Data Types +- Fields +- Schema + +### Serialization + +- File +- Stream + +## Not Implemented + +- Serialization + - Exhaustive validation + - Dictionary Batch + - Can not serialize or deserialize files or streams containing dictionary batches + - Dictionary Encoding + - Schema Metadata + - Schema Field Metadata +- Types + - Tensor + - Table +- Arrays + - Union + - Dense + - Sparse + - Half-Float + - Dictionary +- Array Operations + - Equality / Comparison + - Casting + - Builders +- Compute + - There is currently no API available for a compute / kernel abstraction. + +# Build + +Install the latest `.NET Core SDK` from https://dotnet.microsoft.com/download. + + dotnet build + +## NuGet Build + +To build the NuGet package run the following command to build a debug flavor, preview package into the **artifacts** folder. + + dotnet pack + +When building the officially released version run: (see Note below about current `git` repository) + + dotnet pack -c Release + +Which will build the final/stable package. + +NOTE: When building the officially released version, ensure that your `git` repository has the `origin` remote set to `https://github.com/apache/arrow.git`, which will ensure Source Link is set correctly. See https://github.com/dotnet/sourcelink/blob/master/docs/README.md for more information. + +There are two output artifacts: +1. `Apache.Arrow..nupkg` - this contains the executable assemblies +2. `Apache.Arrow..snupkg` - this contains the debug symbols files + +Both of these artifacts can then be uploaded to https://www.nuget.org/packages/manage/upload. + +## Docker Build + +Build from the Apache Arrow project root. + + docker build -f csharp/build/docker/Dockerfile . + +## Testing + + dotnet test + +All build artifacts are placed in the **artifacts** folder in the project root. + +# Coding Style + +This project follows the coding style specified in [Coding Style](https://github.com/dotnet/runtime/blob/master/docs/coding-guidelines/coding-style.md). + +# Updating FlatBuffers code + +See https://google.github.io/flatbuffers/flatbuffers_guide_use_java_c-sharp.html for how to get the `flatc` executable. + +Run `flatc --csharp` on each `.fbs` file in the [format](../format) folder. And replace the checked in `.cs` files under [FlatBuf](src/Apache.Arrow/Flatbuf) with the generated files. + +Update the non-generated [FlatBuffers](src/Apache.Arrow/Flatbuf/FlatBuffers) `.cs` files with the files from the [google/flatbuffers repo](https://github.com/google/flatbuffers/tree/master/net/FlatBuffers). diff --git a/src/arrow/csharp/examples/Examples.sln b/src/arrow/csharp/examples/Examples.sln new file mode 100644 index 000000000..c0a4199ca --- /dev/null +++ b/src/arrow/csharp/examples/Examples.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27703.2042 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FluentBuilderExample", "FluentBuilderExample\FluentBuilderExample.csproj", "{ECE22119-D91D-44F7-9575-85B98F946289}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow", "..\src\Apache.Arrow\Apache.Arrow.csproj", "{1FE1DE95-FF6E-4895-82E7-909713C53524}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {ECE22119-D91D-44F7-9575-85B98F946289}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ECE22119-D91D-44F7-9575-85B98F946289}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ECE22119-D91D-44F7-9575-85B98F946289}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ECE22119-D91D-44F7-9575-85B98F946289}.Release|Any CPU.Build.0 = Release|Any CPU + {1FE1DE95-FF6E-4895-82E7-909713C53524}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1FE1DE95-FF6E-4895-82E7-909713C53524}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1FE1DE95-FF6E-4895-82E7-909713C53524}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1FE1DE95-FF6E-4895-82E7-909713C53524}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C22A81AD-8B64-4D7C-97AC-49E9F118AE78} + EndGlobalSection +EndGlobal diff --git a/src/arrow/csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj b/src/arrow/csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj new file mode 100644 index 000000000..575a2743e --- /dev/null +++ b/src/arrow/csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp2.1 + + + + + + + \ No newline at end of file diff --git a/src/arrow/csharp/examples/FluentBuilderExample/Program.cs b/src/arrow/csharp/examples/FluentBuilderExample/Program.cs new file mode 100644 index 000000000..6dbdc3d77 --- /dev/null +++ b/src/arrow/csharp/examples/FluentBuilderExample/Program.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.Arrow.Memory; +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace FluentBuilderExample +{ + public class Program + { + public static async Task Main(string[] args) + { + // Use a specific memory pool from which arrays will be allocated (optional) + + var memoryAllocator = new NativeMemoryAllocator(alignment: 64); + + // Build a record batch using the Fluent API + + var recordBatch = new RecordBatch.Builder(memoryAllocator) + .Append("Column A", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10)))) + .Append("Column B", false, col => col.Float(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => Convert.ToSingle(x * 2))))) + .Append("Column C", false, col => col.String(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => $"Item {x+1}")))) + .Append("Column D", false, col => col.Boolean(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => x % 2 == 0)))) + .Build(); + + // Print memory allocation statistics + + Console.WriteLine("Allocations: {0}", memoryAllocator.Statistics.Allocations); + Console.WriteLine("Allocated: {0} byte(s)", memoryAllocator.Statistics.BytesAllocated); + + // Write record batch to a file + + using (var stream = File.OpenWrite("test.arrow")) + using (var writer = new ArrowFileWriter(stream, recordBatch.Schema)) + { + await writer.WriteRecordBatchAsync(recordBatch); + await writer.WriteEndAsync(); + } + + Console.WriteLine("Done"); + Console.ReadKey(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj new file mode 100644 index 000000000..7cfa33c5d --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp3.1 + + + + + + + + + + + diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIEndpointRouteBuilderExtensions.cs b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIEndpointRouteBuilderExtensions.cs new file mode 100644 index 000000000..5902d7b01 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIEndpointRouteBuilderExtensions.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Flight.Server.Internal; +using Microsoft.AspNetCore.Routing; + +namespace Microsoft.AspNetCore.Builder +{ + public static class FlightIEndpointRouteBuilderExtensions + { + public static GrpcServiceEndpointConventionBuilder MapFlightEndpoint(this IEndpointRouteBuilder endpointRouteBuilder) + { + return endpointRouteBuilder.MapGrpcService(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIGrpcServerBuilderExtensions.cs b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIGrpcServerBuilderExtensions.cs new file mode 100644 index 000000000..692e86f62 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight.AspNetCore/Extensions/FlightIGrpcServerBuilderExtensions.cs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Flight.Server; +using Grpc.AspNetCore.Server; + +namespace Microsoft.Extensions.DependencyInjection +{ + public static class FlightIGrpcServerBuilderExtensions + { + public static IGrpcServerBuilder AddFlightServer(this IGrpcServerBuilder grpcServerBuilder) + where T : FlightServer + { + grpcServerBuilder.Services.AddScoped(); + return grpcServerBuilder; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/src/arrow/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj new file mode 100644 index 000000000..bd59268ad --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -0,0 +1,21 @@ + + + + netstandard2.1 + + + + + + + + + + + + + + + + + diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClient.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClient.cs new file mode 100644 index 000000000..8140e0649 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClient.cs @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Internal; +using Apache.Arrow.Flight.Protocol; +using Grpc.Core; +using Grpc.Net.Client; + +namespace Apache.Arrow.Flight.Client +{ + public class FlightClient + { + internal static readonly Empty EmptyInstance = new Empty(); + + private readonly FlightService.FlightServiceClient _client; + + public FlightClient(GrpcChannel grpcChannel) + { + _client = new FlightService.FlightServiceClient(grpcChannel); + } + + public AsyncServerStreamingCall ListFlights(FlightCriteria criteria = null, Metadata headers = null) + { + if(criteria == null) + { + criteria = FlightCriteria.Empty; + } + + var response = _client.ListFlights(criteria.ToProtocol(), headers); + var convertStream = new StreamReader(response.ResponseStream, inFlight => new FlightInfo(inFlight)); + + return new AsyncServerStreamingCall(convertStream, response.ResponseHeadersAsync, response.GetStatus, response.GetTrailers, response.Dispose); + } + + public AsyncServerStreamingCall ListActions(Metadata headers = null) + { + var response = _client.ListActions(EmptyInstance, headers); + var convertStream = new StreamReader(response.ResponseStream, actionType => new FlightActionType(actionType)); + + return new AsyncServerStreamingCall(convertStream, response.ResponseHeadersAsync, response.GetStatus, response.GetTrailers, response.Dispose); + } + + public FlightRecordBatchStreamingCall GetStream(FlightTicket ticket, Metadata headers = null) + { + var stream = _client.DoGet(ticket.ToProtocol(), headers); + var responseStream = new FlightClientRecordBatchStreamReader(stream.ResponseStream); + return new FlightRecordBatchStreamingCall(responseStream, stream.ResponseHeadersAsync, stream.GetStatus, stream.GetTrailers, stream.Dispose); + } + + public AsyncUnaryCall GetInfo(FlightDescriptor flightDescriptor, Metadata headers = null) + { + var flightInfoResult = _client.GetFlightInfoAsync(flightDescriptor.ToProtocol(), headers); + + var flightInfo = flightInfoResult + .ResponseAsync + .ContinueWith(async flightInfo => new FlightInfo(await flightInfo.ConfigureAwait(false))) + .Unwrap(); + + return new AsyncUnaryCall( + flightInfo, + flightInfoResult.ResponseHeadersAsync, + flightInfoResult.GetStatus, + flightInfoResult.GetTrailers, + flightInfoResult.Dispose); + } + + public FlightRecordBatchDuplexStreamingCall StartPut(FlightDescriptor flightDescriptor, Metadata headers = null) + { + var channels = _client.DoPut(headers); + var requestStream = new FlightClientRecordBatchStreamWriter(channels.RequestStream, flightDescriptor); + var readStream = new StreamReader(channels.ResponseStream, putResult => new FlightPutResult(putResult)); + return new FlightRecordBatchDuplexStreamingCall( + requestStream, + readStream, + channels.ResponseHeadersAsync, + channels.GetStatus, + channels.GetTrailers, + channels.Dispose); + } + + public AsyncServerStreamingCall DoAction(FlightAction action, Metadata headers = null) + { + var stream = _client.DoAction(action.ToProtocol(), headers); + var streamReader = new StreamReader(stream.ResponseStream, result => new FlightResult(result)); + return new AsyncServerStreamingCall(streamReader, stream.ResponseHeadersAsync, stream.GetStatus, stream.GetTrailers, stream.Dispose); + } + + public AsyncUnaryCall GetSchema(FlightDescriptor flightDescriptor, Metadata headers = null) + { + var schemaResult = _client.GetSchemaAsync(flightDescriptor.ToProtocol(), headers); + + var schema = schemaResult + .ResponseAsync + .ContinueWith(async schema => FlightMessageSerializer.DecodeSchema((await schemaResult.ResponseAsync.ConfigureAwait(false)).Schema.Memory)) + .Unwrap(); + + return new AsyncUnaryCall( + schema, + schemaResult.ResponseHeadersAsync, + schemaResult.GetStatus, + schemaResult.GetTrailers, + schemaResult.Dispose); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamReader.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamReader.cs new file mode 100644 index 000000000..011af0c83 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamReader.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Flight.Internal; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Client +{ + public class FlightClientRecordBatchStreamReader : FlightRecordBatchStreamReader + { + internal FlightClientRecordBatchStreamReader(IAsyncStreamReader flightDataStream) : base(flightDataStream) + { + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamWriter.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamWriter.cs new file mode 100644 index 000000000..d2e62c42e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightClientRecordBatchStreamWriter.cs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Flight.Internal; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Client +{ + public class FlightClientRecordBatchStreamWriter : FlightRecordBatchStreamWriter, IClientStreamWriter + { + private readonly IClientStreamWriter _clientStreamWriter; + private bool _completed = false; + internal FlightClientRecordBatchStreamWriter(IClientStreamWriter clientStreamWriter, FlightDescriptor flightDescriptor) : base(clientStreamWriter, flightDescriptor) + { + _clientStreamWriter = clientStreamWriter; + } + + protected override void Dispose(bool disposing) + { + if (!_completed) + { + throw new InvalidOperationException("Dispose called before completing the stream."); + } + + base.Dispose(disposing); + } + + public async Task CompleteAsync() + { + if (_completed) + { + return; + } + + await _clientStreamWriter.CompleteAsync().ConfigureAwait(false); + _completed = true; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchDuplexStreamingCall.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchDuplexStreamingCall.cs new file mode 100644 index 000000000..c9e6ecd35 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchDuplexStreamingCall.cs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Client +{ + public class FlightRecordBatchDuplexStreamingCall : IDisposable + { + private readonly Func _getStatusFunc; + private readonly Func _getTrailersFunc; + private readonly Action _disposeAction; + + internal FlightRecordBatchDuplexStreamingCall( + FlightClientRecordBatchStreamWriter requestStream, + IAsyncStreamReader responseStream, + Task responseHeadersAsync, + Func getStatusFunc, + Func getTrailersFunc, + Action disposeAction) + { + RequestStream = requestStream; + ResponseStream = responseStream; + ResponseHeadersAsync = responseHeadersAsync; + _getStatusFunc = getStatusFunc; + _getTrailersFunc = getTrailersFunc; + _disposeAction = disposeAction; + } + + /// + /// Async stream to read streaming responses. + /// + public IAsyncStreamReader ResponseStream { get; } + + /// + /// Async stream to send streaming requests. + /// + public FlightClientRecordBatchStreamWriter RequestStream { get; } + + /// + /// Asynchronous access to response headers. + /// + public Task ResponseHeadersAsync { get; } + + /// + /// Provides means to cleanup after the call. If the call has already finished normally + /// (response stream has been fully read), doesn't do anything. Otherwise, requests + /// cancellation of the call which should terminate all pending async operations + /// associated with the call. As a result, all resources being used by the call should + /// be released eventually. + /// + /// + /// Normally, there is no need for you to dispose the call unless you want to utilize + /// the "Cancel" semantics of invoking Dispose. + /// + public void Dispose() + { + _disposeAction(); + } + + /// + /// Gets the call status if the call has already finished. Throws InvalidOperationException otherwise. + /// + /// + public Status GetStatus() + { + return _getStatusFunc(); + } + + /// + /// Gets the call trailing metadata if the call has already finished. Throws InvalidOperationException otherwise. + /// + /// + public Metadata GetTrailers() + { + return _getTrailersFunc(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchStreamingCall.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchStreamingCall.cs new file mode 100644 index 000000000..246cfa7cd --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Client/FlightRecordBatchStreamingCall.cs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Client +{ + public class FlightRecordBatchStreamingCall : IDisposable + { + private readonly Func _getStatusFunc; + private readonly Func _getTrailersFunc; + private readonly Action _disposeAction; + + internal FlightRecordBatchStreamingCall( + FlightClientRecordBatchStreamReader recordBatchStreamReader, + Task responseHeadersAsync, + Func getStatusFunc, + Func getTrailersFunc, + Action disposeAction) + { + ResponseStream = recordBatchStreamReader; + ResponseHeadersAsync = responseHeadersAsync; + _getStatusFunc = getStatusFunc; + _getTrailersFunc = getTrailersFunc; + _disposeAction = disposeAction; + } + + public FlightClientRecordBatchStreamReader ResponseStream { get; } + + /// + /// Asynchronous access to response headers. + /// + public Task ResponseHeadersAsync { get; } + + /// + /// Gets the call status if the call has already finished. Throws InvalidOperationException otherwise. + /// + /// + public Status GetStatus() + { + return _getStatusFunc(); + } + + /// + /// Gets the call trailing metadata if the call has already finished. Throws InvalidOperationException otherwise. + /// + /// + public Metadata GetTrailers() + { + return _getTrailersFunc(); + } + + /// + /// Provides means to cleanup after the call. If the call has already finished normally + /// (response stream has been fully read), doesn't do anything. Otherwise, requests + /// cancellation of the call which should terminate all pending async operations + /// associated with the call. As a result, all resources being used by the call should + /// be released eventually. + /// + /// + /// Normally, there is no need for you to dispose the call unless you want to utilize + /// the "Cancel" semantics of invoking Dispose. + /// + public void Dispose() + { + _disposeAction(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightAction.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightAction.cs new file mode 100644 index 000000000..4a82fa62a --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightAction.cs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightAction + { + private readonly Protocol.Action _action; + internal FlightAction(Protocol.Action action) + { + _action = action; + } + + public FlightAction(string type, ByteString body) + { + _action = new Protocol.Action() + { + Body = body, + Type = type + }; + } + + public FlightAction(string type, string body) + { + _action = new Protocol.Action() + { + Body = ByteString.CopyFromUtf8(body), + Type = type + }; + } + + public FlightAction(string type, byte[] body) + { + _action = new Protocol.Action() + { + Body = ByteString.CopyFrom(body), + Type = type + }; + } + + public FlightAction(string type) + { + _action = new Protocol.Action() + { + Type = type + }; + } + + public string Type => _action.Type; + + public ByteString Body => _action.Body; + + internal Protocol.Action ToProtocol() + { + return _action; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightActionType.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightActionType.cs new file mode 100644 index 000000000..8df893946 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightActionType.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Apache.Arrow.Flight +{ + public class FlightActionType + { + private readonly Protocol.ActionType _actionType; + internal FlightActionType(Protocol.ActionType actionType) + { + _actionType = actionType; + } + + public FlightActionType(string type, string description) + { + _actionType = new Protocol.ActionType() + { + Description = description, + Type = type + }; + } + + public string Type => _actionType.Type; + public string Description => _actionType.Description; + + internal Protocol.ActionType ToProtocol() + { + return _actionType; + } + + public override bool Equals(object obj) + { + if(obj is FlightActionType other) + { + return Equals(_actionType, other._actionType); + } + return false; + } + + public override int GetHashCode() + { + return _actionType.GetHashCode(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightCriteria.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightCriteria.cs new file mode 100644 index 000000000..6bcb087ac --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightCriteria.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightCriteria + { + internal static readonly FlightCriteria Empty = new FlightCriteria(); + + private readonly Protocol.Criteria _criteria; + + internal FlightCriteria(Protocol.Criteria criteria) + { + _criteria = criteria; + } + + public FlightCriteria() + { + _criteria = new Protocol.Criteria(); + } + + public FlightCriteria(string expression) + { + _criteria = new Protocol.Criteria() + { + Expression = ByteString.CopyFromUtf8(expression) + }; + } + + public FlightCriteria(byte[] bytes) + { + _criteria = new Protocol.Criteria() + { + Expression = ByteString.CopyFrom(bytes) + }; + } + + public FlightCriteria(ByteString byteString) + { + _criteria = new Protocol.Criteria() + { + Expression = byteString + }; + } + + public ByteString Expression => _criteria.Expression; + + internal Protocol.Criteria ToProtocol() + { + return _criteria; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptor.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptor.cs new file mode 100644 index 000000000..7d4433291 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptor.cs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightDescriptor + { + private readonly Protocol.FlightDescriptor _flightDescriptor; + + private FlightDescriptor(ByteString command) + { + _flightDescriptor = new Protocol.FlightDescriptor() + { + Cmd = command, + Type = Protocol.FlightDescriptor.Types.DescriptorType.Cmd + }; + } + + private FlightDescriptor(params string[] paths) + { + _flightDescriptor = new Protocol.FlightDescriptor() + { + Type = Protocol.FlightDescriptor.Types.DescriptorType.Path + }; + + foreach(var path in paths) + { + _flightDescriptor.Path.Add(path); + } + } + + + public static FlightDescriptor CreateCommandDescriptor(byte[] command) + { + return new FlightDescriptor(ByteString.CopyFrom(command)); + } + + public static FlightDescriptor CreateCommandDescriptor(string command) + { + return new FlightDescriptor(ByteString.CopyFromUtf8(command)); + } + + public static FlightDescriptor CreatePathDescriptor(params string[] paths) + { + return new FlightDescriptor(paths); + } + + + internal FlightDescriptor(Protocol.FlightDescriptor flightDescriptor) + { + if(flightDescriptor.Type != Protocol.FlightDescriptor.Types.DescriptorType.Cmd && flightDescriptor.Type != Protocol.FlightDescriptor.Types.DescriptorType.Path) + { + throw new NotSupportedException(); + } + _flightDescriptor = flightDescriptor; + } + + internal Protocol.FlightDescriptor ToProtocol() + { + return _flightDescriptor; + } + + public FlightDescriptorType Type => (FlightDescriptorType)_flightDescriptor.Type; + + public IEnumerable Paths => _flightDescriptor.Path; + + public ByteString Command => _flightDescriptor.Cmd; + + + public override int GetHashCode() + { + return _flightDescriptor.GetHashCode(); + } + + public override bool Equals(object obj) + { + if(obj is FlightDescriptor other) + { + return Equals(_flightDescriptor, other._flightDescriptor); + } + return false; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptorType.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptorType.cs new file mode 100644 index 000000000..120ed225c --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightDescriptorType.cs @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow.Flight +{ + public enum FlightDescriptorType + { + Path = 1, + Command = 2 + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightEndpoint.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightEndpoint.cs new file mode 100644 index 000000000..ab15fed01 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightEndpoint.cs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Apache.Arrow.Flight +{ + public class FlightEndpoint + { + private readonly FlightTicket _ticket; + private readonly IReadOnlyList _locations; + internal FlightEndpoint(Protocol.FlightEndpoint flightEndpoint) + { + _ticket = new FlightTicket(flightEndpoint.Ticket); + _locations = flightEndpoint.Location.Select(x => new FlightLocation(x)).ToList(); + } + + public FlightEndpoint(FlightTicket ticket, IReadOnlyList locations) + { + _ticket = ticket; + _locations = locations; + } + + public FlightTicket Ticket => _ticket; + + public IEnumerable Locations => _locations; + + internal Protocol.FlightEndpoint ToProtocol() + { + var output = new Protocol.FlightEndpoint() + { + Ticket = _ticket.ToProtocol() + }; + + foreach(var location in _locations) + { + output.Location.Add(location.ToProtocol()); + } + return output; + } + + public override bool Equals(object obj) + { + if(obj is FlightEndpoint other) + { + return Equals(_ticket, other._ticket) && + Enumerable.SequenceEqual(_locations, other._locations); + } + return false; + } + + public override int GetHashCode() + { + //Ticket should contain enough to get a good hash code + return _ticket.GetHashCode(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightInfo.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightInfo.cs new file mode 100644 index 000000000..44a7965cc --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightInfo.cs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Apache.Arrow.Flight.Internal; +using Apache.Arrow.Ipc; + +namespace Apache.Arrow.Flight +{ + public class FlightInfo + { + internal FlightInfo(Protocol.FlightInfo flightInfo) + { + Schema = FlightMessageSerializer.DecodeSchema(flightInfo.Schema.Memory); + Descriptor = new FlightDescriptor(flightInfo.FlightDescriptor); + + var endpoints = new List(); + foreach(var endpoint in flightInfo.Endpoint) + { + endpoints.Add(new FlightEndpoint(endpoint)); + } + Endpoints = endpoints; + + TotalBytes = flightInfo.TotalBytes; + TotalRecords = flightInfo.TotalRecords; + } + + public FlightInfo(Schema schema, FlightDescriptor descriptor, IReadOnlyList endpoints, long totalRecords = 0, long totalBytes = 0) + { + Schema = schema; + Descriptor = descriptor; + Endpoints = endpoints; + TotalBytes = totalBytes; + TotalRecords = totalRecords; + } + + public FlightDescriptor Descriptor { get; } + + public Schema Schema { get; } + + public long TotalBytes { get; } + + public long TotalRecords { get; } + + public IReadOnlyList Endpoints { get; } + + internal Protocol.FlightInfo ToProtocol() + { + var serializedSchema = SchemaWriter.SerializeSchema(Schema); + var response = new Protocol.FlightInfo() + { + Schema = serializedSchema, + FlightDescriptor = Descriptor.ToProtocol() + }; + + foreach(var endpoint in Endpoints) + { + response.Endpoint.Add(endpoint.ToProtocol()); + } + + return response; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightLocation.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightLocation.cs new file mode 100644 index 000000000..25b9d5d45 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightLocation.cs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Apache.Arrow.Flight +{ + public class FlightLocation + { + private readonly Protocol.Location _location; + internal FlightLocation(Protocol.Location location) + { + _location = location; + } + + public FlightLocation(string uri) + { + _location = new Protocol.Location() + { + Uri = uri + }; + } + + public string Uri => _location.Uri; + + internal Protocol.Location ToProtocol() + { + return _location; + } + + public override bool Equals(object obj) + { + if(obj is FlightLocation other) + { + return Equals(_location, other._location); + } + return false; + } + + public override int GetHashCode() + { + return _location.GetHashCode(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightPutResult.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightPutResult.cs new file mode 100644 index 000000000..16f278aa5 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightPutResult.cs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightPutResult + { + public static readonly FlightPutResult Empty = new FlightPutResult(); + + private readonly Protocol.PutResult _putResult; + + public FlightPutResult() + { + _putResult = new Protocol.PutResult(); + } + + public FlightPutResult(ByteString applicationMetadata) + { + _putResult = new Protocol.PutResult() + { + AppMetadata = applicationMetadata + }; + } + + public FlightPutResult(byte[] applicationMetadata) + : this(ByteString.CopyFrom(applicationMetadata)) + { + } + + public FlightPutResult(string applicationMetadata) + : this(ByteString.CopyFromUtf8(applicationMetadata)) + { + } + + internal FlightPutResult(Protocol.PutResult putResult) + { + _putResult = putResult; + } + + public ByteString ApplicationMetadata => _putResult.AppMetadata; + + internal Protocol.PutResult ToProtocol() + { + return _putResult; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamReader.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamReader.cs new file mode 100644 index 000000000..588127537 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamReader.cs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow.Flatbuf; +using Apache.Arrow.Flight.Internal; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Ipc; +using Google.Protobuf; +using Grpc.Core; + +namespace Apache.Arrow.Flight +{ + /// + /// Stream of record batches + /// + /// Use MoveNext() and Current to iterate over the batches. + /// There are also gRPC helper functions such as ToListAsync() etc. + /// + public abstract class FlightRecordBatchStreamReader : IAsyncStreamReader, IAsyncEnumerable, IDisposable + { + //Temporary until .NET 5.0 upgrade + private static ValueTask CompletedValueTask = new ValueTask(); + + private readonly RecordBatcReaderImplementation _arrowReaderImplementation; + + private protected FlightRecordBatchStreamReader(IAsyncStreamReader flightDataStream) + { + _arrowReaderImplementation = new RecordBatcReaderImplementation(flightDataStream); + } + + public ValueTask Schema => _arrowReaderImplementation.ReadSchema(); + + internal ValueTask GetFlightDescriptor() + { + return _arrowReaderImplementation.ReadFlightDescriptor(); + } + + /// + /// Get the application metadata from the latest recieved record batch + /// + public IReadOnlyList ApplicationMetadata => _arrowReaderImplementation.ApplicationMetadata; + + public RecordBatch Current { get; private set; } + + public async Task MoveNext(CancellationToken cancellationToken) + { + Current = await _arrowReaderImplementation.ReadNextRecordBatchAsync(cancellationToken); + + return Current != null; + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + return new AsyncEnumerator(this, cancellationToken); + } + + public void Dispose() + { + _arrowReaderImplementation.Dispose(); + } + + private class AsyncEnumerator : IAsyncEnumerator + { + private readonly FlightRecordBatchStreamReader _flightRecordBatchStreamReader; + private readonly CancellationToken _cancellationToken; + + internal AsyncEnumerator(FlightRecordBatchStreamReader flightRecordBatchStreamReader, CancellationToken cancellationToken) + { + _flightRecordBatchStreamReader = flightRecordBatchStreamReader; + _cancellationToken = cancellationToken; + } + + public RecordBatch Current => _flightRecordBatchStreamReader.Current; + + public async ValueTask MoveNextAsync() + { + return await _flightRecordBatchStreamReader.MoveNext(_cancellationToken); + } + + public ValueTask DisposeAsync() + { + _flightRecordBatchStreamReader.Dispose(); + return CompletedValueTask; + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs new file mode 100644 index 000000000..a72be5a82 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Internal; +using Apache.Arrow.Flight.Protocol; +using Google.Protobuf; +using Grpc.Core; + +namespace Apache.Arrow.Flight +{ + public abstract class FlightRecordBatchStreamWriter : IAsyncStreamWriter, IDisposable + { + private FlightDataStream _flightDataStream; + private readonly IAsyncStreamWriter _clientStreamWriter; + private readonly FlightDescriptor _flightDescriptor; + + private bool _disposed; + + private protected FlightRecordBatchStreamWriter(IAsyncStreamWriter clientStreamWriter, FlightDescriptor flightDescriptor) + { + _clientStreamWriter = clientStreamWriter; + _flightDescriptor = flightDescriptor; + } + + private void SetupStream(Schema schema) + { + _flightDataStream = new FlightDataStream(_clientStreamWriter, _flightDescriptor, schema); + } + + public WriteOptions WriteOptions { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + public Task WriteAsync(RecordBatch message) + { + return WriteAsync(message, default); + } + + public Task WriteAsync(RecordBatch message, ByteString applicationMetadata) + { + if (_flightDataStream == null) + { + SetupStream(message.Schema); + } + + return _flightDataStream.Write(message, applicationMetadata); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + _flightDataStream.Dispose(); + _disposed = true; + } + } + + public void Dispose() + { + Dispose(true); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightResult.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightResult.cs new file mode 100644 index 000000000..3ddadd4bc --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightResult.cs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightResult + { + private readonly Protocol.Result _result; + + internal FlightResult(Protocol.Result result) + { + _result = result; + } + + public FlightResult(ByteString body) + { + _result = new Protocol.Result() + { + Body = body + }; + } + + public FlightResult(string body) + : this(ByteString.CopyFromUtf8(body)) + { + } + + public FlightResult(byte[] body) + : this(ByteString.CopyFrom(body)) + { + } + + public ByteString Body => _result.Body; + + internal Protocol.Result ToProtocol() + { + return _result; + } + + public override bool Equals(object obj) + { + if(obj is FlightResult other) + { + return Equals(_result, other._result); + } + return false; + } + + public override int GetHashCode() + { + return _result.GetHashCode(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/FlightTicket.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightTicket.cs new file mode 100644 index 000000000..7b3d6dd75 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/FlightTicket.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf; + +namespace Apache.Arrow.Flight +{ + public class FlightTicket + { + private readonly Protocol.Ticket _ticket; + internal FlightTicket(Protocol.Ticket ticket) + { + _ticket = ticket; + } + + public FlightTicket(ByteString ticket) + { + _ticket = new Protocol.Ticket() + { + Ticket_ = ticket + }; + } + + public FlightTicket(string ticket) + : this(ByteString.CopyFromUtf8(ticket)) + { + } + + public FlightTicket(byte[] bytes) + : this(ByteString.CopyFrom(bytes)) + { + } + + public ByteString Ticket => _ticket.Ticket_; + + internal Protocol.Ticket ToProtocol() + { + return _ticket; + } + + public override bool Equals(object obj) + { + if(obj is FlightTicket other) + { + return Equals(_ticket, other._ticket); + } + return false; + } + + public override int GetHashCode() + { + return _ticket.GetHashCode(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs new file mode 100644 index 000000000..865884572 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow.Flatbuf; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Ipc; +using FlatBuffers; +using Google.Protobuf; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Internal +{ + /// + /// Handles writing record batches as flight data + /// + internal class FlightDataStream : ArrowStreamWriter + { + private readonly FlightDescriptor _flightDescriptor; + private readonly IAsyncStreamWriter _clientStreamWriter; + private Protocol.FlightData _currentFlightData; + + public FlightDataStream(IAsyncStreamWriter clientStreamWriter, FlightDescriptor flightDescriptor, Schema schema) + : base(new MemoryStream(), schema) + { + _clientStreamWriter = clientStreamWriter; + _flightDescriptor = flightDescriptor; + } + + private async Task SendSchema() + { + _currentFlightData = new Protocol.FlightData(); + + if(_flightDescriptor != null) + { + _currentFlightData.FlightDescriptor = _flightDescriptor.ToProtocol(); + } + + var offset = SerializeSchema(Schema); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + await WriteMessageAsync(MessageHeader.Schema, offset, 0, cancellationTokenSource.Token).ConfigureAwait(false); + await _clientStreamWriter.WriteAsync(_currentFlightData).ConfigureAwait(false); + HasWrittenSchema = true; + } + + private void ResetStream() + { + this.BaseStream.Position = 0; + this.BaseStream.SetLength(0); + } + + public async Task Write(RecordBatch recordBatch, ByteString applicationMetadata) + { + if (!HasWrittenSchema) + { + await SendSchema().ConfigureAwait(false); + } + ResetStream(); + + _currentFlightData = new Protocol.FlightData(); + + if(applicationMetadata != null) + { + _currentFlightData.AppMetadata = applicationMetadata; + } + + await WriteRecordBatchInternalAsync(recordBatch).ConfigureAwait(false); + + //Reset stream position + this.BaseStream.Position = 0; + var bodyData = await ByteString.FromStreamAsync(this.BaseStream).ConfigureAwait(false); + + _currentFlightData.DataBody = bodyData; + await _clientStreamWriter.WriteAsync(_currentFlightData).ConfigureAwait(false); + } + + private protected override ValueTask WriteMessageAsync(MessageHeader headerType, Offset headerOffset, int bodyLength, CancellationToken cancellationToken) + { + Offset messageOffset = Flatbuf.Message.CreateMessage( + Builder, CurrentMetadataVersion, headerType, headerOffset.Value, + bodyLength); + + Builder.Finish(messageOffset.Value); + + ReadOnlyMemory messageData = Builder.DataBuffer.ToReadOnlyMemory(Builder.DataBuffer.Position, Builder.Offset); + + _currentFlightData.DataHeader = ByteString.CopyFrom(messageData.Span); + + return new ValueTask(0); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs new file mode 100644 index 000000000..36b13a63d --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Apache.Arrow.Ipc; +using FlatBuffers; + +namespace Apache.Arrow.Flight +{ + internal static class FlightMessageSerializer + { + public static Schema DecodeSchema(ReadOnlyMemory buffer) + { + int bufferPosition = 0; + int schemaMessageLength = BinaryPrimitives.ReadInt32LittleEndian(buffer.Span.Slice(bufferPosition)); + bufferPosition += sizeof(int); + + if (schemaMessageLength == MessageSerializer.IpcContinuationToken) + { + // ARROW-6313, if the first 4 bytes are continuation message, read the next 4 for the length + if (buffer.Length <= bufferPosition + sizeof(int)) + { + throw new InvalidDataException("Corrupted IPC message. Received a continuation token at the end of the message."); + } + + schemaMessageLength = BinaryPrimitives.ReadInt32LittleEndian(buffer.Span.Slice(bufferPosition)); + bufferPosition += sizeof(int); + } + + ByteBuffer schemaBuffer = ArrowReaderImplementation.CreateByteBuffer(buffer.Slice(bufferPosition)); + //DictionaryBatch not supported for now + DictionaryMemo dictionaryMemo = null; + var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage(schemaBuffer), ref dictionaryMemo); + return schema; + } + + internal static Schema DecodeSchema(ByteBuffer schemaBuffer) + { + //DictionaryBatch not supported for now + DictionaryMemo dictionaryMemo = null; + var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage(schemaBuffer), ref dictionaryMemo); + return schema; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/RecordBatcReaderImplementation.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/RecordBatcReaderImplementation.cs new file mode 100644 index 000000000..10d4d731e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/RecordBatcReaderImplementation.cs @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow.Flatbuf; +using Apache.Arrow.Ipc; +using Google.Protobuf; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Internal +{ + internal class RecordBatcReaderImplementation : ArrowReaderImplementation + { + private readonly IAsyncStreamReader _flightDataStream; + private FlightDescriptor _flightDescriptor; + private readonly List _applicationMetadatas; + + public RecordBatcReaderImplementation(IAsyncStreamReader streamReader) + { + _flightDataStream = streamReader; + _applicationMetadatas = new List(); + } + + public override RecordBatch ReadNextRecordBatch() + { + throw new NotImplementedException(); + } + + public IReadOnlyList ApplicationMetadata => _applicationMetadatas; + + public async ValueTask ReadFlightDescriptor() + { + if (!HasReadSchema) + { + await ReadSchema().ConfigureAwait(false); + } + return _flightDescriptor; + } + + public async ValueTask ReadSchema() + { + if (HasReadSchema) + { + return Schema; + } + + var moveNextResult = await _flightDataStream.MoveNext().ConfigureAwait(false); + + if (!moveNextResult) + { + throw new Exception("No records or schema in this flight"); + } + + //AppMetadata will never be null, but length 0 if empty + //Those are skipped + if(_flightDataStream.Current.AppMetadata.Length > 0) + { + _applicationMetadatas.Add(_flightDataStream.Current.AppMetadata); + } + + var header = _flightDataStream.Current.DataHeader.Memory; + Message message = Message.GetRootAsMessage( + ArrowReaderImplementation.CreateByteBuffer(header)); + + + if(_flightDataStream.Current.FlightDescriptor != null) + { + _flightDescriptor = new FlightDescriptor(_flightDataStream.Current.FlightDescriptor); + } + + switch (message.HeaderType) + { + case MessageHeader.Schema: + Schema = FlightMessageSerializer.DecodeSchema(message.ByteBuffer); + break; + default: + throw new Exception($"Expected schema as the first message, but got: {message.HeaderType.ToString()}"); + } + return Schema; + } + + public override async ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken) + { + _applicationMetadatas.Clear(); //Clear any metadata from previous calls + + if (!HasReadSchema) + { + await ReadSchema().ConfigureAwait(false); + } + var moveNextResult = await _flightDataStream.MoveNext().ConfigureAwait(false); + if (moveNextResult) + { + //AppMetadata will never be null, but length 0 if empty + //Those are skipped + if (_flightDataStream.Current.AppMetadata.Length > 0) + { + _applicationMetadatas.Add(_flightDataStream.Current.AppMetadata); + } + + var header = _flightDataStream.Current.DataHeader.Memory; + Message message = Message.GetRootAsMessage(CreateByteBuffer(header)); + + switch (message.HeaderType) + { + case MessageHeader.RecordBatch: + var body = _flightDataStream.Current.DataBody.Memory; + return CreateArrowObjectFromMessage(message, CreateByteBuffer(body.Slice(0, (int)message.BodyLength)), null); + default: + throw new NotImplementedException(); + } + } + return null; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/SchemaWriter.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/SchemaWriter.cs new file mode 100644 index 000000000..c7e7d8135 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/SchemaWriter.cs @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow.Flatbuf; +using Apache.Arrow.Ipc; +using Google.Protobuf; + +namespace Apache.Arrow.Flight.Internal +{ + /// + /// This class handles writing schemas + /// + internal class SchemaWriter : ArrowStreamWriter + { + private SchemaWriter(Stream baseStream, Schema schema) : base(baseStream, schema) + { + } + + public void WriteSchema(Schema schema, CancellationToken cancellationToken) + { + var offset = base.SerializeSchema(schema); + WriteMessage(MessageHeader.Schema, offset, 0); + } + + public static ByteString SerializeSchema(Schema schema, CancellationToken cancellationToken = default(CancellationToken)) + { + using(var memoryStream = new MemoryStream()) + { + var writer = new SchemaWriter(memoryStream, schema); + writer.WriteSchema(schema, cancellationToken); + + memoryStream.Position = 0; + return ByteString.FromStream(memoryStream); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamReader.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamReader.cs new file mode 100644 index 000000000..a2c3db3d3 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamReader.cs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Internal +{ + + /// + /// This is a helper class that allows conversions from gRPC types to the Arrow types. + /// It maintains the stream so data can be read as soon as possible. + /// + /// In paramter from gRPC + /// The arrow type returned + internal class StreamReader : IAsyncStreamReader + { + private readonly IAsyncStreamReader _inputStream; + private readonly Func _convertFunction; + internal StreamReader(IAsyncStreamReader inputStream, Func convertFunction) + { + _inputStream = inputStream; + _convertFunction = convertFunction; + } + + public TOut Current { get; private set; } + + public async Task MoveNext(CancellationToken cancellationToken) + { + var moveNextResult = await _inputStream.MoveNext(cancellationToken).ConfigureAwait(false); + if (moveNextResult) + { + Current = _convertFunction(_inputStream.Current); + } + return moveNextResult; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamWriter.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamWriter.cs new file mode 100644 index 000000000..c50b41e1b --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Internal/StreamWriter.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Internal +{ + internal class StreamWriter : IAsyncStreamWriter + { + private readonly IAsyncStreamWriter _inputStream; + private readonly Func _convertFunction; + internal StreamWriter(IAsyncStreamWriter inputStream, Func convertFunction) + { + _inputStream = inputStream; + _convertFunction = convertFunction; + } + + public WriteOptions WriteOptions + { + get + { + return _inputStream.WriteOptions; + } + set + { + _inputStream.WriteOptions = value; + } + } + + public Task WriteAsync(TIn message) + { + return _inputStream.WriteAsync(_convertFunction(message)); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Properties/AssemblyInfo.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..07934ad05 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Properties/AssemblyInfo.cs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Apache.Arrow.Flight.AspNetCore, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServer.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServer.cs new file mode 100644 index 000000000..30b0409d4 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServer.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Server +{ + public abstract class FlightServer + { + public virtual Task DoPut(FlightServerRecordBatchStreamReader requestStream, IAsyncStreamWriter responseStream, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task DoGet(FlightTicket ticket, FlightServerRecordBatchStreamWriter responseStream, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task ListFlights(FlightCriteria request, IAsyncStreamWriter responseStream, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task ListActions(IAsyncStreamWriter responseStream, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task DoAction(FlightAction request, IAsyncStreamWriter responseStream, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task GetSchema(FlightDescriptor request, ServerCallContext context) + { + throw new NotImplementedException(); + } + + public virtual Task GetFlightInfo(FlightDescriptor request, ServerCallContext context) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamReader.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamReader.cs new file mode 100644 index 000000000..5476d3d0e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamReader.cs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Threading.Tasks; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Flight.Internal; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Server +{ + public class FlightServerRecordBatchStreamReader : FlightRecordBatchStreamReader + { + internal FlightServerRecordBatchStreamReader(IAsyncStreamReader flightDataStream) : base(flightDataStream) + { + } + + public ValueTask FlightDescriptor => GetFlightDescriptor(); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamWriter.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamWriter.cs new file mode 100644 index 000000000..6c1987339 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/FlightServerRecordBatchStreamWriter.cs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Flight.Internal; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Server +{ + public class FlightServerRecordBatchStreamWriter : FlightRecordBatchStreamWriter, IServerStreamWriter + { + internal FlightServerRecordBatchStreamWriter(IServerStreamWriter clientStreamWriter) : base(clientStreamWriter, null) + { + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow.Flight/Server/Internal/FlightServerImplementation.cs b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/Internal/FlightServerImplementation.cs new file mode 100644 index 000000000..dcf6e5768 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow.Flight/Server/Internal/FlightServerImplementation.cs @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Internal; +using Apache.Arrow.Flight.Protocol; +using Apache.Arrow.Flight.Server; +using Grpc.Core; + +namespace Apache.Arrow.Flight.Server.Internal +{ + /// + /// This class has to be internal, since the generated code from proto is set as internal. + /// + internal class FlightServerImplementation : FlightService.FlightServiceBase + { + private readonly FlightServer _flightServer; + public FlightServerImplementation(FlightServer flightServer) + { + _flightServer = flightServer; + } + + public override async Task DoPut(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + { + var readStream = new FlightServerRecordBatchStreamReader(requestStream); + var writeStream = new StreamWriter(responseStream, putResult => putResult.ToProtocol()); + await _flightServer.DoPut(readStream, writeStream, context).ConfigureAwait(false); + } + + public override Task DoGet(Protocol.Ticket request, IServerStreamWriter responseStream, ServerCallContext context) + { + return _flightServer.DoGet(new FlightTicket(request.Ticket_), new FlightServerRecordBatchStreamWriter(responseStream), context); + } + + public override Task ListFlights(Protocol.Criteria request, IServerStreamWriter responseStream, ServerCallContext context) + { + var writeStream = new StreamWriter(responseStream, flightInfo => flightInfo.ToProtocol()); + return _flightServer.ListFlights(new FlightCriteria(request), writeStream, context); + } + + public override Task DoAction(Protocol.Action request, IServerStreamWriter responseStream, ServerCallContext context) + { + var action = new FlightAction(request); + var writeStream = new StreamWriter(responseStream, result => result.ToProtocol()); + return _flightServer.DoAction(action, writeStream, context); + } + + public override async Task GetSchema(Protocol.FlightDescriptor request, ServerCallContext context) + { + var flightDescriptor = new FlightDescriptor(request); + var schema = await _flightServer.GetSchema(flightDescriptor, context).ConfigureAwait(false); + + return new SchemaResult() + { + Schema = SchemaWriter.SerializeSchema(schema) + }; + } + + public override async Task GetFlightInfo(Protocol.FlightDescriptor request, ServerCallContext context) + { + var flightDescriptor = new FlightDescriptor(request); + var flightInfo = await _flightServer.GetFlightInfo(flightDescriptor, context).ConfigureAwait(false); + + return flightInfo.ToProtocol(); + } + + public override Task DoExchange(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + { + //Exchange is not yet implemented + throw new NotImplementedException(); + } + + public override Task Handshake(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + { + //Handshake is not yet implemented + throw new NotImplementedException(); + } + + public override Task ListActions(Empty request, IServerStreamWriter responseStream, ServerCallContext context) + { + var writeStream = new StreamWriter(responseStream, (actionType) => actionType.ToProtocol()); + return _flightServer.ListActions(writeStream, context); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Apache.Arrow.csproj b/src/arrow/csharp/src/Apache.Arrow/Apache.Arrow.csproj new file mode 100644 index 000000000..62574029f --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Apache.Arrow.csproj @@ -0,0 +1,42 @@ + + + + netstandard1.3;netcoreapp2.1 + true + $(DefineConstants);UNSAFE_BYTEBUFFER;BYTEBUFFER_NO_BOUNDS_CHECK;ENABLE_SPAN_T + + Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic operations on modern hardware. + + + + + + + + + + + + + + True + True + Resources.resx + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + + + + + + + diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Array.cs new file mode 100644 index 000000000..a453b0807 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Array.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Runtime.CompilerServices; + +namespace Apache.Arrow +{ + public abstract class Array : IArrowArray + { + public ArrayData Data { get; } + + protected Array(ArrayData data) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + } + + public int Length => Data.Length; + + public int Offset => Data.Offset; + + public int NullCount => Data.NullCount; + + public ArrowBuffer NullBitmapBuffer => Data.Buffers[0]; + + public virtual void Accept(IArrowArrayVisitor visitor) + { + Accept(this, visitor); + } + + public bool IsValid(int index) => + NullCount == 0 || NullBitmapBuffer.IsEmpty || BitUtility.GetBit(NullBitmapBuffer.Span, index + Offset); + + public bool IsNull(int index) => !IsValid(index); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void Accept(T array, IArrowArrayVisitor visitor) + where T : class, IArrowArray + { + switch (visitor) + { + case IArrowArrayVisitor typedVisitor: + typedVisitor.Visit(array); + break; + default: + visitor.Visit(array); + break; + } + } + + public Array Slice(int offset, int length) + { + if (offset > Length) + { + throw new ArgumentException($"Offset {offset} cannot be greater than Length {Length} for Array.Slice"); + } + + length = Math.Min(Data.Length - offset, length); + offset += Data.Offset; + + ArrayData newData = Data.Slice(offset, length); + return ArrowArrayFactory.BuildArray(newData) as Array; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + Data.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayData.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayData.cs new file mode 100644 index 000000000..fb5aa1b5f --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayData.cs @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow +{ + public sealed class ArrayData : IDisposable + { + private const int RecalculateNullCount = -1; + + public readonly IArrowType DataType; + public readonly int Length; + public readonly int NullCount; + public readonly int Offset; + public readonly ArrowBuffer[] Buffers; + public readonly ArrayData[] Children; + public readonly ArrayData Dictionary; // Only used for dictionary type + + // This is left for compatibility with lower version binaries + // before the dictionary type was supported. + public ArrayData( + IArrowType dataType, + int length, int nullCount, int offset, + IEnumerable buffers, IEnumerable children) : + this(dataType, length, nullCount, offset, buffers, children, null) + { } + + // This is left for compatibility with lower version binaries + // before the dictionary type was supported. + public ArrayData( + IArrowType dataType, + int length, int nullCount, int offset, + ArrowBuffer[] buffers, ArrayData[] children) : + this(dataType, length, nullCount, offset, buffers, children, null) + { } + + public ArrayData( + IArrowType dataType, + int length, int nullCount = 0, int offset = 0, + IEnumerable buffers = null, IEnumerable children = null, ArrayData dictionary = null) + { + DataType = dataType ?? NullType.Default; + Length = length; + NullCount = nullCount; + Offset = offset; + Buffers = buffers?.ToArray(); + Children = children?.ToArray(); + Dictionary = dictionary; + } + + public ArrayData( + IArrowType dataType, + int length, int nullCount = 0, int offset = 0, + ArrowBuffer[] buffers = null, ArrayData[] children = null, ArrayData dictionary = null) + { + DataType = dataType ?? NullType.Default; + Length = length; + NullCount = nullCount; + Offset = offset; + Buffers = buffers; + Children = children; + Dictionary = dictionary; + } + + public void Dispose() + { + if (Buffers != null) + { + foreach (ArrowBuffer buffer in Buffers) + { + buffer.Dispose(); + } + } + + if (Children != null) + { + foreach (ArrayData child in Children) + { + child?.Dispose(); + } + } + + Dictionary?.Dispose(); + } + + public ArrayData Slice(int offset, int length) + { + if (offset > Length) + { + throw new ArgumentException($"Offset {offset} cannot be greater than Length {Length} for Array.Slice"); + } + + length = Math.Min(Length - offset, length); + offset += Offset; + + return new ArrayData(DataType, length, RecalculateNullCount, offset, Buffers, Children, Dictionary); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs new file mode 100644 index 000000000..0efb60ab8 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; + +namespace Apache.Arrow +{ + static class ArrayDataConcatenator + { + internal static ArrayData Concatenate(IReadOnlyList arrayDataList, MemoryAllocator allocator = default) + { + if (arrayDataList == null || arrayDataList.Count == 0) + { + return null; + } + + if (arrayDataList.Count == 1) + { + return arrayDataList[0]; + } + + var arrowArrayConcatenationVisitor = new ArrayDataConcatenationVisitor(arrayDataList, allocator); + + IArrowType type = arrayDataList[0].DataType; + type.Accept(arrowArrayConcatenationVisitor); + + return arrowArrayConcatenationVisitor.Result; + } + + private class ArrayDataConcatenationVisitor : + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor + { + public ArrayData Result { get; private set; } + private readonly IReadOnlyList _arrayDataList; + private readonly int _totalLength; + private readonly int _totalNullCount; + private readonly MemoryAllocator _allocator; + + public ArrayDataConcatenationVisitor(IReadOnlyList arrayDataList, MemoryAllocator allocator = default) + { + _arrayDataList = arrayDataList; + _allocator = allocator; + + foreach (ArrayData arrayData in _arrayDataList) + { + _totalLength += arrayData.Length; + _totalNullCount += arrayData.NullCount; + } + } + + public void Visit(BooleanType type) + { + CheckData(type, 2); + ArrowBuffer validityBuffer = ConcatenateValidityBuffer(); + ArrowBuffer valueBuffer = ConcatenateBitmapBuffer(1); + + Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, valueBuffer }); + } + + public void Visit(FixedWidthType type) + { + CheckData(type, 2); + ArrowBuffer validityBuffer = ConcatenateValidityBuffer(); + ArrowBuffer valueBuffer = ConcatenateFixedWidthTypeValueBuffer(type); + + Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, valueBuffer }); + } + + public void Visit(BinaryType type) => ConcatenateVariableBinaryArrayData(type); + + public void Visit(StringType type) => ConcatenateVariableBinaryArrayData(type); + + public void Visit(ListType type) + { + CheckData(type, 2); + ArrowBuffer validityBuffer = ConcatenateValidityBuffer(); + ArrowBuffer offsetBuffer = ConcatenateOffsetBuffer(); + ArrayData child = Concatenate(SelectChildren(0), _allocator); + + Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { child }); + } + + public void Visit(StructType type) + { + CheckData(type, 1); + List children = new List(type.Fields.Count); + + for (int i = 0; i < type.Fields.Count; i++) + { + children.Add(Concatenate(SelectChildren(i), _allocator)); + } + + Result = new ArrayData(type, _arrayDataList[0].Length, _arrayDataList[0].NullCount, 0, _arrayDataList[0].Buffers, children); + } + + public void Visit(IArrowType type) + { + throw new NotImplementedException($"Concatenation for {type.Name} is not supported yet."); + } + + private void CheckData(IArrowType type, int expectedBufferCount) + { + foreach (ArrayData arrayData in _arrayDataList) + { + arrayData.EnsureDataType(type.TypeId); + arrayData.EnsureBufferCount(expectedBufferCount); + } + } + + private void ConcatenateVariableBinaryArrayData(IArrowType type) + { + CheckData(type, 3); + ArrowBuffer validityBuffer = ConcatenateValidityBuffer(); + ArrowBuffer offsetBuffer = ConcatenateOffsetBuffer(); + ArrowBuffer valueBuffer = ConcatenateVariableBinaryValueBuffer(); + + Result = new ArrayData(type, _totalLength, _totalNullCount, 0, new ArrowBuffer[] { validityBuffer, offsetBuffer, valueBuffer }); + } + + private ArrowBuffer ConcatenateValidityBuffer() + { + if (_totalNullCount == 0) + { + return ArrowBuffer.Empty; + } + + return ConcatenateBitmapBuffer(0); + } + + private ArrowBuffer ConcatenateBitmapBuffer(int bufferIndex) + { + var builder = new ArrowBuffer.BitmapBuilder(_totalLength); + + foreach (ArrayData arrayData in _arrayDataList) + { + int length = arrayData.Length; + ReadOnlySpan span = arrayData.Buffers[bufferIndex].Span; + + for (int i = 0; i < length; i++) + { + builder.Append(span.IsEmpty || BitUtility.GetBit(span, i)); + } + } + + return builder.Build(_allocator); + } + + private ArrowBuffer ConcatenateFixedWidthTypeValueBuffer(FixedWidthType type) + { + int typeByteWidth = type.BitWidth / 8; + var builder = new ArrowBuffer.Builder(_totalLength * typeByteWidth); + + foreach (ArrayData arrayData in _arrayDataList) + { + int length = arrayData.Length; + int byteLength = length * typeByteWidth; + + builder.Append(arrayData.Buffers[1].Span.Slice(0, byteLength)); + } + + return builder.Build(_allocator); + } + + private ArrowBuffer ConcatenateVariableBinaryValueBuffer() + { + var builder = new ArrowBuffer.Builder(); + + foreach (ArrayData arrayData in _arrayDataList) + { + int lastOffset = arrayData.Buffers[1].Span.CastTo()[arrayData.Length]; + builder.Append(arrayData.Buffers[2].Span.Slice(0, lastOffset)); + } + + return builder.Build(_allocator); + } + + private ArrowBuffer ConcatenateOffsetBuffer() + { + var builder = new ArrowBuffer.Builder(_totalLength + 1); + int baseOffset = 0; + + builder.Append(0); + + foreach (ArrayData arrayData in _arrayDataList) + { + if (arrayData.Length == 0) + { + continue; + } + + // The first offset is always 0. + // It should be skipped because it duplicate to the last offset of builder. + ReadOnlySpan span = arrayData.Buffers[1].Span.CastTo().Slice(1, arrayData.Length); + + foreach (int offset in span) + { + builder.Append(baseOffset + offset); + } + + // The next offset must start from the current last offset. + baseOffset += span[arrayData.Length - 1]; + } + + return builder.Build(_allocator); + } + + private List SelectChildren(int index) + { + var children = new List(_arrayDataList.Count); + + foreach (ArrayData arrayData in _arrayDataList) + { + children.Add(arrayData.Children[index]); + } + + return children; + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs new file mode 100644 index 000000000..e7360942f --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + static class ArrowArrayBuilderFactory + { + internal static IArrowArrayBuilder> Build(IArrowType dataType) + { + switch (dataType.TypeId) + { + case ArrowTypeId.Boolean: + return new BooleanArray.Builder(); + case ArrowTypeId.UInt8: + return new UInt8Array.Builder(); + case ArrowTypeId.Int8: + return new Int8Array.Builder(); + case ArrowTypeId.UInt16: + return new UInt16Array.Builder(); + case ArrowTypeId.Int16: + return new Int16Array.Builder(); + case ArrowTypeId.UInt32: + return new UInt32Array.Builder(); + case ArrowTypeId.Int32: + return new Int32Array.Builder(); + case ArrowTypeId.UInt64: + return new UInt64Array.Builder(); + case ArrowTypeId.Int64: + return new Int64Array.Builder(); + case ArrowTypeId.Float: + return new FloatArray.Builder(); + case ArrowTypeId.Double: + return new DoubleArray.Builder(); + case ArrowTypeId.String: + return new StringArray.Builder(); + case ArrowTypeId.Binary: + return new BinaryArray.Builder(); + case ArrowTypeId.Timestamp: + return new TimestampArray.Builder(); + case ArrowTypeId.Date64: + return new Date64Array.Builder(); + case ArrowTypeId.Date32: + return new Date32Array.Builder(); + case ArrowTypeId.List: + return new ListArray.Builder(dataType as ListType); + case ArrowTypeId.Decimal128: + return new Decimal128Array.Builder(dataType as Decimal128Type); + case ArrowTypeId.Decimal256: + return new Decimal256Array.Builder(dataType as Decimal256Type); + case ArrowTypeId.Struct: + case ArrowTypeId.Union: + case ArrowTypeId.Dictionary: + case ArrowTypeId.FixedSizedBinary: + case ArrowTypeId.HalfFloat: + case ArrowTypeId.Interval: + case ArrowTypeId.Map: + case ArrowTypeId.Time32: + case ArrowTypeId.Time64: + default: + throw new NotSupportedException($"An ArrowArrayBuilder cannot be built for type {dataType.TypeId}."); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayConcatenator.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayConcatenator.cs new file mode 100644 index 000000000..cc151210a --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayConcatenator.cs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System.Collections.Generic; + +namespace Apache.Arrow +{ + static class ArrowArrayConcatenator + { + internal static IArrowArray Concatenate(IReadOnlyList arrowArrayList , MemoryAllocator allocator = default) + { + if(arrowArrayList == null || arrowArrayList.Count == 0) + { + return null; + } + + if (arrowArrayList.Count == 1) + { + return arrowArrayList[0]; + } + + var arrayDataList = new List(arrowArrayList.Count); + + foreach(IArrowArray array in arrowArrayList) + { + arrayDataList.Add(array.Data); + } + + return ArrowArrayFactory.BuildArray(ArrayDataConcatenator.Concatenate(arrayDataList, allocator)); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs new file mode 100644 index 000000000..2b74709e7 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public static class ArrowArrayFactory + { + public static IArrowArray BuildArray(ArrayData data) + { + switch (data.DataType.TypeId) + { + case ArrowTypeId.Boolean: + return new BooleanArray(data); + case ArrowTypeId.UInt8: + return new UInt8Array(data); + case ArrowTypeId.Int8: + return new Int8Array(data); + case ArrowTypeId.UInt16: + return new UInt16Array(data); + case ArrowTypeId.Int16: + return new Int16Array(data); + case ArrowTypeId.UInt32: + return new UInt32Array(data); + case ArrowTypeId.Int32: + return new Int32Array(data); + case ArrowTypeId.UInt64: + return new UInt64Array(data); + case ArrowTypeId.Int64: + return new Int64Array(data); + case ArrowTypeId.Float: + return new FloatArray(data); + case ArrowTypeId.Double: + return new DoubleArray(data); + case ArrowTypeId.String: + return new StringArray(data); + case ArrowTypeId.FixedSizedBinary: + return new FixedSizeBinaryArray(data); + case ArrowTypeId.Binary: + return new BinaryArray(data); + case ArrowTypeId.Timestamp: + return new TimestampArray(data); + case ArrowTypeId.List: + return new ListArray(data); + case ArrowTypeId.Struct: + return new StructArray(data); + case ArrowTypeId.Union: + return new UnionArray(data); + case ArrowTypeId.Date64: + return new Date64Array(data); + case ArrowTypeId.Date32: + return new Date32Array(data); + case ArrowTypeId.Decimal128: + return new Decimal128Array(data); + case ArrowTypeId.Decimal256: + return new Decimal256Array(data); + case ArrowTypeId.Dictionary: + return new DictionaryArray(data); + case ArrowTypeId.HalfFloat: + case ArrowTypeId.Interval: + case ArrowTypeId.Map: + case ArrowTypeId.Time32: + case ArrowTypeId.Time64: + default: + throw new NotSupportedException($"An ArrowArray cannot be built for type {data.DataType.TypeId}."); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs new file mode 100644 index 000000000..fc56b6601 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow +{ + public abstract class ArrowArrayVisitor : IArrowArrayVisitor + { + public virtual void Visit(IArrowArray array) { } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs new file mode 100644 index 000000000..4fd8059f6 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Apache.Arrow.Memory; + +namespace Apache.Arrow +{ + public class BinaryArray : Array + { + public class Builder : BuilderBase + { + public Builder() : base(BinaryType.Default) { } + public Builder(IArrowType dataType) : base(dataType) { } + + protected override BinaryArray Build(ArrayData data) + { + return new BinaryArray(data); + } + } + + public BinaryArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Binary); + data.EnsureBufferCount(3); + } + + public BinaryArray(ArrowTypeId typeId, ArrayData data) + : base(data) + { + data.EnsureDataType(typeId); + data.EnsureBufferCount(3); + } + + public abstract class BuilderBase : IArrowArrayBuilder + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + protected IArrowType DataType { get; } + protected TBuilder Instance => this as TBuilder; + protected ArrowBuffer.Builder ValueOffsets { get; } + protected ArrowBuffer.Builder ValueBuffer { get; } + protected ArrowBuffer.BitmapBuilder ValidityBuffer { get; } + protected int Offset { get; set; } + protected int NullCount => this.ValidityBuffer.UnsetBitCount; + + protected BuilderBase(IArrowType dataType) + { + DataType = dataType; + ValueOffsets = new ArrowBuffer.Builder(); + ValueBuffer = new ArrowBuffer.Builder(); + ValidityBuffer = new ArrowBuffer.BitmapBuilder(); + + // From the docs: + // + // The offsets buffer contains length + 1 signed integers (either 32-bit or 64-bit, depending on the + // logical type), which encode the start position of each slot in the data buffer. The length of the + // value in each slot is computed using the difference between the offset at that slot’s index and the + // subsequent offset. + // + // In this builder, we choose to append the first offset (zero) upon construction, and each trailing + // offset is then added after each individual item has been appended. + ValueOffsets.Append(this.Offset); + } + + protected abstract TArray Build(ArrayData data); + + /// + /// Gets the length of the array built so far. + /// + public int Length => ValueOffsets.Length - 1; + + /// + /// Build an Arrow array from the appended contents so far. + /// + /// Optional memory allocator. + /// Returns an array of type . + public TArray Build(MemoryAllocator allocator = default) + { + var bufs = new[] + { + NullCount > 0 ? ValidityBuffer.Build(allocator) : ArrowBuffer.Empty, + ValueOffsets.Build(allocator), + ValueBuffer.Build(allocator), + }; + var data = new ArrayData( + DataType, + length: Length, + NullCount, + offset: 0, + bufs); + + return Build(data); + } + + /// + /// Append a single null value to the array. + /// + /// Returns the builder (for fluent-style composition). + public TBuilder AppendNull() + { + // Do not add to the value buffer in the case of a null. + // Note that we do not need to increment the offset as a result. + ValidityBuffer.Append(false); + ValueOffsets.Append(Offset); + return Instance; + } + + /// + /// Appends a value, consisting of a single byte, to the array. + /// + /// Byte value to append. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(byte value) + { + ValueBuffer.Append(value); + ValidityBuffer.Append(true); + Offset++; + ValueOffsets.Append(Offset); + return Instance; + } + + /// + /// Append a value, consisting of a span of bytes, to the array. + /// + /// + /// Note that a single value is added, which consists of arbitrarily many bytes. If multiple values are + /// to be added, use the method. + /// + /// Span of bytes to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(ReadOnlySpan span) + { + ValueBuffer.Append(span); + ValidityBuffer.Append(true); + Offset += span.Length; + ValueOffsets.Append(Offset); + return Instance; + } + + /// + /// Append a value, consisting of an enumerable collection of bytes, to the array. + /// + /// + /// Note that this method appends a single value, which may consist of arbitrarily many bytes. If multiple + /// values are to be added, use the method instead. + /// + /// Enumerable collection of bytes to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(IEnumerable value) + { + if (value == null) + { + return AppendNull(); + } + + // Note: by looking at the length of the value buffer before and after, we avoid having to iterate + // through the enumerable multiple times to get both length and contents. + int priorLength = ValueBuffer.Length; + ValueBuffer.AppendRange(value); + int valueLength = ValueBuffer.Length - priorLength; + Offset += valueLength; + ValidityBuffer.Append(true); + ValueOffsets.Append(Offset); + return Instance; + } + + /// + /// Append an enumerable collection of single-byte values to the array. + /// + /// + /// Note that this method appends multiple values, each of which is a single byte. If a single value is + /// to be added, use the method instead. + /// + /// Single-byte values to add. + /// Returns the builder (for fluent-style composition). + public TBuilder AppendRange(IEnumerable values) + { + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + foreach (byte b in values) + { + Append(b); + } + + return Instance; + } + + /// + /// Append an enumerable collection of values to the array. + /// + /// Values to add. + /// Returns the builder (for fluent-style composition). + public TBuilder AppendRange(IEnumerable values) + { + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + foreach (byte[] arr in values) + { + if (arr == null) + { + AppendNull(); + } + else + { + Append((ReadOnlySpan)arr); + } + } + + return Instance; + } + + public TBuilder Reserve(int capacity) + { + // TODO: [ARROW-9366] Reserve capacity in the value buffer in a more sensible way. + ValueOffsets.Reserve(capacity + 1); + ValueBuffer.Reserve(capacity); + ValidityBuffer.Reserve(capacity + 1); + return Instance; + } + + public TBuilder Resize(int length) + { + // TODO: [ARROW-9366] Resize the value buffer to a safe length based on offsets, not `length`. + ValueOffsets.Resize(length + 1); + ValueBuffer.Resize(length); + ValidityBuffer.Resize(length + 1); + return Instance; + } + + public TBuilder Swap(int i, int j) + { + // TODO: Implement + throw new NotImplementedException(); + } + + public TBuilder Set(int index, byte value) + { + // TODO: Implement + throw new NotImplementedException(); + } + + /// + /// Clear all contents appended so far. + /// + /// Returns the builder (for fluent-style composition). + public TBuilder Clear() + { + ValueOffsets.Clear(); + ValueBuffer.Clear(); + ValidityBuffer.Clear(); + + // Always write the first offset before anything has been written. + Offset = 0; + ValueOffsets.Append(Offset); + return Instance; + } + } + + public BinaryArray(IArrowType dataType, int length, + ArrowBuffer valueOffsetsBuffer, + ArrowBuffer dataBuffer, + ArrowBuffer nullBitmapBuffer, + int nullCount = 0, int offset = 0) + : this(new ArrayData(dataType, length, nullCount, offset, + new[] { nullBitmapBuffer, valueOffsetsBuffer, dataBuffer })) + { } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1]; + + public ArrowBuffer ValueBuffer => Data.Buffers[2]; + + public ReadOnlySpan ValueOffsets => ValueOffsetsBuffer.Span.CastTo().Slice(Offset, Length + 1); + + public ReadOnlySpan Values => ValueBuffer.Span.CastTo(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [Obsolete("This method has been deprecated. Please use ValueOffsets[index] instead.")] + public int GetValueOffset(int index) + { + if (index < 0 || index > Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + return ValueOffsets[index]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetValueLength(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + if (!IsValid(index)) + { + return 0; + } + + ReadOnlySpan offsets = ValueOffsets; + return offsets[index + 1] - offsets[index]; + } + + /// + /// Get the collection of bytes, as a read-only span, at a given index in the array. + /// + /// + /// Note that this method cannot reliably identify null values, which are indistinguishable from empty byte + /// collection values when seen in the context of this method's return type of . + /// Use the method instead to reliably determine null values. + /// + /// Index at which to get bytes. + /// Returns a object. + /// If the index is negative or beyond the length of the array. + /// + public ReadOnlySpan GetBytes(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (IsNull(index)) + { + // Note that `return null;` is valid syntax, but would be misleading as `null` in the context of a span + // is actually returned as an empty span. + return ReadOnlySpan.Empty; + } + + return ValueBuffer.Span.Slice(ValueOffsets[index], GetValueLength(index)); + } + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs new file mode 100644 index 000000000..0915338fe --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; + +namespace Apache.Arrow +{ + public class BooleanArray: Array + { + public class Builder : IArrowArrayBuilder + { + private ArrowBuffer.BitmapBuilder ValueBuffer { get; } + private ArrowBuffer.BitmapBuilder ValidityBuffer { get; } + + public int Length => ValueBuffer.Length; + public int Capacity => ValueBuffer.Capacity; + public int NullCount => ValidityBuffer.UnsetBitCount; + + public Builder() + { + ValueBuffer = new ArrowBuffer.BitmapBuilder(); + ValidityBuffer = new ArrowBuffer.BitmapBuilder(); + } + + public Builder Append(bool value) + { + return NullableAppend(value); + } + + public Builder NullableAppend(bool? value) + { + // Note that we rely on the fact that null values are false in the value buffer. + ValueBuffer.Append(value ?? false); + ValidityBuffer.Append(value.HasValue); + return this; + } + + public Builder Append(ReadOnlySpan span) + { + foreach (bool value in span) + { + Append(value); + } + return this; + } + + public Builder AppendRange(IEnumerable values) + { + foreach (bool value in values) + { + Append(value); + } + return this; + } + + public Builder AppendNull() + { + return NullableAppend(null); + } + + public BooleanArray Build(MemoryAllocator allocator = default) + { + ArrowBuffer validityBuffer = NullCount > 0 + ? ValidityBuffer.Build(allocator) + : ArrowBuffer.Empty; + + return new BooleanArray( + ValueBuffer.Build(allocator), validityBuffer, + Length, NullCount, 0); + } + + public Builder Clear() + { + ValueBuffer.Clear(); + ValidityBuffer.Clear(); + return this; + } + + public Builder Reserve(int capacity) + { + if (capacity < 0) + { + throw new ArgumentOutOfRangeException(nameof(capacity)); + } + + ValueBuffer.Reserve(capacity); + ValidityBuffer.Reserve(capacity); + return this; + } + + public Builder Resize(int length) + { + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + ValueBuffer.Resize(length); + ValidityBuffer.Resize(length); + return this; + } + + public Builder Toggle(int index) + { + CheckIndex(index); + + // If there is a null at this index, assume it was set to false in the value buffer, and so becomes + // true/non-null after toggling. + ValueBuffer.Toggle(index); + ValidityBuffer.Set(index); + return this; + } + + public Builder Set(int index) + { + CheckIndex(index); + ValueBuffer.Set(index); + ValidityBuffer.Set(index); + return this; + } + + public Builder Set(int index, bool value) + { + CheckIndex(index); + ValueBuffer.Set(index, value); + ValidityBuffer.Set(index); + return this; + } + + public Builder Swap(int i, int j) + { + CheckIndex(i); + CheckIndex(j); + ValueBuffer.Swap(i, j); + ValidityBuffer.Swap(i, j); + return this; + } + + private void CheckIndex(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public ArrowBuffer ValueBuffer => Data.Buffers[1]; + public ReadOnlySpan Values => ValueBuffer.Span.Slice(0, (int) Math.Ceiling(Length / 8.0)); + + public BooleanArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(BooleanType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public BooleanArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Boolean); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + [Obsolete("GetBoolean does not support null values. Use GetValue instead (which this method invokes internally).")] + public bool GetBoolean(int index) + { + return GetValue(index).GetValueOrDefault(); + } + + public bool? GetValue(int index) + { + return IsNull(index) + ? (bool?)null + : BitUtility.GetBit(ValueBuffer.Span, index + Offset); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Date32Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Date32Array.cs new file mode 100644 index 000000000..35c0065e1 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Date32Array.cs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + /// + /// The class holds an array of dates in the Date32 format, where each date is + /// stored as the number of days since the dawn of (UNIX) time. + /// + public class Date32Array : PrimitiveArray + { + private static readonly DateTime _epochDate = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + + /// + /// The class can be used to fluently build objects. + /// + public class Builder : DateArrayBuilder + { + private class DateBuilder : PrimitiveArrayBuilder + { + protected override Date32Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Date32Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + /// + /// Construct a new instance of the class. + /// + public Builder() : base(new DateBuilder()) { } + + protected override int Convert(DateTime dateTime) + { + return (int)(dateTime.Date - _epochDate).TotalDays; + } + + protected override int Convert(DateTimeOffset dateTimeOffset) + { + // The internal value stored for a DateTimeOffset can be thought of as the number of 24-hour "blocks" + // of time that have elapsed since the UNIX epoch. This is the same as converting it to UTC first and + // then taking the date element from that. It is not the same as what would result from looking at the + // DateTimeOffset.Date property. + return (int)(dateTimeOffset.UtcDateTime.Date - _epochDate).TotalDays; + } + } + + public Date32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Date32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Date32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Date32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + [Obsolete("Use `GetDateTimeOffset()` instead")] + public DateTimeOffset? GetDate(int index) => GetDateTimeOffset(index); + + /// + /// Get the date at the specified index in the form of a object. + /// + /// + /// The property of the returned object is set to + /// . + /// + /// Index at which to get the date. + /// Returns a object, or null if there is no object at that index. + /// + public DateTime? GetDateTime(int index) + { + int? value = GetValue(index); + return value.HasValue + ? _epochDate.AddDays(value.Value) + : default(DateTime?); + } + + /// + /// Get the date at the specified index in the form of a object. + /// + /// Index at which to get the date. + /// Returns a object, or null if there is no object at that index. + /// + public DateTimeOffset? GetDateTimeOffset(int index) + { + int? value = GetValue(index); + return value.HasValue + ? new DateTimeOffset(_epochDate.AddDays(value.Value), TimeSpan.Zero) + : default(DateTimeOffset?); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Date64Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Date64Array.cs new file mode 100644 index 000000000..cf977b2e4 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Date64Array.cs @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + /// + /// The class holds an array of dates in the Date64 format, where each date is + /// stored as the number of milliseconds since the dawn of (UNIX) time, excluding leap seconds, in multiples of + /// 86400000. + /// + public class Date64Array: PrimitiveArray + { + private const long MillisecondsPerDay = 86400000; + + public Date64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Date64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + /// + /// The class can be used to fluently build objects. + /// + public class Builder : DateArrayBuilder + { + private class DateBuilder: PrimitiveArrayBuilder + { + protected override Date64Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Date64Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + /// + /// Construct a new instance of the class. + /// + public Builder() : base(new DateBuilder()) { } + + protected override long Convert(DateTime dateTime) + { + var dateTimeOffset = new DateTimeOffset( + DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified), + TimeSpan.Zero); + return dateTimeOffset.ToUnixTimeMilliseconds(); + } + + protected override long Convert(DateTimeOffset dateTimeOffset) + { + // The internal value stored for a DateTimeOffset can be thought of as the number of milliseconds, + // in multiples of 86400000, that have passed since the UNIX epoch. It is not the same as what would + // result from encoding the date from the DateTimeOffset.Date property. + long millis = dateTimeOffset.ToUnixTimeMilliseconds(); + long days = millis / MillisecondsPerDay; + return (millis < 0 ? days - 1 : days) * MillisecondsPerDay; + } + } + + public Date64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Date64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + [Obsolete("Use `GetDateTimeOffset()` instead")] + public DateTimeOffset? GetDate(int index) => GetDateTimeOffset(index); + + /// + /// Get the date at the specified index in the form of a object. + /// + /// + /// The property of the returned object is set to + /// . + /// + /// Index at which to get the date. + /// Returns a object, or null if there is no object at that index. + /// + public DateTime? GetDateTime(int index) + { + long? value = GetValue(index); + return value.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value).Date + : default(DateTime?); + } + + /// + /// Get the date at the specified index in the form of a object. + /// + /// Index at which to get the date. + /// Returns a object, or null if there is no object at that index. + /// + public DateTimeOffset? GetDateTimeOffset(int index) + { + long? value = GetValue(index); + return value.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value) + : default(DateTimeOffset?); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs new file mode 100644 index 000000000..4e69f6fe3 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow +{ + /// + /// The class is an abstract array builder that can + /// accept dates in the form of or and convert to some + /// underlying date representation. + /// + public abstract class DateArrayBuilder : + DelegatingArrayBuilder, + IArrowArrayBuilder, + IArrowArrayBuilder + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + /// + /// Construct a new instance of the class. + /// + /// Inner builder that will produce arrays of type . + /// + protected DateArrayBuilder(IArrowArrayBuilder> innerBuilder) + : base(innerBuilder) + { } + + /// + /// Append a date in the form of a object to the array. + /// + /// + /// The value of on the input does not have any effect on the behaviour of this + /// method. + /// + /// Date to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(DateTime value) + { + InnerBuilder.Append(Convert(value)); + return this as TBuilder; + } + + /// + /// Append a date from a object to the array. + /// + /// + /// Note that to convert the supplied parameter to a date, it is first converted to + /// UTC and the date then taken from the UTC date/time. Depending on the value of its + /// property, this may not necessarily be the same as the date obtained by + /// calling its property. + /// + /// Date to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(DateTimeOffset value) + { + InnerBuilder.Append(Convert(value)); + return this as TBuilder; + } + + /// + /// Append a span of dates in the form of objects to the array. + /// + /// + /// The value of on any of the inputs does not have any effect on the behaviour of + /// this method. + /// + /// Span of dates to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(ReadOnlySpan span) + { + InnerBuilder.Reserve(span.Length); + foreach (var item in span) + { + InnerBuilder.Append(Convert(item)); + } + + return this as TBuilder; + } + + /// + /// Append a span of dates in the form of objects to the array. + /// + /// + /// Note that to convert the objects in the parameter to + /// dates, they are first converted to UTC and the date then taken from the UTC date/times. Depending on the + /// value of each property, this may not necessarily be the same as the + /// date obtained by calling the property. + /// + /// Span of dates to add. + /// Returns the builder (for fluent-style composition). + public TBuilder Append(ReadOnlySpan span) + { + InnerBuilder.Reserve(span.Length); + foreach (var item in span) + { + InnerBuilder.Append(Convert(item)); + } + + return this as TBuilder; + } + + /// + /// Append a null date to the array. + /// + /// Returns the builder (for fluent-style composition). + public TBuilder AppendNull() + { + InnerBuilder.AppendNull(); + return this as TBuilder; + } + + /// + /// Append a collection of dates in the form of objects to the array. + /// + /// + /// The value of on any of the inputs does not have any effect on the behaviour of + /// this method. + /// + /// Collection of dates to add. + /// Returns the builder (for fluent-style composition). + public TBuilder AppendRange(IEnumerable values) + { + InnerBuilder.AppendRange(values.Select(Convert)); + return this as TBuilder; + } + + /// + /// Append a collection of dates in the form of objects to the array. + /// + /// + /// Note that to convert the objects in the parameter to + /// dates, they are first converted to UTC and the date then taken from the UTC date/times. Depending on the + /// value of each property, this may not necessarily be the same as the + /// date obtained by calling the property. + /// + /// Collection of dates to add. + /// Returns the builder (for fluent-style composition). + public TBuilder AppendRange(IEnumerable values) + { + InnerBuilder.AppendRange(values.Select(Convert)); + return this as TBuilder; + } + + /// + /// Set the value of a date in the form of a object at the specified index. + /// + /// + /// The value of on the input does not have any effect on the behaviour of this + /// method. + /// + /// Index at which to set value. + /// Date to set. + /// Returns the builder (for fluent-style composition). + public TBuilder Set(int index, DateTime value) + { + InnerBuilder.Set(index, Convert(value)); + return this as TBuilder; + } + + /// + /// Set the value of a date in the form of a object at the specified index. + /// + /// + /// Note that to convert the supplied parameter to a date, it is first converted to + /// UTC and the date then taken from the UTC date/time. Depending on the value of its + /// property, this may not necessarily be the same as the date obtained by + /// calling its property. + /// + /// Index at which to set value. + /// Date to set. + /// Returns the builder (for fluent-style composition). + public TBuilder Set(int index, DateTimeOffset value) + { + InnerBuilder.Set(index, Convert(value)); + return this as TBuilder; + } + + /// + /// Swap the values of the dates at the specified indices. + /// + /// First index. + /// Second index. + /// Returns the builder (for fluent-style composition). + public TBuilder Swap(int i, int j) + { + InnerBuilder.Swap(i, j); + return this as TBuilder; + } + + protected abstract TUnderlying Convert(DateTime dateTime); + + protected abstract TUnderlying Convert(DateTimeOffset dateTimeOffset); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal128Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal128Array.cs new file mode 100644 index 000000000..128e9e5f0 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal128Array.cs @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Decimal128Array : FixedSizeBinaryArray + { + public class Builder : BuilderBase + { + public Builder(Decimal128Type type) : base(type, 16) + { + DataType = type; + } + + protected new Decimal128Type DataType { get; } + + protected override Decimal128Array Build(ArrayData data) + { + return new Decimal128Array(data); + } + + public Builder Append(decimal value) + { + Span bytes = stackalloc byte[DataType.ByteWidth]; + DecimalUtility.GetBytes(value, DataType.Precision, DataType.Scale, DataType.ByteWidth, bytes); + + return Append(bytes); + } + + public Builder AppendRange(IEnumerable values) + { + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + foreach (decimal d in values) + { + Append(d); + } + + return Instance; + } + + public Builder Set(int index, decimal value) + { + Span bytes = stackalloc byte[DataType.ByteWidth]; + DecimalUtility.GetBytes(value, DataType.Precision, DataType.Scale, DataType.ByteWidth, bytes); + + return Set(index, bytes); + } + } + + public Decimal128Array(ArrayData data) + : base(ArrowTypeId.Decimal128, data) + { + data.EnsureDataType(ArrowTypeId.Decimal128); + data.EnsureBufferCount(2); + Debug.Assert(Data.DataType is Decimal128Type); + } + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public int Scale => ((Decimal128Type)Data.DataType).Scale; + public int Precision => ((Decimal128Type)Data.DataType).Precision; + public int ByteWidth => ((Decimal128Type)Data.DataType).ByteWidth; + + public decimal? GetValue(int index) + { + if (IsNull(index)) + { + return null; + } + return DecimalUtility.GetDecimal(ValueBuffer, index, Scale, ByteWidth); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal256Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal256Array.cs new file mode 100644 index 000000000..fb4cd6be3 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Decimal256Array.cs @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Decimal256Array : FixedSizeBinaryArray + { + public class Builder : BuilderBase + { + public Builder(Decimal256Type type) : base(type, 32) + { + DataType = type; + } + + protected new Decimal256Type DataType { get; } + + protected override Decimal256Array Build(ArrayData data) + { + return new Decimal256Array(data); + } + + public Builder Append(decimal value) + { + Span bytes = stackalloc byte[DataType.ByteWidth]; + DecimalUtility.GetBytes(value, DataType.Precision, DataType.Scale, DataType.ByteWidth, bytes); + + return Append(bytes); + } + + public Builder AppendRange(IEnumerable values) + { + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + foreach (decimal d in values) + { + Append(d); + } + + return Instance; + } + + public Builder Set(int index, decimal value) + { + Span bytes = stackalloc byte[DataType.ByteWidth]; + DecimalUtility.GetBytes(value, DataType.Precision, DataType.Scale, DataType.ByteWidth, bytes); + + return Set(index, bytes); + } + } + + public Decimal256Array(ArrayData data) + : base(ArrowTypeId.Decimal256, data) + { + data.EnsureDataType(ArrowTypeId.Decimal256); + data.EnsureBufferCount(2); + Debug.Assert(Data.DataType is Decimal256Type); + } + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public int Scale => ((Decimal256Type)Data.DataType).Scale; + public int Precision => ((Decimal256Type)Data.DataType).Precision; + public int ByteWidth => ((Decimal256Type)Data.DataType).ByteWidth; + + public decimal? GetValue(int index) + { + if (IsNull(index)) + { + return null; + } + + return DecimalUtility.GetDecimal(ValueBuffer, index, Scale, ByteWidth); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs new file mode 100644 index 000000000..f2ab3ee13 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow.Memory; + +namespace Apache.Arrow +{ + /// + /// The class can be used as the base for any array builder + /// that needs to delegate most of its functionality to an inner array builder. + /// + /// + /// The typical use case is when an array builder may accept a number of different types as input, but which are + /// all internally converted to a single type for assembly into an array. + /// + /// Type of item accepted by inner array builder. + /// Type of array produced by this (and the inner) builder. + /// Type of builder (see Curiously-Recurring Template Pattern). + public abstract class DelegatingArrayBuilder : IArrowArrayBuilder + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + /// + /// Gets the inner array builder. + /// + protected IArrowArrayBuilder> InnerBuilder { get; } + + /// + /// Gets the number of items added to the array so far. + /// + public int Length => InnerBuilder.Length; + + /// + /// Construct a new instance of the class. + /// + /// Inner array builder. + protected DelegatingArrayBuilder(IArrowArrayBuilder> innerBuilder) + { + InnerBuilder = innerBuilder ?? throw new ArgumentNullException(nameof(innerBuilder)); + } + + /// + /// Build an Arrow Array from the appended contents so far. + /// + /// Optional memory allocator. + /// Returns the built array. + public TArray Build(MemoryAllocator allocator = default) => InnerBuilder.Build(allocator); + + /// + /// Reserve a given number of items' additional capacity. + /// + /// Number of items of required additional capacity. + /// Returns the builder (for fluent-style composition). + public TBuilder Reserve(int additionalCapacity) + { + InnerBuilder.Reserve(additionalCapacity); + return this as TBuilder; + } + + /// + /// Resize the array to a given size. + /// + /// + /// Note that if the required capacity is larger than the current length of the populated array so far, + /// the array's contents in the new, expanded region are undefined. + /// + /// + /// Note that if the required capacity is smaller than the current length of the populated array so far, + /// the array will be truncated and items at the end of the array will be lost. + /// + /// Number of items of required capacity. + /// Returns the builder (for fluent-style composition). + public TBuilder Resize(int capacity) + { + InnerBuilder.Resize(capacity); + return this as TBuilder; + } + + /// + /// Clear all contents appended so far. + /// + /// Returns the builder (for fluent-style composition). + public TBuilder Clear() + { + InnerBuilder.Clear(); + return this as TBuilder; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/DictionaryArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/DictionaryArray.cs new file mode 100644 index 000000000..29c0f5c84 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/DictionaryArray.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class DictionaryArray : Array + { + public IArrowArray Dictionary { get; } + public IArrowArray Indices { get; } + public ArrowBuffer IndicesBuffer => Data.Buffers[1]; + + public DictionaryArray(ArrayData data) : base(data) + { + data.EnsureBufferCount(2); + data.EnsureDataType(ArrowTypeId.Dictionary); + + if (data.Dictionary == null) + { + throw new ArgumentException($"{nameof(data.Dictionary)} must not be null"); + } + + var dicType = (DictionaryType)data.DataType; + data.Dictionary.EnsureDataType(dicType.ValueType.TypeId); + + var indicesData = new ArrayData(dicType.IndexType, data.Length, data.NullCount, data.Offset, data.Buffers, data.Children); + + Indices = ArrowArrayFactory.BuildArray(indicesData); + Dictionary = ArrowArrayFactory.BuildArray(data.Dictionary); + } + + public DictionaryArray(DictionaryType dataType, IArrowArray indicesArray, IArrowArray dictionary) : + base(new ArrayData(dataType, indicesArray.Length, indicesArray.Data.NullCount, indicesArray.Data.Offset, indicesArray.Data.Buffers, indicesArray.Data.Children, dictionary.Data)) + { + Data.EnsureBufferCount(2); + + indicesArray.Data.EnsureDataType(dataType.IndexType.TypeId); + dictionary.Data.EnsureDataType(dataType.ValueType.TypeId); + + Indices = indicesArray; + Dictionary = dictionary; + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs new file mode 100644 index 000000000..6450aa140 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class DoubleArray : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override DoubleArray Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new DoubleArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public DoubleArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(DoubleType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public DoubleArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Double); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/FixedSizeBinaryArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/FixedSizeBinaryArray.cs new file mode 100644 index 000000000..7d3d87547 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/FixedSizeBinaryArray.cs @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Arrays +{ + public class FixedSizeBinaryArray : Array + { + public FixedSizeBinaryArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.FixedSizedBinary); + data.EnsureBufferCount(2); + } + + public FixedSizeBinaryArray(ArrowTypeId typeId, ArrayData data) + : base(data) + { + data.EnsureDataType(typeId); + data.EnsureBufferCount(2); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public ArrowBuffer ValueBuffer => Data.Buffers[1]; + + /// + /// Get the collection of bytes, as a read-only span, at a given index in the array. + /// + /// + /// Note that this method cannot reliably identify null values, which are indistinguishable from empty byte + /// collection values when seen in the context of this method's return type of . + /// Use the method instead to reliably determine null values. + /// + /// Index at which to get bytes. + /// Returns a object. + /// If the index is negative or beyond the length of the array. + /// + public ReadOnlySpan GetBytes(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (IsNull(index)) + { + // Note that `return null;` is valid syntax, but would be misleading as `null` in the context of a span + // is actually returned as an empty span. + return ReadOnlySpan.Empty; + } + + int size = ((FixedSizeBinaryType)Data.DataType).ByteWidth; + return ValueBuffer.Span.Slice(index * size, size); + } + + public abstract class BuilderBase : IArrowArrayBuilder + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + protected IArrowType DataType { get; } + protected TBuilder Instance => this as TBuilder; + protected int ByteWidth { get; } + protected ArrowBuffer.Builder ValueBuffer { get; } + protected ArrowBuffer.BitmapBuilder ValidityBuffer { get; } + public int Length => ValueBuffer.Length / ByteWidth; + protected int NullCount => this.ValidityBuffer.UnsetBitCount; + protected abstract TArray Build(ArrayData data); + + protected BuilderBase(IArrowType dataType, int byteWidth) + { + DataType = dataType; + ByteWidth = byteWidth; + ValueBuffer = new ArrowBuffer.Builder(); + ValidityBuffer = new ArrowBuffer.BitmapBuilder(); + } + + public TArray Build(MemoryAllocator allocator = default) + { + var bufs = new[] + { + NullCount > 0 ? ValidityBuffer.Build(allocator) : ArrowBuffer.Empty, + ValueBuffer.Build(ByteWidth, allocator), + }; + var data = new ArrayData( + DataType, + Length, + NullCount, + 0, + bufs); + + return Build(data); + } + + public TBuilder Reserve(int capacity) + { + ValueBuffer.Reserve(capacity * ByteWidth); + ValidityBuffer.Reserve(capacity + 1); + return Instance; + } + + public TBuilder Resize(int length) + { + ValueBuffer.Resize(length * ByteWidth); + ValidityBuffer.Resize(length + 1); + return Instance; + } + + public TBuilder Clear() { + + ValueBuffer.Clear(); + ValidityBuffer.Clear(); + + return Instance; + } + + public TBuilder Append(byte[] value) + { + if(value.Length % ByteWidth != 0) + throw new ArgumentOutOfRangeException("Bytes of length: " + value.Length + " do not conform to the fixed size: " + ByteWidth); + return Append(value.AsSpan()); + } + public TBuilder Append(ReadOnlySpan span) + { + foreach (var b in span) + { + Append(b); + } + + return Instance; + } + + public TBuilder AppendRange(IEnumerable values) + { + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + foreach (byte[] b in values) + { + Append(b); + } + + return Instance; + } + + public TBuilder Append(ReadOnlySpan span) + { + ValueBuffer.Append(span); + ValidityBuffer.Append(true); + return Instance; + } + + public TBuilder AppendNull() + { + ValueBuffer.Append(new byte[ByteWidth]); + ValidityBuffer.Append(false); + return Instance; + } + + public TBuilder Swap(int i, int j) + { + int iStart = i * ByteWidth; + int jStart = j * ByteWidth; + byte[] iBytes = ValueBuffer.Span.Slice(iStart, ByteWidth).ToArray(); + Span jBytes = ValueBuffer.Span.Slice(jStart, ByteWidth); + + for (int m = 0; m < ByteWidth; m++) + { + ValueBuffer.Span[iStart + m] = jBytes[m]; + ValueBuffer.Span[jStart + m] = iBytes[m]; + } + + ValidityBuffer.Swap(i, j); + return Instance; + } + + public TBuilder Set(int index, byte[] value) + { + return Set(index, value.AsSpan()); + } + + public TBuilder Set(int index, ReadOnlySpan value) + { + int startIndex = index * ByteWidth; + for (int i = 0; i < ByteWidth; i++) + { + ValueBuffer.Span[startIndex + i] = value[i]; + } + + ValidityBuffer.Set(index, true); + return Instance; + } + + public TBuilder SetNull(int index) + { + int startIndex = index * ByteWidth; + for (int i = 0; i < ByteWidth; i++) + { + ValueBuffer.Span[startIndex + i] = 0; + } + + ValidityBuffer.Set(index, false); + return Instance; + } + + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/FloatArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/FloatArray.cs new file mode 100644 index 000000000..8feca3233 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/FloatArray.cs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class FloatArray : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override FloatArray Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new FloatArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public FloatArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(FloatType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public FloatArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Float); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Int16Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int16Array.cs new file mode 100644 index 000000000..0401865c9 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int16Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int16Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override Int16Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Int16Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public Int16Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int16Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int16Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int16); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Int32Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int32Array.cs new file mode 100644 index 000000000..ef356c7a6 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int32Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int32Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override Int32Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Int32Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public Int32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Int64Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int64Array.cs new file mode 100644 index 000000000..fe8fbc62a --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int64Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int64Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override Int64Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Int64Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public Int64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/Int8Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int8Array.cs new file mode 100644 index 000000000..58d543a10 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/Int8Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int8Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override Int8Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new Int8Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public Int8Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int8Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int8Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int8); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/ListArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/ListArray.cs new file mode 100644 index 000000000..97673cb48 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/ListArray.cs @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class ListArray : Array + { + public class Builder : IArrowArrayBuilder + { + public IArrowArrayBuilder> ValueBuilder { get; } + + public int Length => ValueOffsetsBufferBuilder.Length; + + private ArrowBuffer.Builder ValueOffsetsBufferBuilder { get; } + + private ArrowBuffer.BitmapBuilder ValidityBufferBuilder { get; } + + public int NullCount { get; protected set; } + + private IArrowType DataType { get; } + + public Builder(IArrowType valueDataType) : this(new ListType(valueDataType)) + { + } + + public Builder(Field valueField) : this(new ListType(valueField)) + { + } + + internal Builder(ListType dataType) + { + ValueBuilder = ArrowArrayBuilderFactory.Build(dataType.ValueDataType); + ValueOffsetsBufferBuilder = new ArrowBuffer.Builder(); + ValidityBufferBuilder = new ArrowBuffer.BitmapBuilder(); + DataType = dataType; + } + + /// + /// Start a new variable-length list slot + /// + /// This function should be called before beginning to append elements to the + /// value builder + /// + /// + public Builder Append() + { + ValueOffsetsBufferBuilder.Append(ValueBuilder.Length); + ValidityBufferBuilder.Append(true); + + return this; + } + + public Builder AppendNull() + { + ValueOffsetsBufferBuilder.Append(ValueBuilder.Length); + ValidityBufferBuilder.Append(false); + NullCount++; + + return this; + } + + public ListArray Build(MemoryAllocator allocator = default) + { + ValueOffsetsBufferBuilder.Append(ValueBuilder.Length); + + ArrowBuffer validityBuffer = NullCount > 0 + ? ValidityBufferBuilder.Build(allocator) + : ArrowBuffer.Empty; + + return new ListArray(DataType, Length - 1, + ValueOffsetsBufferBuilder.Build(allocator), ValueBuilder.Build(allocator), + validityBuffer, NullCount, 0); + } + + public Builder Reserve(int capacity) + { + ValueOffsetsBufferBuilder.Reserve(capacity + 1); + ValidityBufferBuilder.Reserve(capacity + 1); + return this; + } + + public Builder Resize(int length) + { + ValueOffsetsBufferBuilder.Resize(length + 1); + ValidityBufferBuilder.Resize(length + 1); + return this; + } + + public Builder Clear() + { + ValueOffsetsBufferBuilder.Clear(); + ValueBuilder.Clear(); + ValidityBufferBuilder.Clear(); + return this; + } + + } + + public IArrowArray Values { get; } + + public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1]; + + public ReadOnlySpan ValueOffsets => ValueOffsetsBuffer.Span.CastTo().Slice(Offset, Length + 1); + + public ListArray(IArrowType dataType, int length, + ArrowBuffer valueOffsetsBuffer, IArrowArray values, + ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0) + : this(new ArrayData(dataType, length, nullCount, offset, + new[] { nullBitmapBuffer, valueOffsetsBuffer }, new[] { values.Data }), + values) + { + } + + public ListArray(ArrayData data) + : this(data, ArrowArrayFactory.BuildArray(data.Children[0])) + { + } + + private ListArray(ArrayData data, IArrowArray values) : base(data) + { + data.EnsureBufferCount(2); + data.EnsureDataType(ArrowTypeId.List); + Values = values; + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + + [Obsolete("This method has been deprecated. Please use ValueOffsets[index] instead.")] + public int GetValueOffset(int index) + { + if (index < 0 || index > Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + return ValueOffsets[index]; + } + + public int GetValueLength(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (IsNull(index)) + { + return 0; + } + + ReadOnlySpan offsets = ValueOffsets; + return offsets[index + 1] - offsets[index]; + } + + public IArrowArray GetSlicedValues(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (IsNull(index)) + { + return null; + } + + if (!(Values is Array array)) + { + return default; + } + + return array.Slice(ValueOffsets[index], GetValueLength(index)); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + Values?.Dispose(); + } + base.Dispose(disposing); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs new file mode 100644 index 000000000..7365a77b6 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Apache.Arrow +{ + public abstract class PrimitiveArray : Array + where T : struct + { + protected PrimitiveArray(ArrayData data) + : base(data) + { + data.EnsureBufferCount(2); + } + + public ArrowBuffer ValueBuffer => Data.Buffers[1]; + + public ReadOnlySpan Values => ValueBuffer.Span.CastTo().Slice(Offset, Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T? GetValue(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + return IsValid(index) ? Values[index] : (T?)null; + } + + public IList ToList(bool includeNulls = false) + { + ReadOnlySpan span = Values; + var list = new List(span.Length); + + for (int i = 0; i < span.Length; i++) + { + T? value = GetValue(i); + + if (value.HasValue) + { + list.Add(value.Value); + } + else + { + if (includeNulls) + { + list.Add(null); + } + } + } + + return list; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs new file mode 100644 index 000000000..326f04558 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow +{ + public abstract class PrimitiveArrayBuilder : IArrowArrayBuilder + where TTo : struct + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + protected TBuilder Instance => this as TBuilder; + protected IArrowArrayBuilder> ArrayBuilder { get; } + + public int Length => ArrayBuilder.Length; + + internal PrimitiveArrayBuilder(IArrowArrayBuilder> builder) + { + ArrayBuilder = builder ?? throw new ArgumentNullException(nameof(builder)); + } + + public TArray Build(MemoryAllocator allocator = default) => ArrayBuilder.Build(allocator); + + public TBuilder Append(TFrom value) + { + ArrayBuilder.Append(ConvertTo(value)); + return Instance; + } + + public TBuilder Append(ReadOnlySpan span) + { + ArrayBuilder.Reserve(span.Length); + foreach (TFrom value in span) + { + Append(value); + } + return Instance; + } + + public TBuilder AppendRange(IEnumerable values) + { + ArrayBuilder.AppendRange(values.Select(ConvertTo)); + return Instance; + } + + public TBuilder AppendNull() + { + ArrayBuilder.AppendNull(); + return Instance; + } + + public TBuilder Reserve(int capacity) + { + ArrayBuilder.Reserve(capacity); + return Instance; + } + + public TBuilder Resize(int length) + { + ArrayBuilder.Resize(length); + return Instance; + } + + public TBuilder Swap(int i, int j) + { + ArrayBuilder.Swap(i, j); + return Instance; + } + + public TBuilder Set(int index, TFrom value) + { + ArrayBuilder.Set(index, ConvertTo(value)); + return Instance; + } + + public TBuilder Clear() + { + ArrayBuilder.Clear(); + return Instance; + } + + protected abstract TTo ConvertTo(TFrom value); + } + + public abstract class PrimitiveArrayBuilder : IArrowArrayBuilder + where T : struct + where TArray : IArrowArray + where TBuilder : class, IArrowArrayBuilder + { + protected TBuilder Instance => this as TBuilder; + protected ArrowBuffer.Builder ValueBuffer { get; } + protected ArrowBuffer.BitmapBuilder ValidityBuffer { get; } + + public int Length => ValueBuffer.Length; + protected int NullCount => ValidityBuffer.UnsetBitCount; + + internal PrimitiveArrayBuilder() + { + ValueBuffer = new ArrowBuffer.Builder(); + ValidityBuffer = new ArrowBuffer.BitmapBuilder(); + } + + public TBuilder Resize(int length) + { + ValueBuffer.Resize(length); + ValidityBuffer.Resize(length); + return Instance; + } + + public TBuilder Reserve(int capacity) + { + ValueBuffer.Reserve(capacity); + ValidityBuffer.Reserve(capacity); + return Instance; + } + + public TBuilder Append(T value) + { + ValueBuffer.Append(value); + ValidityBuffer.Append(true); + return Instance; + } + + public TBuilder Append(ReadOnlySpan span) + { + int len = ValueBuffer.Length; + ValueBuffer.Append(span); + ValidityBuffer.AppendRange(Enumerable.Repeat(true, ValueBuffer.Length - len)); + return Instance; + } + + public TBuilder AppendRange(IEnumerable values) + { + int len = ValueBuffer.Length; + ValueBuffer.AppendRange(values); + ValidityBuffer.AppendRange(Enumerable.Repeat(true, ValueBuffer.Length - len)); + return Instance; + } + + public TBuilder AppendNull() + { + ValidityBuffer.Append(false); + ValueBuffer.Append(default(T)); + return Instance; + } + + public TBuilder Clear() + { + ValueBuffer.Clear(); + ValidityBuffer.Clear(); + return Instance; + } + + public TBuilder Set(int index, T value) + { + ValueBuffer.Span[index] = value; + ValidityBuffer.Set(index, true); + return Instance; + } + + public TBuilder Swap(int i, int j) + { + T x = ValueBuffer.Span[i]; + ValueBuffer.Span[i] = ValueBuffer.Span[j]; + ValueBuffer.Span[j] = x; + ValidityBuffer.Swap(i, j); + return Instance; + } + + public TArray Build(MemoryAllocator allocator = default) + { + ArrowBuffer validityBuffer = NullCount > 0 + ? ValidityBuffer.Build(allocator) + : ArrowBuffer.Empty; + + return Build( + ValueBuffer.Build(allocator), validityBuffer, + ValueBuffer.Length, NullCount, 0); + } + + protected abstract TArray Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/StringArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/StringArray.cs new file mode 100644 index 000000000..f008f56fa --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/StringArray.cs @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace Apache.Arrow +{ + public class StringArray: BinaryArray + { + public static readonly Encoding DefaultEncoding = Encoding.UTF8; + + public new class Builder : BuilderBase + { + public Builder() : base(StringType.Default) { } + + protected override StringArray Build(ArrayData data) + { + return new StringArray(data); + } + + public Builder Append(string value, Encoding encoding = null) + { + if (value == null) + { + return AppendNull(); + } + encoding = encoding ?? DefaultEncoding; + byte[] span = encoding.GetBytes(value); + return Append(span.AsSpan()); + } + + public Builder AppendRange(IEnumerable values, Encoding encoding = null) + { + foreach (string value in values) + { + Append(value, encoding); + } + + return this; + } + } + + public StringArray(ArrayData data) + : base(ArrowTypeId.String, data) { } + + public StringArray(int length, + ArrowBuffer valueOffsetsBuffer, + ArrowBuffer dataBuffer, + ArrowBuffer nullBitmapBuffer, + int nullCount = 0, int offset = 0) + : this(new ArrayData(StringType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueOffsetsBuffer, dataBuffer })) + { } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public string GetString(int index, Encoding encoding = default) + { + encoding = encoding ?? DefaultEncoding; + + ReadOnlySpan bytes = GetBytes(index); + + if (bytes == default) + { + return null; + } + if (bytes.Length == 0) + { + return string.Empty; + } + + unsafe + { + fixed (byte* data = &MemoryMarshal.GetReference(bytes)) + return encoding.GetString(data, bytes.Length); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/StructArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/StructArray.cs new file mode 100644 index 000000000..31aea9b41 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/StructArray.cs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Apache.Arrow +{ + public class StructArray : Array + { + private IReadOnlyList _fields; + + public IReadOnlyList Fields => + LazyInitializer.EnsureInitialized(ref _fields, () => InitializeFields()); + + public StructArray( + IArrowType dataType, int length, + IEnumerable children, + ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0) + : this(new ArrayData( + dataType, length, nullCount, offset, new[] { nullBitmapBuffer }, + children.Select(child => child.Data))) + { + _fields = children.ToArray(); + } + + public StructArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Struct); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + private IReadOnlyList InitializeFields() + { + IArrowArray[] result = new IArrowArray[Data.Children.Length]; + for (int i = 0; i < Data.Children.Length; i++) + { + result[i] = ArrowArrayFactory.BuildArray(Data.Children[i]); + } + return result; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs new file mode 100644 index 000000000..0269768f4 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Diagnostics; +using System.IO; + +namespace Apache.Arrow +{ + public class TimestampArray: PrimitiveArray + { + private static readonly DateTimeOffset s_epoch = new DateTimeOffset(1970, 1, 1, 0, 0, 0, 0, TimeSpan.Zero); + + public class Builder: PrimitiveArrayBuilder + { + internal class TimestampBuilder : PrimitiveArrayBuilder + { + internal TimestampBuilder(TimestampType type) + { + DataType = type ?? throw new ArgumentNullException(nameof(type)); + } + + protected TimestampType DataType { get; } + + protected override TimestampArray Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new TimestampArray(DataType, valueBuffer, nullBitmapBuffer, + length, nullCount, offset); + } + + protected TimestampType DataType { get; } + + public Builder() + : this(TimestampType.Default) { } + + public Builder(TimeUnit unit, TimeZoneInfo timezone) + : this(new TimestampType(unit, timezone)) { } + + public Builder(TimeUnit unit = TimeUnit.Millisecond, string timezone = "+00:00") + : this(new TimestampType(unit, timezone)) { } + + public Builder(TimeUnit unit) + : this(new TimestampType(unit, (string) null)) { } + + public Builder(TimestampType type) + : base(new TimestampBuilder(type)) + { + DataType = type; + } + + protected override long ConvertTo(DateTimeOffset value) + { + // We must return the absolute time since the UNIX epoch while + // respecting the timezone offset; the calculation is as follows: + // + // - Compute time span between epoch and specified time + // - Compute time divisions per tick + + TimeSpan timeSpan = value - s_epoch; + long ticks = timeSpan.Ticks; + + switch (DataType.Unit) + { + case TimeUnit.Nanosecond: + return ticks * 100; + case TimeUnit.Microsecond: + return ticks / 10; + case TimeUnit.Millisecond: + return ticks / TimeSpan.TicksPerMillisecond; + case TimeUnit.Second: + return ticks / TimeSpan.TicksPerSecond; + default: + throw new InvalidOperationException($"unsupported time unit <{DataType.Unit}>"); + } + } + } + + public TimestampArray( + TimestampType type, + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(type, length, nullCount, offset, + new[] {nullBitmapBuffer, valueBuffer})) { } + + public TimestampArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Timestamp); + + Debug.Assert(Data.DataType is TimestampType); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public DateTimeOffset GetTimestampUnchecked(int index) + { + var type = (TimestampType) Data.DataType; + long value = Values[index]; + + long ticks; + + switch (type.Unit) + { + case TimeUnit.Nanosecond: + ticks = value / 100; + break; + case TimeUnit.Microsecond: + ticks = value * 10; + break; + case TimeUnit.Millisecond: + ticks = value * TimeSpan.TicksPerMillisecond; + break; + case TimeUnit.Second: + ticks = value * TimeSpan.TicksPerSecond; + break; + default: + throw new InvalidDataException( + $"Unsupported timestamp unit <{type.Unit}>"); + } + + return new DateTimeOffset(s_epoch.Ticks + ticks, TimeSpan.Zero); + } + + public DateTimeOffset? GetTimestamp(int index) + { + if (IsNull(index)) + { + return null; + } + + return GetTimestampUnchecked(index); + } + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs new file mode 100644 index 000000000..bba244fe8 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt16Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override UInt16Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new UInt16Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public UInt16Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt16Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt16Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt16); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs new file mode 100644 index 000000000..65320be6f --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt32Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override UInt32Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new UInt32Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public UInt32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs new file mode 100644 index 000000000..617949fab --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt64Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override UInt64Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new UInt64Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public UInt64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs new file mode 100644 index 000000000..5cde7918c --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt8Array : PrimitiveArray + { + public class Builder : PrimitiveArrayBuilder + { + protected override UInt8Array Build( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) => + new UInt8Array(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + + public UInt8Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt8Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) { } + + public UInt8Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt8); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Arrays/UnionArray.cs b/src/arrow/csharp/src/Apache.Arrow/Arrays/UnionArray.cs new file mode 100644 index 000000000..8bccea2b5 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Arrays/UnionArray.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public class UnionArray: Array + { + public UnionType Type => Data.DataType as UnionType; + + public UnionMode Mode => Type.Mode; + + public ArrowBuffer TypeBuffer => Data.Buffers[1]; + + public ArrowBuffer ValueOffsetBuffer => Data.Buffers[2]; + + public ReadOnlySpan TypeIds => TypeBuffer.Span; + + public ReadOnlySpan ValueOffsets => ValueOffsetBuffer.Span.CastTo().Slice(0, Length + 1); + + public UnionArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Union); + data.EnsureBufferCount(3); + } + + public IArrowArray GetChild(int index) + { + // TODO: Implement + throw new NotImplementedException(); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.BitmapBuilder.cs b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.BitmapBuilder.cs new file mode 100644 index 000000000..c27ef35e8 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.BitmapBuilder.cs @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Apache.Arrow.Memory; + +namespace Apache.Arrow +{ + public partial struct ArrowBuffer + { + /// + /// The class is a complement to + /// and is designed for boolean fields, which are efficiently bit-packed into byte-aligned memory. + /// + public class BitmapBuilder + { + private const int DefaultBitCapacity = 64; + + /// + /// Gets the number of bits that can be contained in the memory allocated by the current instance. + /// + public int Capacity { get; private set; } + + /// + /// Gets the number of bits currently appended. + /// + public int Length { get; private set; } + + /// + /// Gets the raw byte memory underpinning the builder. + /// + public Memory Memory { get; private set; } + + /// + /// Gets the span of (bit-packed byte) memory underpinning the builder. + /// + public Span Span => Memory.Span; + + /// + /// Gets the number of set bits (i.e. set to 1). + /// + public int SetBitCount { get; private set; } + + /// + /// Gets the number of unset bits (i.e. set to 0). + /// + public int UnsetBitCount => Length - SetBitCount; + + /// + /// Creates an instance of the class. + /// + /// Number of bits of initial capacity to reserve. + public BitmapBuilder(int capacity = DefaultBitCapacity) + { + Memory = new byte[BitUtility.ByteCount(capacity)]; + Capacity = capacity; + } + + /// + /// Append a single bit. + /// + /// Bit to append. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Append(bool value) + { + if (Length % 8 == 0) + { + // Append a new byte to the buffer when needed. + EnsureAdditionalCapacity(1); + } + + BitUtility.SetBit(Span, Length, value); + Length++; + SetBitCount += value ? 1 : 0; + return this; + } + + /// + /// Append multiple bits. + /// + /// Bits to append. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder AppendRange(IEnumerable values) + { + if (values != null) + { + foreach (var v in values) + { + Append(v); + } + } + + return this; + } + + /// + /// Toggle the bit at a particular index. + /// + /// Index of bit to toggle. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Toggle(int index) + { + CheckIndex(index); + bool priorValue = BitUtility.GetBit(Span, index); + SetBitCount += priorValue ? -1 : 1; + BitUtility.ToggleBit(Span, index); + return this; + } + + /// + /// Set the bit at a particular index to 1. + /// + /// Index of bit to set. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Set(int index) + { + CheckIndex(index); + bool priorValue = BitUtility.GetBit(Span, index); + SetBitCount += priorValue ? 0 : 1; + BitUtility.SetBit(Span, index); + return this; + } + + /// + /// Set the bit at a particular index to a given value. + /// + /// Index of bit to set/unset. + /// Value of bit. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Set(int index, bool value) + { + CheckIndex(index); + bool priorValue = BitUtility.GetBit(Span, index); + SetBitCount -= priorValue ? 1 : 0; + SetBitCount += value ? 1 : 0; + BitUtility.SetBit(Span, index, value); + return this; + } + + /// + /// Swap the bits at two given indices. + /// + /// First index. + /// Second index. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Swap(int i, int j) + { + CheckIndex(i); + CheckIndex(j); + bool bi = BitUtility.GetBit(Span, i); + bool bj = BitUtility.GetBit(Span, j); + BitUtility.SetBit(Span, i, bj); + BitUtility.SetBit(Span, j, bi); + return this; + } + + /// + /// Reserve a given number of bits' additional capacity. + /// + /// Number of bits of required additional capacity. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Reserve(int additionalCapacity) + { + if (additionalCapacity < 0) + { + throw new ArgumentOutOfRangeException(nameof(additionalCapacity)); + } + + EnsureAdditionalCapacity(additionalCapacity); + return this; + } + + /// + /// Resize the buffer to a given size. + /// + /// + /// Note that if the required capacity is larger than the current length of the populated buffer so far, + /// the buffer's contents in the new, expanded region are undefined. + /// + /// + /// Note that if the required capacity is smaller than the current length of the populated buffer so far, + /// the buffer will be truncated and items at the end of the buffer will be lost. + /// + /// Number of bits of required capacity. + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Resize(int capacity) + { + if (capacity < 0) + { + throw new ArgumentOutOfRangeException(nameof(capacity), "Capacity must be non-negative"); + } + + EnsureCapacity(capacity); + Length = capacity; + + SetBitCount = BitUtility.CountBits(Span, 0, Length); + + return this; + } + + /// + /// Clear all contents appended so far. + /// + /// Returns the builder (for fluent-style composition). + public BitmapBuilder Clear() + { + Span.Fill(default); + Length = 0; + SetBitCount = 0; + return this; + } + + /// + /// Build an Arrow buffer from the appended contents so far. + /// + /// Optional memory allocator. + /// Returns an object. + public ArrowBuffer Build(MemoryAllocator allocator = default) + { + int bufferLength = checked((int)BitUtility.RoundUpToMultipleOf64(Memory.Length)); + var memoryAllocator = allocator ?? MemoryAllocator.Default.Value; + var memoryOwner = memoryAllocator.Allocate(bufferLength); + Memory.Slice(0, Memory.Length).CopyTo(memoryOwner.Memory); + return new ArrowBuffer(memoryOwner); + } + + private void CheckIndex(int index) + { + if (index < 0 || index >= Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + } + + private void EnsureAdditionalCapacity(int additionalCapacity) + { + EnsureCapacity(checked(Length + additionalCapacity)); + } + + private void EnsureCapacity(int requiredCapacity) + { + if (requiredCapacity > Capacity) + { + // TODO: specifiable growth strategy + // Double the length of the in-memory array, or use the byte count of the capacity, whichever is + // greater. + int byteCount = Math.Max(BitUtility.ByteCount(requiredCapacity), Memory.Length * 2); + Reallocate(byteCount); + Capacity = byteCount * 8; + } + } + + private void Reallocate(int numBytes) + { + if (numBytes != 0) + { + Debug.Assert(numBytes > Memory.Length); + var memory = new Memory(new byte[numBytes]); + Memory.CopyTo(memory); + + Memory = memory; + } + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs new file mode 100644 index 000000000..7c03027fe --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Apache.Arrow +{ + public partial struct ArrowBuffer + { + /// + /// The class is able to append value-type items, with fluent-style methods, to build + /// up an of contiguous items. + /// + /// + /// Note that is not supported as a generic type argument for this class. Please use + /// instead. + /// + /// Value-type of item to build into a buffer. + public class Builder + where T : struct + { + private const int DefaultCapacity = 8; + + private readonly int _size; + + /// + /// Gets the number of items that can be contained in the memory allocated by the current instance. + /// + public int Capacity => Memory.Length / _size; + + /// + /// Gets the number of items currently appended. + /// + public int Length { get; private set; } + + /// + /// Gets the raw byte memory underpinning the builder. + /// + public Memory Memory { get; private set; } + + /// + /// Gets the span of memory underpinning the builder. + /// + public Span Span + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Memory.Span.CastTo(); + } + + /// + /// Creates an instance of the class. + /// + /// Number of items of initial capacity to reserve. + public Builder(int capacity = DefaultCapacity) + { + // Using `bool` as the template argument, if used in an unrestricted fashion, would result in a buffer + // with inappropriate contents being produced. Because C# does not support template specialisation, + // and because generic type constraints do not support negation, we will throw a runtime error to + // indicate that such a template type is not supported. + if (typeof(T) == typeof(bool)) + { + throw new NotSupportedException( + $"An instance of {nameof(Builder)} cannot be instantiated, as `bool` is not an " + + $"appropriate generic type to use with this class - please use {nameof(BitmapBuilder)} " + + $"instead"); + } + + _size = Unsafe.SizeOf(); + + Memory = new byte[capacity * _size]; + Length = 0; + } + + /// + /// Append a buffer, assumed to contain items of the same type. + /// + /// Buffer to append. + /// Returns the builder (for fluent-style composition). + public Builder Append(ArrowBuffer buffer) + { + Append(buffer.Span.CastTo()); + return this; + } + + /// + /// Append a single item. + /// + /// Item to append. + /// Returns the builder (for fluent-style composition). + public Builder Append(T value) + { + EnsureAdditionalCapacity(1); + Span[Length++] = value; + return this; + } + + /// + /// Append a span of items. + /// + /// Source of item span. + /// Returns the builder (for fluent-style composition). + public Builder Append(ReadOnlySpan source) + { + EnsureAdditionalCapacity(source.Length); + source.CopyTo(Span.Slice(Length, source.Length)); + Length += source.Length; + return this; + } + + /// + /// Append a number of items. + /// + /// Items to append. + /// Returns the builder (for fluent-style composition). + public Builder AppendRange(IEnumerable values) + { + if (values != null) + { + foreach (T v in values) + { + Append(v); + } + } + + return this; + } + + /// + /// Reserve a given number of items' additional capacity. + /// + /// Number of items of required additional capacity. + /// Returns the builder (for fluent-style composition). + public Builder Reserve(int additionalCapacity) + { + if (additionalCapacity < 0) + { + throw new ArgumentOutOfRangeException(nameof(additionalCapacity)); + } + + EnsureAdditionalCapacity(additionalCapacity); + return this; + } + + /// + /// Resize the buffer to a given size. + /// + /// + /// Note that if the required capacity is larger than the current length of the populated buffer so far, + /// the buffer's contents in the new, expanded region are undefined. + /// + /// + /// Note that if the required capacity is smaller than the current length of the populated buffer so far, + /// the buffer will be truncated and items at the end of the buffer will be lost. + /// + /// Number of items of required capacity. + /// Returns the builder (for fluent-style composition). + public Builder Resize(int capacity) + { + if (capacity < 0) + { + throw new ArgumentOutOfRangeException(nameof(capacity), "Capacity must be non-negative"); + } + + EnsureCapacity(capacity); + Length = capacity; + + return this; + } + + /// + /// Clear all contents appended so far. + /// + /// Returns the builder (for fluent-style composition). + public Builder Clear() + { + Span.Fill(default); + Length = 0; + return this; + } + + /// + /// Build an Arrow buffer from the appended contents so far. + /// + /// Optional memory allocator. + /// Returns an object. + public ArrowBuffer Build(MemoryAllocator allocator = default) + { + return Build(64, allocator); + } + + /// + /// Build an Arrow buffer from the appended contents so far of the specified byte size. + /// + /// Optional memory allocator. + /// Returns an object. + internal ArrowBuffer Build(int byteSize, MemoryAllocator allocator = default) + { + int currentBytesLength = Length * _size; + int bufferLength = checked((int)BitUtility.RoundUpToMultiplePowerOfTwo(currentBytesLength, byteSize)); + + MemoryAllocator memoryAllocator = allocator ?? MemoryAllocator.Default.Value; + IMemoryOwner memoryOwner = memoryAllocator.Allocate(bufferLength); + Memory.Slice(0, currentBytesLength).CopyTo(memoryOwner.Memory); + + return new ArrowBuffer(memoryOwner); + } + + private void EnsureAdditionalCapacity(int additionalCapacity) + { + EnsureCapacity(checked(Length + additionalCapacity)); + } + + private void EnsureCapacity(int requiredCapacity) + { + if (requiredCapacity > Capacity) + { + // TODO: specifiable growth strategy + // Double the length of the in-memory array, or use the byte count of the capacity, whichever is + // greater. + int capacity = Math.Max(requiredCapacity * _size, Memory.Length * 2); + Reallocate(capacity); + } + } + + private void Reallocate(int numBytes) + { + if (numBytes != 0) + { + var memory = new Memory(new byte[numBytes]); + Memory.CopyTo(memory); + + Memory = memory; + } + } + + } + + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.cs b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.cs new file mode 100644 index 000000000..f8e675921 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/ArrowBuffer.cs @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using Apache.Arrow.Memory; + +namespace Apache.Arrow +{ + public readonly partial struct ArrowBuffer : IEquatable, IDisposable + { + private readonly IMemoryOwner _memoryOwner; + private readonly ReadOnlyMemory _memory; + + public static ArrowBuffer Empty => new ArrowBuffer(Memory.Empty); + + public ArrowBuffer(ReadOnlyMemory data) + { + _memoryOwner = null; + _memory = data; + } + + internal ArrowBuffer(IMemoryOwner memoryOwner) + { + // When wrapping an IMemoryOwner, don't cache the Memory + // since the owner may be disposed, and the cached Memory would + // be invalid. + + _memoryOwner = memoryOwner; + _memory = Memory.Empty; + } + + public ReadOnlyMemory Memory => + _memoryOwner != null ? _memoryOwner.Memory : _memory; + + public bool IsEmpty => Memory.IsEmpty; + + public int Length => Memory.Length; + + public ReadOnlySpan Span + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Memory.Span; + } + + public ArrowBuffer Clone(MemoryAllocator allocator = default) + { + return new Builder(Span.Length) + .Append(Span) + .Build(allocator); + } + + public bool Equals(ArrowBuffer other) + { + return Span.SequenceEqual(other.Span); + } + + public void Dispose() + { + _memoryOwner?.Dispose(); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/BitUtility.cs b/src/arrow/csharp/src/Apache.Arrow/BitUtility.cs new file mode 100644 index 000000000..19417bbbe --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/BitUtility.cs @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Apache.Arrow +{ + public static class BitUtility + { + private static ReadOnlySpan PopcountTable => new byte[] { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, + }; + + private static ReadOnlySpan BitMask => new byte[] { + 1, 2, 4, 8, 16, 32, 64, 128 + }; + + public static bool GetBit(byte data, int index) => + ((data >> index) & 1) != 0; + + public static bool GetBit(ReadOnlySpan data, int index) => + (data[index / 8] & BitMask[index % 8]) != 0; + + public static void ClearBit(Span data, int index) + { + data[index / 8] &= (byte) ~BitMask[index % 8]; + } + + public static void SetBit(Span data, int index) + { + data[index / 8] |= BitMask[index % 8]; + } + + public static void SetBit(Span data, int index, bool value) + { + int idx = index / 8; + int mod = index % 8; + data[idx] = value + ? (byte)(data[idx] | BitMask[mod]) + : (byte)(data[idx] & ~BitMask[mod]); + } + + public static void ToggleBit(Span data, int index) + { + data[index / 8] ^= BitMask[index % 8]; + } + + /// + /// Counts the number of set bits in a span of bytes starting + /// at a specific bit offset. + /// + /// Span to count bits + /// Bit offset to start counting from + /// Count of set (one) bits + public static int CountBits(ReadOnlySpan data, int offset) => + CountBits(data, offset, data.Length * 8 - offset); + + /// + /// Counts the number of set bits in a span of bytes starting + /// at a specific bit offset, and limiting to a certain number of bits + /// in the span. + /// + /// Span to count bits. + /// Bit offset to start counting from. + /// Maximum of bits in the span to consider. + /// Count of set (one) bits + public static int CountBits(ReadOnlySpan data, int offset, int length) + { + int startByteIndex = offset / 8; + int startBitOffset = offset % 8; + int endByteIndex = (offset + length - 1) / 8; + int endBitOffset = (offset + length - 1) % 8; + if (startBitOffset < 0) + return 0; + + int count = 0; + if (startByteIndex == endByteIndex) + { + // Range starts and ends within the same byte. + var slice = data.Slice(startByteIndex, 1); + for (int i = startBitOffset; i <= endBitOffset; i++) + count += GetBit(slice, i) ? 1 : 0; + + return count; + } + + // If the starting index and ending index are not byte-aligned, + // we'll need to count bits the slow way. If they are + // byte-aligned, and for all other bytes in the 'middle', we + // can use a faster byte-aligned count. + int fullByteStartIndex = startBitOffset == 0 ? startByteIndex : startByteIndex + 1; + int fullByteEndIndex = endBitOffset == 7 ? endByteIndex : endByteIndex - 1; + + if (startBitOffset != 0) + { + var slice = data.Slice(startByteIndex, 1); + for (int i = startBitOffset; i <= 7; i++) + count += GetBit(slice, i) ? 1 : 0; + } + + if (fullByteEndIndex >= fullByteStartIndex) + { + var slice = data.Slice(fullByteStartIndex, fullByteEndIndex - fullByteStartIndex + 1); + count += CountBits(slice); + } + + if (endBitOffset != 7) + { + var slice = data.Slice(endByteIndex, 1); + for (int i = 0; i <= endBitOffset; i++) + count += GetBit(slice, i) ? 1 : 0; + } + + return count; + } + + /// + /// Counts the number of set bits in a span of bytes. + /// + /// Span to count bits + /// Count of set (one) bits. + public static int CountBits(ReadOnlySpan data) + { + int count = 0; + foreach (byte t in data) + count += PopcountTable[t]; + return count; + } + + /// + /// Rounds an integer to the nearest multiple of 64. + /// + /// Integer to round. + /// Integer rounded to the nearest multiple of 64. + public static long RoundUpToMultipleOf64(long n) => + RoundUpToMultiplePowerOfTwo(n, 64); + + /// + /// Rounds an integer to the nearest multiple of 8. + /// + /// Integer to round. + /// Integer rounded to the nearest multiple of 8. + public static long RoundUpToMultipleOf8(long n) => + RoundUpToMultiplePowerOfTwo(n, 8); + + /// + /// Rounds an integer up to the nearest multiple of factor, where + /// factor must be a power of two. + /// + /// This function does not throw when the factor is not a power of two. + /// + /// Integer to round up. + /// Power of two factor to round up to. + /// Integer rounded up to the nearest power of two. + public static long RoundUpToMultiplePowerOfTwo(long n, int factor) + { + // Assert that factor is a power of two. + Debug.Assert(factor > 0 && (factor & (factor - 1)) == 0); + return (n + (factor - 1)) & ~(factor - 1); + } + + internal static bool IsMultipleOf8(long n) => n % 8 == 0; + + /// + /// Calculates the number of bytes required to store n bits. + /// + /// number of bits + /// number of bytes + public static int ByteCount(int n) + { + Debug.Assert(n >= 0); + return n / 8 + (n % 8 != 0 ? 1 : 0); // ceil(n / 8) + } + + internal static int ReadInt32(ReadOnlyMemory value) + { + Debug.Assert(value.Length >= sizeof(int)); + + return Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(value.Span)); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/ChunkedArray.cs b/src/arrow/csharp/src/Apache.Arrow/ChunkedArray.cs new file mode 100644 index 000000000..5f25acfe0 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/ChunkedArray.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + /// + /// A data structure to manage a list of primitive Array arrays logically as one large array + /// + public class ChunkedArray + { + private IList Arrays { get; } + public IArrowType DataType { get; } + public long Length { get; } + public long NullCount { get; } + + public int ArrayCount + { + get => Arrays.Count; + } + + public Array Array(int index) => Arrays[index]; + + public ChunkedArray(IList arrays) + { + Arrays = arrays ?? throw new ArgumentNullException(nameof(arrays)); + if (arrays.Count < 1) + { + throw new ArgumentException($"Count must be at least 1. Got {arrays.Count} instead"); + } + DataType = arrays[0].Data.DataType; + foreach (Array array in arrays) + { + Length += array.Length; + NullCount += array.NullCount; + } + } + + public ChunkedArray(Array array) : this(new[] { array }) { } + + public ChunkedArray Slice(long offset, long length) + { + if (offset >= Length) + { + throw new ArgumentException($"Index {offset} cannot be greater than the Column's Length {Length}"); + } + + int curArrayIndex = 0; + int numArrays = Arrays.Count; + while (curArrayIndex < numArrays && offset > Arrays[curArrayIndex].Length) + { + offset -= Arrays[curArrayIndex].Length; + curArrayIndex++; + } + + IList newArrays = new List(); + while (curArrayIndex < numArrays && length > 0) + { + newArrays.Add(Arrays[curArrayIndex].Slice((int)offset, + length > Arrays[curArrayIndex].Length ? Arrays[curArrayIndex].Length : (int)length)); + length -= Arrays[curArrayIndex].Length - offset; + offset = 0; + curArrayIndex++; + } + return new ChunkedArray(newArrays); + } + + public ChunkedArray Slice(long offset) + { + return Slice(offset, Length - offset); + } + + // TODO: Flatten for Structs + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Column.cs b/src/arrow/csharp/src/Apache.Arrow/Column.cs new file mode 100644 index 000000000..6e76e6745 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Column.cs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + /// + /// A Column data structure that logically represents a column in a dataset + /// + public class Column + { + public Field Field { get; } + public ChunkedArray Data { get; } + + public Column(Field field, IList arrays) + { + Data = new ChunkedArray(arrays); + Field = field; + if (!ValidateArrayDataTypes()) + { + throw new ArgumentException($"{Field.DataType} must match {Data.DataType}"); + } + } + + private Column(Field field, ChunkedArray arrays) + { + Field = field; + Data = arrays; + } + + public long Length => Data.Length; + public long NullCount => Data.NullCount; + public string Name => Field.Name; + public IArrowType Type => Field.DataType; + + public Column Slice(int offset, int length) + { + return new Column(Field, Data.Slice(offset, length)); + } + + public Column Slice(int offset) + { + return new Column(Field, Data.Slice(offset)); + } + + private bool ValidateArrayDataTypes() + { + for (int i = 0; i < Data.ArrayCount; i++) + { + if (Data.Array(i).Data.DataType != Field.DataType) + { + return false; + } + } + return true; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/DecimalUtility.cs b/src/arrow/csharp/src/Apache.Arrow/DecimalUtility.cs new file mode 100644 index 000000000..b7ee6b9a8 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/DecimalUtility.cs @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Numerics; + +namespace Apache.Arrow +{ + /// + /// This is semi-optimised best attempt at converting to / from decimal and the buffers + /// + internal static class DecimalUtility + { + private static readonly BigInteger _maxDecimal = new BigInteger(decimal.MaxValue); + private static readonly BigInteger _minDecimal = new BigInteger(decimal.MinValue); + private static readonly ulong[] s_powersOfTen = + { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, + 1000000000000, 10000000000000, 100000000000000, 1000000000000000, 10000000000000000, 100000000000000000, + 1000000000000000000, 10000000000000000000 + }; + + private static int PowersOfTenLength => s_powersOfTen.Length - 1; + + internal static decimal GetDecimal(in ArrowBuffer valueBuffer, int index, int scale, int byteWidth) + { + int startIndex = index * byteWidth; + ReadOnlySpan value = valueBuffer.Span.Slice(startIndex, byteWidth); + BigInteger integerValue; + +#if NETCOREAPP + integerValue = new BigInteger(value); +#else + integerValue = new BigInteger(value.ToArray()); +#endif + + if (integerValue > _maxDecimal || integerValue < _minDecimal) + { + BigInteger scaleBy = BigInteger.Pow(10, scale); + BigInteger integerPart = BigInteger.DivRem(integerValue, scaleBy, out BigInteger fractionalPart); + if (integerPart > _maxDecimal || integerPart < _minDecimal) // decimal overflow, not much we can do here - C# needs a BigDecimal + { + throw new OverflowException($"Value: {integerPart} too big or too small to be represented as a decimal"); + } + return (decimal)integerPart + DivideByScale(fractionalPart, scale); + } + else + { + return DivideByScale(integerValue, scale); + } + } + + private static decimal DivideByScale(BigInteger integerValue, int scale) + { + decimal result = (decimal)integerValue; // this cast is safe here + int drop = scale; + while (drop > PowersOfTenLength) + { + result /= s_powersOfTen[PowersOfTenLength]; + drop -= PowersOfTenLength; + } + + result /= s_powersOfTen[drop]; + return result; + } + + internal static void GetBytes(decimal value, int precision, int scale, int byteWidth, Span bytes) + { + // create BigInteger from decimal + BigInteger bigInt; + int[] decimalBits = decimal.GetBits(value); + int decScale = (decimalBits[3] >> 16) & 0x7F; +#if NETCOREAPP + Span bigIntBytes = stackalloc byte[12]; + + for (int i = 0; i < 3; i++) + { + int bit = decimalBits[i]; + Span intBytes = stackalloc byte[4]; + if (!BitConverter.TryWriteBytes(intBytes, bit)) + throw new OverflowException($"Could not extract bytes from int {bit}"); + + for (int j = 0; j < 4; j++) + { + bigIntBytes[4 * i + j] = intBytes[j]; + } + } + bigInt = new BigInteger(bigIntBytes); +#else + byte[] bigIntBytes = new byte[12]; + for (int i = 0; i < 3; i++) + { + int bit = decimalBits[i]; + byte[] intBytes = BitConverter.GetBytes(bit); + for (int j = 0; j < intBytes.Length; j++) + { + bigIntBytes[4 * i + j] = intBytes[j]; + } + } + bigInt = new BigInteger(bigIntBytes); +#endif + + if (value < 0) + { + bigInt = -bigInt; + } + + // validate precision and scale + if (decScale > scale) + throw new OverflowException($"Decimal scale cannot be greater than that in the Arrow vector: {decScale} != {scale}"); + + if (bigInt >= BigInteger.Pow(10, precision)) + throw new OverflowException($"Decimal precision cannot be greater than that in the Arrow vector: {value} has precision > {precision}"); + + if (decScale < scale) // pad with trailing zeros + { + bigInt *= BigInteger.Pow(10, scale - decScale); + } + + // extract bytes from BigInteger + if (bytes.Length != byteWidth) + { + throw new OverflowException($"ValueBuffer size not equal to {byteWidth} byte width: {bytes.Length}"); + } + + int bytesWritten; +#if NETCOREAPP + if (!bigInt.TryWriteBytes(bytes, out bytesWritten, false, !BitConverter.IsLittleEndian)) + throw new OverflowException("Could not extract bytes from integer value " + bigInt); +#else + byte[] tempBytes = bigInt.ToByteArray(); + tempBytes.CopyTo(bytes); + bytesWritten = tempBytes.Length; +#endif + + if (bytes.Length > byteWidth) + { + throw new OverflowException($"Decimal size greater than {byteWidth} bytes: {bytes.Length}"); + } + + if (bigInt.Sign == -1) + { + for (int i = bytesWritten; i < byteWidth; i++) + { + bytes[i] = 255; + } + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs new file mode 100644 index 000000000..399d9bf5e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + internal static class ArrayDataExtensions + { + public static void EnsureBufferCount(this ArrayData data, int count) + { + if (data.Buffers.Length != count) + { + // TODO: Use localizable string resource + throw new ArgumentException( + $"Buffer count <{data.Buffers.Length}> must be at least <{count}>", + nameof(data.Buffers.Length)); + } + } + + public static void EnsureDataType(this ArrayData data, ArrowTypeId id) + { + if (data.DataType.TypeId != id) + { + // TODO: Use localizable string resource + throw new ArgumentException( + $"Specified array type <{data.DataType.TypeId}> does not match expected type(s) <{id}>", + nameof(data.DataType.TypeId)); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs new file mode 100644 index 000000000..9dd9589c0 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace Apache.Arrow +{ + internal static class ArrayPoolExtensions + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void RentReturn(this ArrayPool pool, int length, Action> action) + { + byte[] array = null; + + try + { + array = pool.Rent(length); + action(array.AsMemory(0, length)); + } + finally + { + if (array != null) + { + pool.Return(array); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ValueTask RentReturnAsync(this ArrayPool pool, int length, Func, ValueTask> action) + { + byte[] array = null; + + try + { + array = pool.Rent(length); + return action(array.AsMemory(0, length)); + } + finally + { + if (array != null) + { + pool.Return(array); + } + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs new file mode 100644 index 000000000..5b0407451 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public static class ArrowTypeExtensions + { + private static readonly ISet s_integralTypes = + new HashSet(new[] + { + ArrowTypeId.Int8, ArrowTypeId.Int16, ArrowTypeId.Int32, ArrowTypeId.Int64, + ArrowTypeId.UInt8, ArrowTypeId.UInt16, ArrowTypeId.UInt32, ArrowTypeId.UInt64, + }); + + private static readonly ISet s_floatingPointTypes = + new HashSet(new[] + { + ArrowTypeId.HalfFloat, ArrowTypeId.Float, ArrowTypeId.Double + }); + + public static bool IsIntegral(this IArrowType type) + => s_integralTypes.Contains(type.TypeId); + + public static bool IsFloatingPoint(this IArrowType type) + => s_floatingPointTypes.Contains(type.TypeId); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs new file mode 100644 index 000000000..d2a70bca9 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Apache.Arrow +{ + internal static class FlatbufExtensions + { + public static bool IsFixedPrimitive(this Flatbuf.Type t) + { + if (t == Flatbuf.Type.Utf8 || t == Flatbuf.Type.Binary) + return false; + return true; + } + + public static bool IsFixedPrimitive(this Types.IArrowType t) + { + return t.TypeId.IsFixedPrimitive(); + } + + public static bool IsFixedPrimitive(this Types.ArrowTypeId t) + { + if (t == Types.ArrowTypeId.String || t == Types.ArrowTypeId.Binary) + return false; + return true; + } + + public static Types.IntervalUnit ToArrow(this Flatbuf.IntervalUnit unit) + { + switch (unit) + { + case Flatbuf.IntervalUnit.DAY_TIME: + return Types.IntervalUnit.DayTime; + case Flatbuf.IntervalUnit.YEAR_MONTH: + return Types.IntervalUnit.YearMonth; + default: + throw new ArgumentException($"Unexpected Flatbuf IntervalUnit", nameof(unit)); + } + } + + public static Types.DateUnit ToArrow(this Flatbuf.DateUnit unit) + { + switch (unit) + { + case Flatbuf.DateUnit.DAY: + return Types.DateUnit.Day; + case Flatbuf.DateUnit.MILLISECOND: + return Types.DateUnit.Milliseconds; + default: + throw new ArgumentException($"Unexpected Flatbuf IntervalUnit", nameof(unit)); + } + } + + public static Types.TimeUnit ToArrow(this Flatbuf.TimeUnit unit) + { + switch (unit) + { + case Flatbuf.TimeUnit.MICROSECOND: + return Types.TimeUnit.Microsecond; + case Flatbuf.TimeUnit.MILLISECOND: + return Types.TimeUnit.Millisecond; + case Flatbuf.TimeUnit.NANOSECOND: + return Types.TimeUnit.Nanosecond; + case Flatbuf.TimeUnit.SECOND: + return Types.TimeUnit.Second; + default: + throw new ArgumentException($"Unexpected Flatbuf TimeUnit", nameof(unit)); + } + } + } +} + diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/SpanExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/SpanExtensions.cs new file mode 100644 index 000000000..b759f3806 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/SpanExtensions.cs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Runtime.InteropServices; + +namespace Apache.Arrow +{ + public static class SpanExtensions + { + public static Span CastTo(this Span span) + where T: struct => + MemoryMarshal.Cast(span); + + public static ReadOnlySpan CastTo(this ReadOnlySpan span) + where T: struct => + MemoryMarshal.Cast(span); + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.cs new file mode 100644 index 000000000..1767d23e3 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Apache.Arrow +{ + internal static partial class StreamExtensions + { + public static async ValueTask ReadFullBufferAsync(this Stream stream, Memory buffer, CancellationToken cancellationToken = default) + { + int totalBytesRead = 0; + do + { + int bytesRead = + await stream.ReadAsync( + buffer.Slice(totalBytesRead, buffer.Length - totalBytesRead), + cancellationToken) + .ConfigureAwait(false); + + if (bytesRead == 0) + { + // reached the end of the stream + return totalBytesRead; + } + + totalBytesRead += bytesRead; + } + while (totalBytesRead < buffer.Length); + + return totalBytesRead; + } + + public static int ReadFullBuffer(this Stream stream, Memory buffer) + { + int totalBytesRead = 0; + do + { + int bytesRead = stream.Read( + buffer.Slice(totalBytesRead, buffer.Length - totalBytesRead)); + + if (bytesRead == 0) + { + // reached the end of the stream + return totalBytesRead; + } + + totalBytesRead += bytesRead; + } + while (totalBytesRead < buffer.Length); + + return totalBytesRead; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs new file mode 100644 index 000000000..efcacdc84 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netcoreapp2.1.cs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; + +namespace Apache.Arrow +{ + // Helpers to read from Stream to Memory on netcoreapp + internal static partial class StreamExtensions + { + public static int Read(this Stream stream, Memory buffer) + { + return stream.Read(buffer.Span); + } + + public static void Write(this Stream stream, ReadOnlyMemory buffer) + { + stream.Write(buffer.Span); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs new file mode 100644 index 000000000..b983be0fd --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Apache.Arrow +{ + // Helpers to write Memory to Stream on netstandard + internal static partial class StreamExtensions + { + public static int Read(this Stream stream, Memory buffer) + { + if (MemoryMarshal.TryGetArray(buffer, out ArraySegment array)) + { + return stream.Read(array.Array, array.Offset, array.Count); + } + else + { + byte[] sharedBuffer = ArrayPool.Shared.Rent(buffer.Length); + try + { + int result = stream.Read(sharedBuffer, 0, buffer.Length); + new Span(sharedBuffer, 0, result).CopyTo(buffer.Span); + return result; + } + finally + { + ArrayPool.Shared.Return(sharedBuffer); + } + } + } + + public static ValueTask ReadAsync(this Stream stream, Memory buffer, CancellationToken cancellationToken = default) + { + if (MemoryMarshal.TryGetArray(buffer, out ArraySegment array)) + { + return new ValueTask(stream.ReadAsync(array.Array, array.Offset, array.Count, cancellationToken)); + } + else + { + byte[] sharedBuffer = ArrayPool.Shared.Rent(buffer.Length); + return FinishReadAsync(stream.ReadAsync(sharedBuffer, 0, buffer.Length, cancellationToken), sharedBuffer, buffer); + + async ValueTask FinishReadAsync(Task readTask, byte[] localBuffer, Memory localDestination) + { + try + { + int result = await readTask.ConfigureAwait(false); + new Span(localBuffer, 0, result).CopyTo(localDestination.Span); + return result; + } + finally + { + ArrayPool.Shared.Return(localBuffer); + } + } + } + } + + public static void Write(this Stream stream, ReadOnlyMemory buffer) + { + if (MemoryMarshal.TryGetArray(buffer, out ArraySegment array)) + { + stream.Write(array.Array, array.Offset, array.Count); + } + else + { + byte[] sharedBuffer = ArrayPool.Shared.Rent(buffer.Length); + try + { + buffer.Span.CopyTo(sharedBuffer); + stream.Write(sharedBuffer, 0, buffer.Length); + } + finally + { + ArrayPool.Shared.Return(sharedBuffer); + } + } + } + + public static ValueTask WriteAsync(this Stream stream, ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + if (MemoryMarshal.TryGetArray(buffer, out ArraySegment array)) + { + return new ValueTask(stream.WriteAsync(array.Array, array.Offset, array.Count, cancellationToken)); + } + else + { + byte[] sharedBuffer = ArrayPool.Shared.Rent(buffer.Length); + buffer.Span.CopyTo(sharedBuffer); + return FinishWriteAsync(stream.WriteAsync(sharedBuffer, 0, buffer.Length, cancellationToken), sharedBuffer); + } + } + + private static async ValueTask FinishWriteAsync(Task writeTask, byte[] localBuffer) + { + try + { + await writeTask.ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(localBuffer); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/TimeSpanExtensions.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/TimeSpanExtensions.cs new file mode 100644 index 000000000..133156d7b --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/TimeSpanExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Apache.Arrow +{ + public static class TimeSpanExtensions + { + /// + /// Formats a TimeSpan into an ISO 8601 compliant time offset string. + /// + /// timeSpan to format + /// ISO 8601 offset string + public static string ToTimeZoneOffsetString(this TimeSpan timeSpan) + { + string sign = timeSpan.Hours >= 0 ? "+" : "-"; + int hours = Math.Abs(timeSpan.Hours); + int minutes = Math.Abs(timeSpan.Minutes); + return sign + hours.ToString("00") + ":" + minutes.ToString("00"); + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs b/src/arrow/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs new file mode 100644 index 000000000..fe42075f1 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Extensions/TupleExtensions.netstandard.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Apache.Arrow +{ + // Helpers to Deconstruct Tuples on netstandard + internal static partial class TupleExtensions + { + public static void Deconstruct(this Tuple value, out T1 item1, out T2 item2) + { + item1 = value.Item1; + item2 = value.Item2; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Field.Builder.cs b/src/arrow/csharp/src/Apache.Arrow/Field.Builder.cs new file mode 100644 index 000000000..1e7aa192e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Field.Builder.cs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; + +namespace Apache.Arrow +{ + public partial class Field + { + public class Builder + { + private Dictionary _metadata; + private string _name; + private IArrowType _type; + private bool _nullable; + + public Builder() + { + _name = string.Empty; + _type = NullType.Default; + _nullable = true; + } + + public Builder Name(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentNullException(nameof(value)); + } + + _name = value; + return this; + } + + public Builder DataType(IArrowType type) + { + _type = type ?? NullType.Default; + return this; + } + + public Builder Nullable(bool value) + { + _nullable = value; + return this; + } + + public Builder Metadata(string key, string value) + { + if (string.IsNullOrWhiteSpace(key)) + { + throw new ArgumentNullException(nameof(key)); + } + + _metadata ??= new Dictionary(); + + _metadata[key] = value; + return this; + } + + public Builder Metadata(IEnumerable> dictionary) + { + if (dictionary == null) + { + throw new ArgumentNullException(nameof(dictionary)); + } + foreach (KeyValuePair entry in dictionary) + { + Metadata(entry.Key, entry.Value); + } + return this; + } + + public Field Build() + { + return new Field(_name, _type, _nullable, _metadata); + } + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Field.cs b/src/arrow/csharp/src/Apache.Arrow/Field.cs new file mode 100644 index 000000000..6e507b642 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Field.cs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public partial class Field + { + public IArrowType DataType { get; } + + public string Name { get; } + + public bool IsNullable { get; } + + public bool HasMetadata => Metadata?.Count > 0; + + public IReadOnlyDictionary Metadata { get; } + + public Field(string name, IArrowType dataType, bool nullable, + IEnumerable> metadata = default) + : this(name, dataType, nullable) + { + Metadata = metadata?.ToDictionary(kv => kv.Key, kv => kv.Value); + + } + + internal Field(string name, IArrowType dataType, bool nullable, + IReadOnlyDictionary metadata, bool copyCollections) + : this(name, dataType, nullable) + { + Debug.Assert(copyCollections == false, "This internal constructor is to not copy the collections."); + + Metadata = metadata; + } + + private Field(string name, IArrowType dataType, bool nullable) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + Name = name; + DataType = dataType ?? NullType.Default; + IsNullable = nullable; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Block.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Block.cs new file mode 100644 index 000000000..89c065b20 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Block.cs @@ -0,0 +1,37 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +internal struct Block : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Block __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Index to the start of the RecordBlock (note this is past the Message header) + public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// Length of the metadata + public int MetaDataLength { get { return __p.bb.GetInt(__p.bb_pos + 8); } } + /// Length of the data (this is aligned so there can be a gap between this and + /// the metatdata). + public long BodyLength { get { return __p.bb.GetLong(__p.bb_pos + 16); } } + + public static Offset CreateBlock(FlatBufferBuilder builder, long Offset, int MetaDataLength, long BodyLength) { + builder.Prep(8, 24); + builder.PutLong(BodyLength); + builder.Pad(4); + builder.PutInt(MetaDataLength); + builder.PutLong(Offset); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/BodyCompression.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/BodyCompression.cs new file mode 100644 index 000000000..dda0dd403 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/BodyCompression.cs @@ -0,0 +1,47 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// Optional compression for the memory buffers constituting IPC message +/// bodies. Intended for use with RecordBatch but could be used for other +/// message types +internal struct BodyCompression : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static BodyCompression GetRootAsBodyCompression(ByteBuffer _bb) { return GetRootAsBodyCompression(_bb, new BodyCompression()); } + public static BodyCompression GetRootAsBodyCompression(ByteBuffer _bb, BodyCompression obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public BodyCompression __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Compressor library + public CompressionType Codec { get { int o = __p.__offset(4); return o != 0 ? (CompressionType)__p.bb.GetSbyte(o + __p.bb_pos) : CompressionType.LZ4_FRAME; } } + /// Indicates the way the record batch body was compressed + public BodyCompressionMethod Method { get { int o = __p.__offset(6); return o != 0 ? (BodyCompressionMethod)__p.bb.GetSbyte(o + __p.bb_pos) : BodyCompressionMethod.BUFFER; } } + + public static Offset CreateBodyCompression(FlatBufferBuilder builder, + CompressionType codec = CompressionType.LZ4_FRAME, + BodyCompressionMethod method = BodyCompressionMethod.BUFFER) { + builder.StartObject(2); + BodyCompression.AddMethod(builder, method); + BodyCompression.AddCodec(builder, codec); + return BodyCompression.EndBodyCompression(builder); + } + + public static void StartBodyCompression(FlatBufferBuilder builder) { builder.StartObject(2); } + public static void AddCodec(FlatBufferBuilder builder, CompressionType codec) { builder.AddSbyte(0, (sbyte)codec, 0); } + public static void AddMethod(FlatBufferBuilder builder, BodyCompressionMethod method) { builder.AddSbyte(1, (sbyte)method, 0); } + public static Offset EndBodyCompression(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs new file mode 100644 index 000000000..7b2315cab --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs @@ -0,0 +1,36 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// A Buffer represents a single contiguous memory segment +internal struct Buffer : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Buffer __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The relative offset into the shared memory page where the bytes for this + /// buffer starts + public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// The absolute length (in bytes) of the memory buffer. The memory is found + /// from offset (inclusive) to offset + length (non-inclusive). + public long Length { get { return __p.bb.GetLong(__p.bb_pos + 8); } } + + public static Offset CreateBuffer(FlatBufferBuilder builder, long Offset, long Length) { + builder.Prep(8, 16); + builder.PutLong(Length); + builder.PutLong(Offset); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs new file mode 100644 index 000000000..e3afafdd5 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs @@ -0,0 +1,54 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// For sending dictionary encoding information. Any Field can be +/// dictionary-encoded, but in this case none of its children may be +/// dictionary-encoded. +/// There is one vector / column per dictionary, but that vector / column +/// may be spread across multiple dictionary batches by using the isDelta +/// flag +internal struct DictionaryBatch : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb) { return GetRootAsDictionaryBatch(_bb, new DictionaryBatch()); } + public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb, DictionaryBatch obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public DictionaryBatch __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } } + public RecordBatch? Data { get { int o = __p.__offset(6); return o != 0 ? (RecordBatch?)(new RecordBatch()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + /// If isDelta is true the values in the dictionary are to be appended to a + /// dictionary with the indicated id + public bool IsDelta { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + + public static Offset CreateDictionaryBatch(FlatBufferBuilder builder, + long id = 0, + Offset dataOffset = default(Offset), + bool isDelta = false) { + builder.StartObject(3); + DictionaryBatch.AddId(builder, id); + DictionaryBatch.AddData(builder, dataOffset); + DictionaryBatch.AddIsDelta(builder, isDelta); + return DictionaryBatch.EndDictionaryBatch(builder); + } + + public static void StartDictionaryBatch(FlatBufferBuilder builder) { builder.StartObject(3); } + public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); } + public static void AddData(FlatBufferBuilder builder, Offset dataOffset) { builder.AddOffset(1, dataOffset.Value, 0); } + public static void AddIsDelta(FlatBufferBuilder builder, bool isDelta) { builder.AddBool(2, isDelta, false); } + public static Offset EndDictionaryBatch(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs new file mode 100644 index 000000000..02a35fdd4 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs @@ -0,0 +1,57 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Dictionary encoding metadata +internal struct DictionaryEncoding : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb) { return GetRootAsDictionaryEncoding(_bb, new DictionaryEncoding()); } + public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb, DictionaryEncoding obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public DictionaryEncoding __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The known dictionary id in the application where this data is used. In + /// the file or streaming formats, the dictionary ids are found in the + /// DictionaryBatch messages + public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } } + /// The dictionary indices are constrained to be positive integers. If this + /// field is null, the indices must be signed int32 + public Int? IndexType { get { int o = __p.__offset(6); return o != 0 ? (Int?)(new Int()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + /// By default, dictionaries are not ordered, or the order does not have + /// semantic meaning. In some statistical, applications, dictionary-encoding + /// is used to represent ordered categorical data, and we provide a way to + /// preserve that metadata here + public bool IsOrdered { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + + public static Offset CreateDictionaryEncoding(FlatBufferBuilder builder, + long id = 0, + Offset indexTypeOffset = default(Offset), + bool isOrdered = false) { + builder.StartObject(3); + DictionaryEncoding.AddId(builder, id); + DictionaryEncoding.AddIndexType(builder, indexTypeOffset); + DictionaryEncoding.AddIsOrdered(builder, isOrdered); + return DictionaryEncoding.EndDictionaryEncoding(builder); + } + + public static void StartDictionaryEncoding(FlatBufferBuilder builder) { builder.StartObject(3); } + public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); } + public static void AddIndexType(FlatBufferBuilder builder, Offset indexTypeOffset) { builder.AddOffset(1, indexTypeOffset.Value, 0); } + public static void AddIsOrdered(FlatBufferBuilder builder, bool isOrdered) { builder.AddBool(2, isOrdered, false); } + public static Offset EndDictionaryEncoding(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/BodyCompressionMethod.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/BodyCompressionMethod.cs new file mode 100644 index 000000000..e9f6b6e83 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/BodyCompressionMethod.cs @@ -0,0 +1,24 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// Provided for forward compatibility in case we need to support different +/// strategies for compressing the IPC message body (like whole-body +/// compression rather than buffer-level) in the future +internal enum BodyCompressionMethod : sbyte +{ + /// Each constituent buffer is first compressed with the indicated + /// compressor, and then written with the uncompressed length in the first 8 + /// bytes as a 64-bit little-endian signed integer followed by the compressed + /// buffer bytes (and then padding as required by the protocol). The + /// uncompressed length may be set to -1 to indicate that the data that + /// follows is not compressed, which can be useful for cases where + /// compression does not yield appreciable savings. + BUFFER = 0, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/CompressionType.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/CompressionType.cs new file mode 100644 index 000000000..3d886c508 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/CompressionType.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum CompressionType : sbyte +{ + LZ4_FRAME = 0, + ZSTD = 1, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs new file mode 100644 index 000000000..46fd0cc4c --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum DateUnit : short +{ + DAY = 0, + MILLISECOND = 1, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs new file mode 100644 index 000000000..a0e64f4ff --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs @@ -0,0 +1,17 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// Endianness of the platform producing the data +internal enum Endianness : short +{ + Little = 0, + Big = 1, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Feature.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Feature.cs new file mode 100644 index 000000000..a05b6cf49 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Feature.cs @@ -0,0 +1,39 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// Represents Arrow Features that might not have full support +/// within implementations. This is intended to be used in +/// two scenarios: +/// 1. A mechanism for readers of Arrow Streams +/// and files to understand that the stream or file makes +/// use of a feature that isn't supported or unknown to +/// the implementation (and therefore can meet the Arrow +/// forward compatibility guarantees). +/// 2. A means of negotiating between a client and server +/// what features a stream is allowed to use. The enums +/// values here are intented to represent higher level +/// features, additional details maybe negotiated +/// with key-value pairs specific to the protocol. +/// +/// Enums added to this list should be assigned power-of-two values +/// to facilitate exchanging and comparing bitmaps for supported +/// features. +internal enum Feature : long +{ + /// Needed to make flatbuffers happy. + UNUSED = 0, + /// The stream makes use of multiple full dictionaries with the + /// same ID and assumes clients implement dictionary replacement + /// correctly. + DICTIONARY_REPLACEMENT = 1, + /// The stream makes use of compressed bodies as described + /// in Message.fbs. + COMPRESSED_BODY = 2, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs new file mode 100644 index 000000000..d1363968d --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum IntervalUnit : short +{ + YEAR_MONTH = 0, + DAY_TIME = 1, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs new file mode 100644 index 000000000..94d239bfa --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs @@ -0,0 +1,26 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// The root Message type +/// This union enables us to easily send different message types without +/// redundant storage, and in the future we can easily add new message types. +/// +/// Arrow implementations do not need to implement all of the message types, +/// which may include experimental metadata types. For maximum compatibility, +/// it is best to send data using RecordBatch +internal enum MessageHeader : byte +{ + NONE = 0, + Schema = 1, + DictionaryBatch = 2, + RecordBatch = 3, + Tensor = 4, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs new file mode 100644 index 000000000..1e893e8cb --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs @@ -0,0 +1,29 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum MetadataVersion : short +{ + /// 0.1.0 (October 2016). + V1 = 0, + /// 0.2.0 (February 2017). Non-backwards compatible with V1. + V2 = 1, + /// 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2. + V3 = 2, + /// >= 0.8.0 (December 2017). Non-backwards compatible with V3. + V4 = 3, + /// >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 + /// metadata and IPC messages). Implementations are recommended to provide a + /// V4 compatibility mode with V5 format changes disabled. + /// + /// Incompatible changes between V4 and V5: + /// - Union buffer layout has changed. In V5, Unions don't have a validity + /// bitmap buffer. + V5 = 4, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs new file mode 100644 index 000000000..3f47a2c0b --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs @@ -0,0 +1,16 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum Precision : short +{ + HALF = 0, + SINGLE = 1, + DOUBLE = 2, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs new file mode 100644 index 000000000..300b835d9 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs @@ -0,0 +1,17 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum TimeUnit : short +{ + SECOND = 0, + MILLISECOND = 1, + MICROSECOND = 2, + NANOSECOND = 3, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs new file mode 100644 index 000000000..e8a7932a7 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs @@ -0,0 +1,38 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// Top-level Type value, enabling extensible type-specific metadata. We can +/// add new logical types to Type without breaking backwards compatibility +internal enum Type : byte +{ + NONE = 0, + Null = 1, + Int = 2, + FloatingPoint = 3, + Binary = 4, + Utf8 = 5, + Bool = 6, + Decimal = 7, + Date = 8, + Time = 9, + Timestamp = 10, + Interval = 11, + List = 12, + Struct_ = 13, + Union = 14, + FixedSizeBinary = 15, + FixedSizeList = 16, + Map = 17, + Duration = 18, + LargeBinary = 19, + LargeUtf8 = 20, + LargeList = 21, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs new file mode 100644 index 000000000..724ff4ac0 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +internal enum UnionMode : short +{ + Sparse = 0, + Dense = 1, +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Field.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Field.cs new file mode 100644 index 000000000..a4f9e3057 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Field.cs @@ -0,0 +1,83 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// A field represents a named column in a record / row batch or child of a +/// nested type. +/// +/// - children is only for nested Arrow arrays +/// - For primitive types, children will have length 0 +/// - nullable should default to true in general +internal struct Field : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static Field GetRootAsField(ByteBuffer _bb) { return GetRootAsField(_bb, new Field()); } + public static Field GetRootAsField(ByteBuffer _bb, Field obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Field __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public string Name { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } +#if ENABLE_SPAN_T + public Span GetNameBytes() { return __p.__vector_as_span(4); } +#else + public ArraySegment? GetNameBytes() { return __p.__vector_as_arraysegment(4); } +#endif + public byte[] GetNameArray() { return __p.__vector_as_array(4); } + public bool Nullable { get { int o = __p.__offset(6); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + public Type TypeType { get { int o = __p.__offset(8); return o != 0 ? (Type)__p.bb.Get(o + __p.bb_pos) : Flatbuf.Type.NONE; } } + public TTable? Type() where TTable : struct, IFlatbufferObject { int o = __p.__offset(10); return o != 0 ? (TTable?)__p.__union(o) : null; } + public DictionaryEncoding? Dictionary { get { int o = __p.__offset(12); return o != 0 ? (DictionaryEncoding?)(new DictionaryEncoding()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + public Field? Children(int j) { int o = __p.__offset(14); return o != 0 ? (Field?)(new Field()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; } + public int ChildrenLength { get { int o = __p.__offset(14); return o != 0 ? __p.__vector_len(o) : 0; } } + public KeyValue? CustomMetadata(int j) { int o = __p.__offset(16); return o != 0 ? (KeyValue?)(new KeyValue()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; } + public int CustomMetadataLength { get { int o = __p.__offset(16); return o != 0 ? __p.__vector_len(o) : 0; } } + + public static Offset CreateField(FlatBufferBuilder builder, + StringOffset nameOffset = default(StringOffset), + bool nullable = false, + Type type_type = Flatbuf.Type.NONE, + int typeOffset = 0, + Offset dictionaryOffset = default(Offset), + VectorOffset childrenOffset = default(VectorOffset), + VectorOffset custom_metadataOffset = default(VectorOffset)) { + builder.StartObject(7); + Field.AddCustomMetadata(builder, custom_metadataOffset); + Field.AddChildren(builder, childrenOffset); + Field.AddDictionary(builder, dictionaryOffset); + Field.AddType(builder, typeOffset); + Field.AddName(builder, nameOffset); + Field.AddTypeType(builder, type_type); + Field.AddNullable(builder, nullable); + return Field.EndField(builder); + } + + public static void StartField(FlatBufferBuilder builder) { builder.StartObject(7); } + public static void AddName(FlatBufferBuilder builder, StringOffset nameOffset) { builder.AddOffset(0, nameOffset.Value, 0); } + public static void AddNullable(FlatBufferBuilder builder, bool nullable) { builder.AddBool(1, nullable, false); } + public static void AddTypeType(FlatBufferBuilder builder, Type typeType) { builder.AddByte(2, (byte)typeType, 0); } + public static void AddType(FlatBufferBuilder builder, int typeOffset) { builder.AddOffset(3, typeOffset, 0); } + public static void AddDictionary(FlatBufferBuilder builder, Offset dictionaryOffset) { builder.AddOffset(4, dictionaryOffset.Value, 0); } + public static void AddChildren(FlatBufferBuilder builder, VectorOffset childrenOffset) { builder.AddOffset(5, childrenOffset.Value, 0); } + public static VectorOffset CreateChildrenVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateChildrenVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } + public static void StartChildrenVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } + public static void AddCustomMetadata(FlatBufferBuilder builder, VectorOffset customMetadataOffset) { builder.AddOffset(6, customMetadataOffset.Value, 0); } + public static VectorOffset CreateCustomMetadataVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static VectorOffset CreateCustomMetadataVectorBlock(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); } + public static void StartCustomMetadataVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } + public static Offset EndField(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs new file mode 100644 index 000000000..811e10ea4 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs @@ -0,0 +1,44 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Data structures for describing a table row batch (a collection of +/// equal-length Arrow arrays) +/// Metadata about a field at some level of a nested type tree (but not +/// its children). +/// +/// For example, a List with values [[1, 2, 3], null, [4], [5, 6], null] +/// would have {length: 5, null_count: 2} for its List node, and {length: 6, +/// null_count: 0} for its Int16 node, as separate FieldNode structs +internal struct FieldNode : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FieldNode __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The number of value slots in the Arrow array at this level of a nested + /// tree + public long Length { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// The number of observed nulls. Fields with null_count == 0 may choose not + /// to write their physical validity bitmap out as a materialized buffer, + /// instead setting the length of the bitmap buffer to 0. + public long NullCount { get { return __p.bb.GetLong(__p.bb_pos + 8); } } + + public static Offset CreateFieldNode(FlatBufferBuilder builder, long Length, long NullCount) { + builder.Prep(8, 16); + builder.PutLong(NullCount); + builder.PutLong(Length); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs new file mode 100644 index 000000000..b6414a23b --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs @@ -0,0 +1,39 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +internal struct FixedSizeBinary : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static FixedSizeBinary GetRootAsFixedSizeBinary(ByteBuffer _bb) { return GetRootAsFixedSizeBinary(_bb, new FixedSizeBinary()); } + public static FixedSizeBinary GetRootAsFixedSizeBinary(ByteBuffer _bb, FixedSizeBinary obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FixedSizeBinary __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Number of bytes per value + public int ByteWidth { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetInt(o + __p.bb_pos) : (int)0; } } + + public static Offset CreateFixedSizeBinary(FlatBufferBuilder builder, + int byteWidth = 0) { + builder.StartObject(1); + FixedSizeBinary.AddByteWidth(builder, byteWidth); + return FixedSizeBinary.EndFixedSizeBinary(builder); + } + + public static void StartFixedSizeBinary(FlatBufferBuilder builder) { builder.StartObject(1); } + public static void AddByteWidth(FlatBufferBuilder builder, int byteWidth) { builder.AddInt(0, byteWidth, 0); } + public static Offset EndFixedSizeBinary(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs new file mode 100644 index 000000000..0ca69b7a1 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs @@ -0,0 +1,39 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +internal struct FixedSizeList : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static FixedSizeList GetRootAsFixedSizeList(ByteBuffer _bb) { return GetRootAsFixedSizeList(_bb, new FixedSizeList()); } + public static FixedSizeList GetRootAsFixedSizeList(ByteBuffer _bb, FixedSizeList obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FixedSizeList __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Number of list items per value + public int ListSize { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetInt(o + __p.bb_pos) : (int)0; } } + + public static Offset CreateFixedSizeList(FlatBufferBuilder builder, + int listSize = 0) { + builder.StartObject(1); + FixedSizeList.AddListSize(builder, listSize); + return FixedSizeList.EndFixedSizeList(builder); + } + + public static void StartFixedSizeList(FlatBufferBuilder builder) { builder.StartObject(1); } + public static void AddListSize(FlatBufferBuilder builder, int listSize) { builder.AddInt(0, listSize, 0); } + public static Offset EndFixedSizeList(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs new file mode 100644 index 000000000..91cd5cccb --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs @@ -0,0 +1,891 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// There are 3 #defines that have an impact on performance / features of this ByteBuffer implementation +// +// UNSAFE_BYTEBUFFER +// This will use unsafe code to manipulate the underlying byte array. This +// can yield a reasonable performance increase. +// +// BYTEBUFFER_NO_BOUNDS_CHECK +// This will disable the bounds check asserts to the byte array. This can +// yield a small performance gain in normal code.. +// +// ENABLE_SPAN_T +// This will enable reading and writing blocks of memory with a Span instead if just +// T[]. You can also enable writing directly to shared memory or other types of memory +// by providing a custom implementation of ByteBufferAllocator. +// ENABLE_SPAN_T also requires UNSAFE_BYTEBUFFER to be defined +// +// Using UNSAFE_BYTEBUFFER and BYTEBUFFER_NO_BOUNDS_CHECK together can yield a +// performance gain of ~15% for some operations, however doing so is potentially +// dangerous. Do so at your own risk! +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +#if ENABLE_SPAN_T +using System.Buffers.Binary; +#endif + +#if ENABLE_SPAN_T && !UNSAFE_BYTEBUFFER +#error ENABLE_SPAN_T requires UNSAFE_BYTEBUFFER to also be defined +#endif + +namespace FlatBuffers +{ + internal abstract class ByteBufferAllocator + { +#if ENABLE_SPAN_T + public abstract Span Span { get; } + public abstract ReadOnlySpan ReadOnlySpan { get; } + public abstract Memory Memory { get; } + public abstract ReadOnlyMemory ReadOnlyMemory { get; } + +#else + public byte[] Buffer + { + get; + protected set; + } +#endif + + public int Length + { + get; + protected set; + } + + public abstract void GrowFront(int newSize); + } + + internal sealed class ByteArrayAllocator : ByteBufferAllocator + { + private byte[] _buffer; + + public ByteArrayAllocator(byte[] buffer) + { + _buffer = buffer; + InitBuffer(); + } + + public override void GrowFront(int newSize) + { + if ((Length & 0xC0000000) != 0) + throw new Exception( + "ByteBuffer: cannot grow buffer beyond 2 gigabytes."); + + if (newSize < Length) + throw new Exception("ByteBuffer: cannot truncate buffer."); + + byte[] newBuffer = new byte[newSize]; + System.Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length, Length); + _buffer = newBuffer; + InitBuffer(); + } + +#if ENABLE_SPAN_T + public override Span Span => _buffer; + public override ReadOnlySpan ReadOnlySpan => _buffer; + public override Memory Memory => _buffer; + public override ReadOnlyMemory ReadOnlyMemory => _buffer; +#endif + + private void InitBuffer() + { + Length = _buffer.Length; +#if !ENABLE_SPAN_T + Buffer = _buffer; +#endif + } + } + + /// + /// Class to mimic Java's ByteBuffer which is used heavily in Flatbuffers. + /// + internal class ByteBuffer + { + private ByteBufferAllocator _buffer; + private int _pos; // Must track start of the buffer. + + public ByteBuffer(ByteBufferAllocator allocator, int position) + { + _buffer = allocator; + _pos = position; + } + + public ByteBuffer(int size) : this(new byte[size]) { } + + public ByteBuffer(byte[] buffer) : this(buffer, 0) { } + + public ByteBuffer(byte[] buffer, int pos) + { + _buffer = new ByteArrayAllocator(buffer); + _pos = pos; + } + + public int Position + { + get { return _pos; } + set { _pos = value; } + } + + public int Length { get { return _buffer.Length; } } + + public void Reset() + { + _pos = 0; + } + + // Create a new ByteBuffer on the same underlying data. + // The new ByteBuffer's position will be same as this buffer's. + public ByteBuffer Duplicate() + { + return new ByteBuffer(_buffer, Position); + } + + // Increases the size of the ByteBuffer, and copies the old data towards + // the end of the new buffer. + public void GrowFront(int newSize) + { + _buffer.GrowFront(newSize); + } + + public byte[] ToArray(int pos, int len) + { + return ToArray(pos, len); + } + + /// + /// A lookup of type sizes. Used instead of Marshal.SizeOf() which has additional + /// overhead, but also is compatible with generic functions for simplified code. + /// + private static Dictionary genericSizes = new Dictionary() + { + { typeof(bool), sizeof(bool) }, + { typeof(float), sizeof(float) }, + { typeof(double), sizeof(double) }, + { typeof(sbyte), sizeof(sbyte) }, + { typeof(byte), sizeof(byte) }, + { typeof(short), sizeof(short) }, + { typeof(ushort), sizeof(ushort) }, + { typeof(int), sizeof(int) }, + { typeof(uint), sizeof(uint) }, + { typeof(ulong), sizeof(ulong) }, + { typeof(long), sizeof(long) }, + }; + + /// + /// Get the wire-size (in bytes) of a type supported by flatbuffers. + /// + /// The type to get the wire size of + /// + public static int SizeOf() + { + return genericSizes[typeof(T)]; + } + + /// + /// Checks if the Type provided is supported as scalar value + /// + /// The Type to check + /// True if the type is a scalar type that is supported, falsed otherwise + public static bool IsSupportedType() + { + return genericSizes.ContainsKey(typeof(T)); + } + + /// + /// Get the wire-size (in bytes) of a typed array + /// + /// The type of the array + /// The array to get the size of + /// The number of bytes the array takes on wire + public static int ArraySize(T[] x) + { + return SizeOf() * x.Length; + } + +#if ENABLE_SPAN_T + public static int ArraySize(Span x) + { + return SizeOf() * x.Length; + } +#endif + + // Get a portion of the buffer casted into an array of type T, given + // the buffer position and length. +#if ENABLE_SPAN_T + public T[] ToArray(int pos, int len) + where T : struct + { + AssertOffsetAndLength(pos, len); + return MemoryMarshal.Cast(_buffer.ReadOnlySpan.Slice(pos)).Slice(0, len).ToArray(); + } +#else + public T[] ToArray(int pos, int len) + where T : struct + { + AssertOffsetAndLength(pos, len); + T[] arr = new T[len]; + Buffer.BlockCopy(_buffer.Buffer, pos, arr, 0, ArraySize(arr)); + return arr; + } +#endif + + public byte[] ToSizedArray() + { + return ToArray(Position, Length - Position); + } + + public byte[] ToFullArray() + { + return ToArray(0, Length); + } + +#if ENABLE_SPAN_T + public ReadOnlyMemory ToReadOnlyMemory(int pos, int len) + { + return _buffer.ReadOnlyMemory.Slice(pos, len); + } + + public Memory ToMemory(int pos, int len) + { + return _buffer.Memory.Slice(pos, len); + } + + public Span ToSpan(int pos, int len) + { + return _buffer.Span.Slice(pos, len); + } +#else + public ArraySegment ToArraySegment(int pos, int len) + { + return new ArraySegment(_buffer.Buffer, pos, len); + } + + public MemoryStream ToMemoryStream(int pos, int len) + { + return new MemoryStream(_buffer.Buffer, pos, len); + } +#endif + +#if !UNSAFE_BYTEBUFFER + // Pre-allocated helper arrays for conversion. + private float[] floathelper = new[] { 0.0f }; + private int[] inthelper = new[] { 0 }; + private double[] doublehelper = new[] { 0.0 }; + private ulong[] ulonghelper = new[] { 0UL }; +#endif // !UNSAFE_BYTEBUFFER + + // Helper functions for the unsafe version. + static public ushort ReverseBytes(ushort input) + { + return (ushort)(((input & 0x00FFU) << 8) | + ((input & 0xFF00U) >> 8)); + } + static public uint ReverseBytes(uint input) + { + return ((input & 0x000000FFU) << 24) | + ((input & 0x0000FF00U) << 8) | + ((input & 0x00FF0000U) >> 8) | + ((input & 0xFF000000U) >> 24); + } + static public ulong ReverseBytes(ulong input) + { + return (((input & 0x00000000000000FFUL) << 56) | + ((input & 0x000000000000FF00UL) << 40) | + ((input & 0x0000000000FF0000UL) << 24) | + ((input & 0x00000000FF000000UL) << 8) | + ((input & 0x000000FF00000000UL) >> 8) | + ((input & 0x0000FF0000000000UL) >> 24) | + ((input & 0x00FF000000000000UL) >> 40) | + ((input & 0xFF00000000000000UL) >> 56)); + } + +#if !UNSAFE_BYTEBUFFER + // Helper functions for the safe (but slower) version. + protected void WriteLittleEndian(int offset, int count, ulong data) + { + if (BitConverter.IsLittleEndian) + { + for (int i = 0; i < count; i++) + { + _buffer.Buffer[offset + i] = (byte)(data >> i * 8); + } + } + else + { + for (int i = 0; i < count; i++) + { + _buffer.Buffer[offset + count - 1 - i] = (byte)(data >> i * 8); + } + } + } + + protected ulong ReadLittleEndian(int offset, int count) + { + AssertOffsetAndLength(offset, count); + ulong r = 0; + if (BitConverter.IsLittleEndian) + { + for (int i = 0; i < count; i++) + { + r |= (ulong)_buffer.Buffer[offset + i] << i * 8; + } + } + else + { + for (int i = 0; i < count; i++) + { + r |= (ulong)_buffer.Buffer[offset + count - 1 - i] << i * 8; + } + } + return r; + } +#endif // !UNSAFE_BYTEBUFFER + + private void AssertOffsetAndLength(int offset, int length) + { +#if !BYTEBUFFER_NO_BOUNDS_CHECK + if (offset < 0 || + offset > _buffer.Length - length) + throw new ArgumentOutOfRangeException(); +#endif + } + +#if ENABLE_SPAN_T + + public void PutSbyte(int offset, sbyte value) + { + AssertOffsetAndLength(offset, sizeof(sbyte)); + _buffer.Span[offset] = (byte)value; + } + + public void PutByte(int offset, byte value) + { + AssertOffsetAndLength(offset, sizeof(byte)); + _buffer.Span[offset] = value; + } + + public void PutByte(int offset, byte value, int count) + { + AssertOffsetAndLength(offset, sizeof(byte) * count); + Span span = _buffer.Span.Slice(offset, count); + for (var i = 0; i < span.Length; ++i) + span[i] = value; + } +#else + public void PutSbyte(int offset, sbyte value) + { + AssertOffsetAndLength(offset, sizeof(sbyte)); + _buffer.Buffer[offset] = (byte)value; + } + + public void PutByte(int offset, byte value) + { + AssertOffsetAndLength(offset, sizeof(byte)); + _buffer.Buffer[offset] = value; + } + + public void PutByte(int offset, byte value, int count) + { + AssertOffsetAndLength(offset, sizeof(byte) * count); + for (var i = 0; i < count; ++i) + _buffer.Buffer[offset + i] = value; + } +#endif + + // this method exists in order to conform with Java ByteBuffer standards + public void Put(int offset, byte value) + { + PutByte(offset, value); + } + +#if ENABLE_SPAN_T + public unsafe void PutStringUTF8(int offset, string value) + { + AssertOffsetAndLength(offset, value.Length); + fixed (char* s = value) + { + fixed (byte* buffer = &MemoryMarshal.GetReference(_buffer.Span)) + { + Encoding.UTF8.GetBytes(s, value.Length, buffer + offset, Length - offset); + } + } + } +#else + public void PutStringUTF8(int offset, string value) + { + AssertOffsetAndLength(offset, value.Length); + Encoding.UTF8.GetBytes(value, 0, value.Length, + _buffer.Buffer, offset); + } +#endif + +#if UNSAFE_BYTEBUFFER + // Unsafe but more efficient versions of Put*. + public void PutShort(int offset, short value) + { + PutUshort(offset, (ushort)value); + } + + public unsafe void PutUshort(int offset, ushort value) + { + AssertOffsetAndLength(offset, sizeof(ushort)); +#if ENABLE_SPAN_T + Span span = _buffer.Span.Slice(offset); + BinaryPrimitives.WriteUInt16LittleEndian(span, value); +#else + fixed (byte* ptr = _buffer.Buffer) + { + *(ushort*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } +#endif + } + + public void PutInt(int offset, int value) + { + PutUint(offset, (uint)value); + } + + public unsafe void PutUint(int offset, uint value) + { + AssertOffsetAndLength(offset, sizeof(uint)); +#if ENABLE_SPAN_T + Span span = _buffer.Span.Slice(offset); + BinaryPrimitives.WriteUInt32LittleEndian(span, value); +#else + fixed (byte* ptr = _buffer.Buffer) + { + *(uint*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } +#endif + } + + public unsafe void PutLong(int offset, long value) + { + PutUlong(offset, (ulong)value); + } + + public unsafe void PutUlong(int offset, ulong value) + { + AssertOffsetAndLength(offset, sizeof(ulong)); +#if ENABLE_SPAN_T + Span span = _buffer.Span.Slice(offset); + BinaryPrimitives.WriteUInt64LittleEndian(span, value); +#else + fixed (byte* ptr = _buffer.Buffer) + { + *(ulong*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } +#endif + } + + public unsafe void PutFloat(int offset, float value) + { + AssertOffsetAndLength(offset, sizeof(float)); +#if ENABLE_SPAN_T + fixed (byte* ptr = &MemoryMarshal.GetReference(_buffer.Span)) +#else + fixed (byte* ptr = _buffer.Buffer) +#endif + { + if (BitConverter.IsLittleEndian) + { + *(float*)(ptr + offset) = value; + } + else + { + *(uint*)(ptr + offset) = ReverseBytes(*(uint*)(&value)); + } + } + } + + public unsafe void PutDouble(int offset, double value) + { + AssertOffsetAndLength(offset, sizeof(double)); +#if ENABLE_SPAN_T + fixed (byte* ptr = &MemoryMarshal.GetReference(_buffer.Span)) +#else + fixed (byte* ptr = _buffer.Buffer) +#endif + { + if (BitConverter.IsLittleEndian) + { + *(double*)(ptr + offset) = value; + } + else + { + *(ulong*)(ptr + offset) = ReverseBytes(*(ulong*)(&value)); + } + } + } +#else // !UNSAFE_BYTEBUFFER + // Slower versions of Put* for when unsafe code is not allowed. + public void PutShort(int offset, short value) + { + AssertOffsetAndLength(offset, sizeof(short)); + WriteLittleEndian(offset, sizeof(short), (ulong)value); + } + + public void PutUshort(int offset, ushort value) + { + AssertOffsetAndLength(offset, sizeof(ushort)); + WriteLittleEndian(offset, sizeof(ushort), (ulong)value); + } + + public void PutInt(int offset, int value) + { + AssertOffsetAndLength(offset, sizeof(int)); + WriteLittleEndian(offset, sizeof(int), (ulong)value); + } + + public void PutUint(int offset, uint value) + { + AssertOffsetAndLength(offset, sizeof(uint)); + WriteLittleEndian(offset, sizeof(uint), (ulong)value); + } + + public void PutLong(int offset, long value) + { + AssertOffsetAndLength(offset, sizeof(long)); + WriteLittleEndian(offset, sizeof(long), (ulong)value); + } + + public void PutUlong(int offset, ulong value) + { + AssertOffsetAndLength(offset, sizeof(ulong)); + WriteLittleEndian(offset, sizeof(ulong), value); + } + + public void PutFloat(int offset, float value) + { + AssertOffsetAndLength(offset, sizeof(float)); + floathelper[0] = value; + Buffer.BlockCopy(floathelper, 0, inthelper, 0, sizeof(float)); + WriteLittleEndian(offset, sizeof(float), (ulong)inthelper[0]); + } + + public void PutDouble(int offset, double value) + { + AssertOffsetAndLength(offset, sizeof(double)); + doublehelper[0] = value; + Buffer.BlockCopy(doublehelper, 0, ulonghelper, 0, sizeof(double)); + WriteLittleEndian(offset, sizeof(double), ulonghelper[0]); + } + +#endif // UNSAFE_BYTEBUFFER + +#if ENABLE_SPAN_T + public sbyte GetSbyte(int index) + { + AssertOffsetAndLength(index, sizeof(sbyte)); + return (sbyte)_buffer.ReadOnlySpan[index]; + } + + public byte Get(int index) + { + AssertOffsetAndLength(index, sizeof(byte)); + return _buffer.ReadOnlySpan[index]; + } +#else + public sbyte GetSbyte(int index) + { + AssertOffsetAndLength(index, sizeof(sbyte)); + return (sbyte)_buffer.Buffer[index]; + } + + public byte Get(int index) + { + AssertOffsetAndLength(index, sizeof(byte)); + return _buffer.Buffer[index]; + } +#endif + +#if ENABLE_SPAN_T + public unsafe string GetStringUTF8(int startPos, int len) + { + fixed (byte* buffer = &MemoryMarshal.GetReference(_buffer.ReadOnlySpan.Slice(startPos))) + { + return Encoding.UTF8.GetString(buffer, len); + } + } +#else + public string GetStringUTF8(int startPos, int len) + { + return Encoding.UTF8.GetString(_buffer.Buffer, startPos, len); + } +#endif + +#if UNSAFE_BYTEBUFFER + // Unsafe but more efficient versions of Get*. + public short GetShort(int offset) + { + return (short)GetUshort(offset); + } + + public unsafe ushort GetUshort(int offset) + { + AssertOffsetAndLength(offset, sizeof(ushort)); +#if ENABLE_SPAN_T + ReadOnlySpan span = _buffer.ReadOnlySpan.Slice(offset); + return BinaryPrimitives.ReadUInt16LittleEndian(span); +#else + fixed (byte* ptr = _buffer.Buffer) + { + return BitConverter.IsLittleEndian + ? *(ushort*)(ptr + offset) + : ReverseBytes(*(ushort*)(ptr + offset)); + } +#endif + } + + public int GetInt(int offset) + { + return (int)GetUint(offset); + } + + public unsafe uint GetUint(int offset) + { + AssertOffsetAndLength(offset, sizeof(uint)); +#if ENABLE_SPAN_T + ReadOnlySpan span = _buffer.ReadOnlySpan.Slice(offset); + return BinaryPrimitives.ReadUInt32LittleEndian(span); +#else + fixed (byte* ptr = _buffer.Buffer) + { + return BitConverter.IsLittleEndian + ? *(uint*)(ptr + offset) + : ReverseBytes(*(uint*)(ptr + offset)); + } +#endif + } + + public long GetLong(int offset) + { + return (long)GetUlong(offset); + } + + public unsafe ulong GetUlong(int offset) + { + AssertOffsetAndLength(offset, sizeof(ulong)); +#if ENABLE_SPAN_T + ReadOnlySpan span = _buffer.ReadOnlySpan.Slice(offset); + return BinaryPrimitives.ReadUInt64LittleEndian(span); +#else + fixed (byte* ptr = _buffer.Buffer) + { + return BitConverter.IsLittleEndian + ? *(ulong*)(ptr + offset) + : ReverseBytes(*(ulong*)(ptr + offset)); + } +#endif + } + + public unsafe float GetFloat(int offset) + { + AssertOffsetAndLength(offset, sizeof(float)); +#if ENABLE_SPAN_T + fixed (byte* ptr = &MemoryMarshal.GetReference(_buffer.ReadOnlySpan)) +#else + fixed (byte* ptr = _buffer.Buffer) +#endif + { + if (BitConverter.IsLittleEndian) + { + return *(float*)(ptr + offset); + } + else + { + uint uvalue = ReverseBytes(*(uint*)(ptr + offset)); + return *(float*)(&uvalue); + } + } + } + + public unsafe double GetDouble(int offset) + { + AssertOffsetAndLength(offset, sizeof(double)); +#if ENABLE_SPAN_T + fixed (byte* ptr = &MemoryMarshal.GetReference(_buffer.ReadOnlySpan)) +#else + fixed (byte* ptr = _buffer.Buffer) +#endif + { + if (BitConverter.IsLittleEndian) + { + return *(double*)(ptr + offset); + } + else + { + ulong uvalue = ReverseBytes(*(ulong*)(ptr + offset)); + return *(double*)(&uvalue); + } + } + } +#else // !UNSAFE_BYTEBUFFER + // Slower versions of Get* for when unsafe code is not allowed. + public short GetShort(int index) + { + return (short)ReadLittleEndian(index, sizeof(short)); + } + + public ushort GetUshort(int index) + { + return (ushort)ReadLittleEndian(index, sizeof(ushort)); + } + + public int GetInt(int index) + { + return (int)ReadLittleEndian(index, sizeof(int)); + } + + public uint GetUint(int index) + { + return (uint)ReadLittleEndian(index, sizeof(uint)); + } + + public long GetLong(int index) + { + return (long)ReadLittleEndian(index, sizeof(long)); + } + + public ulong GetUlong(int index) + { + return ReadLittleEndian(index, sizeof(ulong)); + } + + public float GetFloat(int index) + { + int i = (int)ReadLittleEndian(index, sizeof(float)); + inthelper[0] = i; + Buffer.BlockCopy(inthelper, 0, floathelper, 0, sizeof(float)); + return floathelper[0]; + } + + public double GetDouble(int index) + { + ulong i = ReadLittleEndian(index, sizeof(double)); + // There's Int64BitsToDouble but it uses unsafe code internally. + ulonghelper[0] = i; + Buffer.BlockCopy(ulonghelper, 0, doublehelper, 0, sizeof(double)); + return doublehelper[0]; + } +#endif // UNSAFE_BYTEBUFFER + + /// + /// Copies an array of type T into this buffer, ending at the given + /// offset into this buffer. The starting offset is calculated based on the length + /// of the array and is the value returned. + /// + /// The type of the input data (must be a struct) + /// The offset into this buffer where the copy will end + /// The array to copy data from + /// The 'start' location of this buffer now, after the copy completed + public int Put(int offset, T[] x) + where T : struct + { + if (x == null) + { + throw new ArgumentNullException("Cannot put a null array"); + } + + if (x.Length == 0) + { + throw new ArgumentException("Cannot put an empty array"); + } + + if (!IsSupportedType()) + { + throw new ArgumentException("Cannot put an array of type " + + typeof(T) + " into this buffer"); + } + + if (BitConverter.IsLittleEndian) + { + int numBytes = ByteBuffer.ArraySize(x); + offset -= numBytes; + AssertOffsetAndLength(offset, numBytes); + // if we are LE, just do a block copy +#if ENABLE_SPAN_T + MemoryMarshal.Cast(x).CopyTo(_buffer.Span.Slice(offset, numBytes)); +#else + Buffer.BlockCopy(x, 0, _buffer.Buffer, offset, numBytes); +#endif + } + else + { + throw new NotImplementedException("Big Endian Support not implemented yet " + + "for putting typed arrays"); + // if we are BE, we have to swap each element by itself + //for(int i = x.Length - 1; i >= 0; i--) + //{ + // todo: low priority, but need to genericize the Put() functions + //} + } + return offset; + } + +#if ENABLE_SPAN_T + public int Put(int offset, Span x) + where T : struct + { + if (x.Length == 0) + { + throw new ArgumentException("Cannot put an empty array"); + } + + if (!IsSupportedType()) + { + throw new ArgumentException("Cannot put an array of type " + + typeof(T) + " into this buffer"); + } + + if (BitConverter.IsLittleEndian) + { + int numBytes = ByteBuffer.ArraySize(x); + offset -= numBytes; + AssertOffsetAndLength(offset, numBytes); + // if we are LE, just do a block copy + MemoryMarshal.Cast(x).CopyTo(_buffer.Span.Slice(offset, numBytes)); + } + else + { + throw new NotImplementedException("Big Endian Support not implemented yet " + + "for putting typed arrays"); + // if we are BE, we have to swap each element by itself + //for(int i = x.Length - 1; i >= 0; i--) + //{ + // todo: low priority, but need to genericize the Put() functions + //} + } + return offset; + } +#endif + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs new file mode 100644 index 000000000..cfba4305e --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace FlatBuffers +{ + /// + /// Class that collects utility functions around `ByteBuffer`. + /// + internal class ByteBufferUtil + { + // Extract the size prefix from a `ByteBuffer`. + public static int GetSizePrefix(ByteBuffer bb) { + return bb.GetInt(bb.Position); + } + + // Create a duplicate of a size-prefixed `ByteBuffer` that has its position + // advanced just past the size prefix. + public static ByteBuffer RemoveSizePrefix(ByteBuffer bb) { + ByteBuffer s = bb.Duplicate(); + s.Position += FlatBufferConstants.SizePrefixLength; + return s; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs new file mode 100644 index 000000000..65873a634 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs @@ -0,0 +1,812 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +using System; +using System.Text; + +/// @file +/// @addtogroup flatbuffers_csharp_api +/// @{ + +namespace FlatBuffers +{ + /// + /// Responsible for building up and accessing a FlatBuffer formatted byte + /// array (via ByteBuffer). + /// + internal class FlatBufferBuilder + { + private int _space; + private ByteBuffer _bb; + private int _minAlign = 1; + + // The vtable for the current table (if _vtableSize >= 0) + private int[] _vtable = new int[16]; + // The size of the vtable. -1 indicates no vtable + private int _vtableSize = -1; + // Starting offset of the current struct/table. + private int _objectStart; + // List of offsets of all vtables. + private int[] _vtables = new int[16]; + // Number of entries in `vtables` in use. + private int _numVtables = 0; + // For the current vector being built. + private int _vectorNumElems = 0; + + /// + /// Create a FlatBufferBuilder with a given initial size. + /// + /// + /// The initial size to use for the internal buffer. + /// + public FlatBufferBuilder(int initialSize) + { + if (initialSize <= 0) + throw new ArgumentOutOfRangeException("initialSize", + initialSize, "Must be greater than zero"); + _space = initialSize; + _bb = new ByteBuffer(initialSize); + } + + /// + /// Create a FlatBufferBuilder backed by the passed in ByteBuffer + /// + /// The ByteBuffer to write to + public FlatBufferBuilder(ByteBuffer buffer) + { + _bb = buffer; + _space = buffer.Length; + buffer.Reset(); + } + + /// + /// Reset the FlatBufferBuilder by purging all data that it holds. + /// + public void Clear() + { + _space = _bb.Length; + _bb.Reset(); + _minAlign = 1; + while (_vtableSize > 0) _vtable[--_vtableSize] = 0; + _vtableSize = -1; + _objectStart = 0; + _numVtables = 0; + _vectorNumElems = 0; + } + + /// + /// Gets and sets a Boolean to disable the optimization when serializing + /// default values to a Table. + /// + /// In order to save space, fields that are set to their default value + /// don't get serialized into the buffer. + /// + public bool ForceDefaults { get; set; } + + /// @cond FLATBUFFERS_INTERNAL + + public int Offset { get { return _bb.Length - _space; } } + + public void Pad(int size) + { + _bb.PutByte(_space -= size, 0, size); + } + + // Doubles the size of the ByteBuffer, and copies the old data towards + // the end of the new buffer (since we build the buffer backwards). + void GrowBuffer() + { + _bb.GrowFront(_bb.Length << 1); + } + + // Prepare to write an element of `size` after `additional_bytes` + // have been written, e.g. if you write a string, you need to align + // such the int length field is aligned to SIZEOF_INT, and the string + // data follows it directly. + // If all you need to do is align, `additional_bytes` will be 0. + public void Prep(int size, int additionalBytes) + { + // Track the biggest thing we've ever aligned to. + if (size > _minAlign) + _minAlign = size; + // Find the amount of alignment needed such that `size` is properly + // aligned after `additional_bytes` + var alignSize = + ((~((int)_bb.Length - _space + additionalBytes)) + 1) & + (size - 1); + // Reallocate the buffer if needed. + while (_space < alignSize + size + additionalBytes) + { + var oldBufSize = (int)_bb.Length; + GrowBuffer(); + _space += (int)_bb.Length - oldBufSize; + + } + if (alignSize > 0) + Pad(alignSize); + } + + public void PutBool(bool x) + { + _bb.PutByte(_space -= sizeof(byte), (byte)(x ? 1 : 0)); + } + + public void PutSbyte(sbyte x) + { + _bb.PutSbyte(_space -= sizeof(sbyte), x); + } + + public void PutByte(byte x) + { + _bb.PutByte(_space -= sizeof(byte), x); + } + + public void PutShort(short x) + { + _bb.PutShort(_space -= sizeof(short), x); + } + + public void PutUshort(ushort x) + { + _bb.PutUshort(_space -= sizeof(ushort), x); + } + + public void PutInt(int x) + { + _bb.PutInt(_space -= sizeof(int), x); + } + + public void PutUint(uint x) + { + _bb.PutUint(_space -= sizeof(uint), x); + } + + public void PutLong(long x) + { + _bb.PutLong(_space -= sizeof(long), x); + } + + public void PutUlong(ulong x) + { + _bb.PutUlong(_space -= sizeof(ulong), x); + } + + public void PutFloat(float x) + { + _bb.PutFloat(_space -= sizeof(float), x); + } + + /// + /// Puts an array of type T into this builder at the + /// current offset + /// + /// The type of the input data + /// The array to copy data from + public void Put(T[] x) + where T : struct + { + _space = _bb.Put(_space, x); + } + +#if ENABLE_SPAN_T + /// + /// Puts a span of type T into this builder at the + /// current offset + /// + /// The type of the input data + /// The span to copy data from + public void Put(Span x) + where T : struct + { + _space = _bb.Put(_space, x); + } +#endif + + public void PutDouble(double x) + { + _bb.PutDouble(_space -= sizeof(double), x); + } + /// @endcond + + /// + /// Add a `bool` to the buffer (aligns the data and grows if necessary). + /// + /// The `bool` to add to the buffer. + public void AddBool(bool x) { Prep(sizeof(byte), 0); PutBool(x); } + + /// + /// Add a `sbyte` to the buffer (aligns the data and grows if necessary). + /// + /// The `sbyte` to add to the buffer. + public void AddSbyte(sbyte x) { Prep(sizeof(sbyte), 0); PutSbyte(x); } + + /// + /// Add a `byte` to the buffer (aligns the data and grows if necessary). + /// + /// The `byte` to add to the buffer. + public void AddByte(byte x) { Prep(sizeof(byte), 0); PutByte(x); } + + /// + /// Add a `short` to the buffer (aligns the data and grows if necessary). + /// + /// The `short` to add to the buffer. + public void AddShort(short x) { Prep(sizeof(short), 0); PutShort(x); } + + /// + /// Add an `ushort` to the buffer (aligns the data and grows if necessary). + /// + /// The `ushort` to add to the buffer. + public void AddUshort(ushort x) { Prep(sizeof(ushort), 0); PutUshort(x); } + + /// + /// Add an `int` to the buffer (aligns the data and grows if necessary). + /// + /// The `int` to add to the buffer. + public void AddInt(int x) { Prep(sizeof(int), 0); PutInt(x); } + + /// + /// Add an `uint` to the buffer (aligns the data and grows if necessary). + /// + /// The `uint` to add to the buffer. + public void AddUint(uint x) { Prep(sizeof(uint), 0); PutUint(x); } + + /// + /// Add a `long` to the buffer (aligns the data and grows if necessary). + /// + /// The `long` to add to the buffer. + public void AddLong(long x) { Prep(sizeof(long), 0); PutLong(x); } + + /// + /// Add an `ulong` to the buffer (aligns the data and grows if necessary). + /// + /// The `ulong` to add to the buffer. + public void AddUlong(ulong x) { Prep(sizeof(ulong), 0); PutUlong(x); } + + /// + /// Add a `float` to the buffer (aligns the data and grows if necessary). + /// + /// The `float` to add to the buffer. + public void AddFloat(float x) { Prep(sizeof(float), 0); PutFloat(x); } + + /// + /// Add an array of type T to the buffer (aligns the data and grows if necessary). + /// + /// The type of the input data + /// The array to copy data from + public void Add(T[] x) + where T : struct + { + if (x == null) + { + throw new ArgumentNullException("Cannot add a null array"); + } + + if( x.Length == 0) + { + // don't do anything if the array is empty + return; + } + + if(!ByteBuffer.IsSupportedType()) + { + throw new ArgumentException("Cannot add this Type array to the builder"); + } + + int size = ByteBuffer.SizeOf(); + // Need to prep on size (for data alignment) and then we pass the + // rest of the length (minus 1) as additional bytes + Prep(size, size * (x.Length - 1)); + Put(x); + } + +#if ENABLE_SPAN_T + /// + /// Add a span of type T to the buffer (aligns the data and grows if necessary). + /// + /// The type of the input data + /// The span to copy data from + public void Add(Span x) + where T : struct + { + if (!ByteBuffer.IsSupportedType()) + { + throw new ArgumentException("Cannot add this Type array to the builder"); + } + + int size = ByteBuffer.SizeOf(); + // Need to prep on size (for data alignment) and then we pass the + // rest of the length (minus 1) as additional bytes + Prep(size, size * (x.Length - 1)); + Put(x); + } +#endif + + /// + /// Add a `double` to the buffer (aligns the data and grows if necessary). + /// + /// The `double` to add to the buffer. + public void AddDouble(double x) { Prep(sizeof(double), 0); + PutDouble(x); } + + /// + /// Adds an offset, relative to where it will be written. + /// + /// The offset to add to the buffer. + public void AddOffset(int off) + { + Prep(sizeof(int), 0); // Ensure alignment is already done. + if (off > Offset) + throw new ArgumentException(); + + off = Offset - off + sizeof(int); + PutInt(off); + } + + /// @cond FLATBUFFERS_INTERNAL + public void StartVector(int elemSize, int count, int alignment) + { + NotNested(); + _vectorNumElems = count; + Prep(sizeof(int), elemSize * count); + Prep(alignment, elemSize * count); // Just in case alignment > int. + } + /// @endcond + + /// + /// Writes data necessary to finish a vector construction. + /// + public VectorOffset EndVector() + { + PutInt(_vectorNumElems); + return new VectorOffset(Offset); + } + + /// + /// Creates a vector of tables. + /// + /// Offsets of the tables. + public VectorOffset CreateVectorOfTables(Offset[] offsets) where T : struct + { + NotNested(); + StartVector(sizeof(int), offsets.Length, sizeof(int)); + for (int i = offsets.Length - 1; i >= 0; i--) AddOffset(offsets[i].Value); + return EndVector(); + } + + /// @cond FLATBUFFERS_INTERNAL + public void Nested(int obj) + { + // Structs are always stored inline, so need to be created right + // where they are used. You'll get this assert if you created it + // elsewhere. + if (obj != Offset) + throw new Exception( + "FlatBuffers: struct must be serialized inline."); + } + + public void NotNested() + { + // You should not be creating any other objects or strings/vectors + // while an object is being constructed + if (_vtableSize >= 0) + throw new Exception( + "FlatBuffers: object serialization must not be nested."); + } + + public void StartObject(int numfields) + { + if (numfields < 0) + throw new ArgumentOutOfRangeException("Flatbuffers: invalid numfields"); + + NotNested(); + + if (_vtable.Length < numfields) + _vtable = new int[numfields]; + + _vtableSize = numfields; + _objectStart = Offset; + } + + + // Set the current vtable at `voffset` to the current location in the + // buffer. + public void Slot(int voffset) + { + if (voffset >= _vtableSize) + throw new IndexOutOfRangeException("Flatbuffers: invalid voffset"); + + _vtable[voffset] = Offset; + } + + /// + /// Adds a Boolean to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddBool(int o, bool x, bool d) { if (ForceDefaults || x != d) { AddBool(x); Slot(o); } } + + /// + /// Adds a SByte to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddSbyte(int o, sbyte x, sbyte d) { if (ForceDefaults || x != d) { AddSbyte(x); Slot(o); } } + + /// + /// Adds a Byte to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddByte(int o, byte x, byte d) { if (ForceDefaults || x != d) { AddByte(x); Slot(o); } } + + /// + /// Adds a Int16 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddShort(int o, short x, int d) { if (ForceDefaults || x != d) { AddShort(x); Slot(o); } } + + /// + /// Adds a UInt16 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUshort(int o, ushort x, ushort d) { if (ForceDefaults || x != d) { AddUshort(x); Slot(o); } } + + /// + /// Adds an Int32 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddInt(int o, int x, int d) { if (ForceDefaults || x != d) { AddInt(x); Slot(o); } } + + /// + /// Adds a UInt32 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUint(int o, uint x, uint d) { if (ForceDefaults || x != d) { AddUint(x); Slot(o); } } + + /// + /// Adds an Int64 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddLong(int o, long x, long d) { if (ForceDefaults || x != d) { AddLong(x); Slot(o); } } + + /// + /// Adds a UInt64 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUlong(int o, ulong x, ulong d) { if (ForceDefaults || x != d) { AddUlong(x); Slot(o); } } + + /// + /// Adds a Single to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddFloat(int o, float x, double d) { if (ForceDefaults || x != d) { AddFloat(x); Slot(o); } } + + /// + /// Adds a Double to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddDouble(int o, double x, double d) { if (ForceDefaults || x != d) { AddDouble(x); Slot(o); } } + + /// + /// Adds a buffer offset to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddOffset(int o, int x, int d) { if (ForceDefaults || x != d) { AddOffset(x); Slot(o); } } + /// @endcond + + /// + /// Encode the string `s` in the buffer using UTF-8. + /// + /// The string to encode. + /// + /// The offset in the buffer where the encoded string starts. + /// + public StringOffset CreateString(string s) + { + NotNested(); + AddByte(0); + var utf8StringLen = Encoding.UTF8.GetByteCount(s); + StartVector(1, utf8StringLen, 1); + _bb.PutStringUTF8(_space -= utf8StringLen, s); + return new StringOffset(EndVector().Value); + } + + +#if ENABLE_SPAN_T + /// + /// Creates a string in the buffer from a Span containing + /// a UTF8 string. + /// + /// the UTF8 string to add to the buffer + /// + /// The offset in the buffer where the encoded string starts. + /// + public StringOffset CreateUTF8String(Span chars) + { + NotNested(); + AddByte(0); + var utf8StringLen = chars.Length; + StartVector(1, utf8StringLen, 1); + _space = _bb.Put(_space, chars); + return new StringOffset(EndVector().Value); + } +#endif + + /// @cond FLATBUFFERS_INTERNAL + // Structs are stored inline, so nothing additional is being added. + // `d` is always 0. + public void AddStruct(int voffset, int x, int d) + { + if (x != d) + { + Nested(x); + Slot(voffset); + } + } + + public int EndObject() + { + if (_vtableSize < 0) + throw new InvalidOperationException( + "Flatbuffers: calling endObject without a startObject"); + + AddInt((int)0); + var vtableloc = Offset; + // Write out the current vtable. + int i = _vtableSize - 1; + // Trim trailing zeroes. + for (; i >= 0 && _vtable[i] == 0; i--) {} + int trimmedSize = i + 1; + for (; i >= 0 ; i--) { + // Offset relative to the start of the table. + short off = (short)(_vtable[i] != 0 + ? vtableloc - _vtable[i] + : 0); + AddShort(off); + + // clear out written entry + _vtable[i] = 0; + } + + const int standardFields = 2; // The fields below: + AddShort((short)(vtableloc - _objectStart)); + AddShort((short)((trimmedSize + standardFields) * + sizeof(short))); + + // Search for an existing vtable that matches the current one. + int existingVtable = 0; + for (i = 0; i < _numVtables; i++) { + int vt1 = _bb.Length - _vtables[i]; + int vt2 = _space; + short len = _bb.GetShort(vt1); + if (len == _bb.GetShort(vt2)) { + for (int j = sizeof(short); j < len; j += sizeof(short)) { + if (_bb.GetShort(vt1 + j) != _bb.GetShort(vt2 + j)) { + goto endLoop; + } + } + existingVtable = _vtables[i]; + break; + } + + endLoop: { } + } + + if (existingVtable != 0) { + // Found a match: + // Remove the current vtable. + _space = _bb.Length - vtableloc; + // Point table to existing vtable. + _bb.PutInt(_space, existingVtable - vtableloc); + } else { + // No match: + // Add the location of the current vtable to the list of + // vtables. + if (_numVtables == _vtables.Length) + { + // Arrays.CopyOf(vtables num_vtables * 2); + var newvtables = new int[ _numVtables * 2]; + Array.Copy(_vtables, newvtables, _vtables.Length); + + _vtables = newvtables; + }; + _vtables[_numVtables++] = Offset; + // Point table to current vtable. + _bb.PutInt(_bb.Length - vtableloc, Offset - vtableloc); + } + + _vtableSize = -1; + return vtableloc; + } + + // This checks a required field has been set in a given table that has + // just been constructed. + public void Required(int table, int field) + { + int table_start = _bb.Length - table; + int vtable_start = table_start - _bb.GetInt(table_start); + bool ok = _bb.GetShort(vtable_start + field) != 0; + // If this fails, the caller will show what field needs to be set. + if (!ok) + throw new InvalidOperationException("FlatBuffers: field " + field + + " must be set"); + } + /// @endcond + + /// + /// Finalize a buffer, pointing to the given `root_table`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// Whether to prefix the size to the buffer. + /// + protected void Finish(int rootTable, bool sizePrefix) + { + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0)); + AddOffset(rootTable); + if (sizePrefix) { + AddInt(_bb.Length - _space); + } + _bb.Position = _space; + } + + /// + /// Finalize a buffer, pointing to the given `root_table`. + /// + /// + /// An offset to be added to the buffer. + /// + public void Finish(int rootTable) + { + Finish(rootTable, false); + } + + /// + /// Finalize a buffer, pointing to the given `root_table`, with the size prefixed. + /// + /// + /// An offset to be added to the buffer. + /// + public void FinishSizePrefixed(int rootTable) + { + Finish(rootTable, true); + } + + /// + /// Get the ByteBuffer representing the FlatBuffer. + /// + /// + /// This is typically only called after you call `Finish()`. + /// The actual data starts at the ByteBuffer's current position, + /// not necessarily at `0`. + /// + /// + /// Returns the ByteBuffer for this FlatBuffer. + /// + public ByteBuffer DataBuffer { get { return _bb; } } + + /// + /// A utility function to copy and return the ByteBuffer data as a + /// `byte[]`. + /// + /// + /// A full copy of the FlatBuffer data. + /// + public byte[] SizedByteArray() + { + return _bb.ToSizedArray(); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + /// + /// Whether to prefix the size to the buffer. + /// + protected void Finish(int rootTable, string fileIdentifier, bool sizePrefix) + { + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0) + + FlatBufferConstants.FileIdentifierLength); + if (fileIdentifier.Length != + FlatBufferConstants.FileIdentifierLength) + throw new ArgumentException( + "FlatBuffers: file identifier must be length " + + FlatBufferConstants.FileIdentifierLength, + "fileIdentifier"); + for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0; + i--) + { + AddByte((byte)fileIdentifier[i]); + } + Finish(rootTable, sizePrefix); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + public void Finish(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, false); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`, with the size prefixed. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + public void FinishSizePrefixed(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, true); + } + } +} + +/// @} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs new file mode 100644 index 000000000..68a4e9cad --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs @@ -0,0 +1,29 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace FlatBuffers +{ + internal static class FlatBufferConstants + { + public const int FileIdentifierLength = 4; + public const int SizePrefixLength = 4; + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs new file mode 100644 index 000000000..1be932c98 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs @@ -0,0 +1,28 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// This is the base for both structs and tables. + /// + internal interface IFlatbufferObject + { + void __init(int _i, ByteBuffer _bb); + + ByteBuffer ByteBuffer { get; } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs new file mode 100644 index 000000000..70b75b6de --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs @@ -0,0 +1,48 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// Offset class for typesafe assignments. + /// + internal struct Offset where T : struct + { + public int Value; + public Offset(int value) + { + Value = value; + } + } + + internal struct StringOffset + { + public int Value; + public StringOffset(int value) + { + Value = value; + } + } + + internal struct VectorOffset + { + public int Value; + public VectorOffset(int value) + { + Value = value; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs new file mode 100644 index 000000000..ca44ff920 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs @@ -0,0 +1,27 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// All structs in the generated code derive from this class, and add their own accessors. + /// + internal struct Struct + { + public int bb_pos; + public ByteBuffer bb; + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs new file mode 100644 index 000000000..f809072ba --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs @@ -0,0 +1,195 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Text; + +namespace FlatBuffers +{ + /// + /// All tables in the generated code derive from this struct, and add their own accessors. + /// + internal struct Table + { + public int bb_pos; + public ByteBuffer bb; + + public ByteBuffer ByteBuffer { get { return bb; } } + + // Look up a field in the vtable, return an offset into the object, or 0 if the field is not + // present. + public int __offset(int vtableOffset) + { + int vtable = bb_pos - bb.GetInt(bb_pos); + return vtableOffset < bb.GetShort(vtable) ? (int)bb.GetShort(vtable + vtableOffset) : 0; + } + + public static int __offset(int vtableOffset, int offset, ByteBuffer bb) + { + int vtable = bb.Length - offset; + return (int)bb.GetShort(vtable + vtableOffset - bb.GetInt(vtable)) + vtable; + } + + // Retrieve the relative offset stored at "offset" + public int __indirect(int offset) + { + return offset + bb.GetInt(offset); + } + + public static int __indirect(int offset, ByteBuffer bb) + { + return offset + bb.GetInt(offset); + } + + // Create a .NET String from UTF-8 data stored inside the flatbuffer. + public string __string(int offset) + { + offset += bb.GetInt(offset); + var len = bb.GetInt(offset); + var startPos = offset + sizeof(int); + return bb.GetStringUTF8(startPos, len); + } + + // Get the length of a vector whose offset is stored at "offset" in this object. + public int __vector_len(int offset) + { + offset += bb_pos; + offset += bb.GetInt(offset); + return bb.GetInt(offset); + } + + // Get the start of data of a vector whose offset is stored at "offset" in this object. + public int __vector(int offset) + { + offset += bb_pos; + return offset + bb.GetInt(offset) + sizeof(int); // data starts after the length + } + +#if ENABLE_SPAN_T + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // Spant<byte>. If the vector is not present in the ByteBuffer, + // then an empty span will be returned. + public Span __vector_as_span(int offset) + { + var o = this.__offset(offset); + if (0 == o) + { + return new Span(); + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToSpan(pos, len); + } +#else + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // ArraySegment<byte>. If the vector is not present in the ByteBuffer, + // then a null value will be returned. + public ArraySegment? __vector_as_arraysegment(int offset) + { + var o = this.__offset(offset); + if (0 == o) + { + return null; + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToArraySegment(pos, len); + } +#endif + + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // T[]. If the vector is not present in the ByteBuffer, then a null value will be + // returned. + public T[] __vector_as_array(int offset) + where T : struct + { + if(!BitConverter.IsLittleEndian) + { + throw new NotSupportedException("Getting typed arrays on a Big Endian " + + "system is not support"); + } + + var o = this.__offset(offset); + if (0 == o) + { + return null; + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToArray(pos, len); + } + + // Initialize any Table-derived type to point to the union at the given offset. + public T __union(int offset) where T : struct, IFlatbufferObject + { + offset += bb_pos; + T t = new T(); + t.__init(offset + bb.GetInt(offset), bb); + return t; + } + + public static bool __has_identifier(ByteBuffer bb, string ident) + { + if (ident.Length != FlatBufferConstants.FileIdentifierLength) + throw new ArgumentException("FlatBuffers: file identifier must be length " + FlatBufferConstants.FileIdentifierLength, "ident"); + + for (var i = 0; i < FlatBufferConstants.FileIdentifierLength; i++) + { + if (ident[i] != (char)bb.Get(bb.Position + sizeof(int) + i)) return false; + } + + return true; + } + + // Compare strings in the ByteBuffer. + public static int CompareStrings(int offset_1, int offset_2, ByteBuffer bb) + { + offset_1 += bb.GetInt(offset_1); + offset_2 += bb.GetInt(offset_2); + var len_1 = bb.GetInt(offset_1); + var len_2 = bb.GetInt(offset_2); + var startPos_1 = offset_1 + sizeof(int); + var startPos_2 = offset_2 + sizeof(int); + var len = Math.Min(len_1, len_2); + for(int i = 0; i < len; i++) { + byte b1 = bb.Get(i + startPos_1); + byte b2 = bb.Get(i + startPos_2); + if (b1 != b2) + return b1 - b2; + } + return len_1 - len_2; + } + + // Compare string from the ByteBuffer with the string object + public static int CompareStrings(int offset_1, byte[] key, ByteBuffer bb) + { + offset_1 += bb.GetInt(offset_1); + var len_1 = bb.GetInt(offset_1); + var len_2 = key.Length; + var startPos_1 = offset_1 + sizeof(int); + var len = Math.Min(len_1, len_2); + for (int i = 0; i < len; i++) { + byte b = bb.Get(i + startPos_1); + if (b != key[i]) + return b - key[i]; + } + return len_1 - len_2; + } + } +} diff --git a/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Footer.cs b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Footer.cs new file mode 100644 index 000000000..37dbfef94 --- /dev/null +++ b/src/arrow/csharp/src/Apache.Arrow/Flatbuf/Footer.cs @@ -0,0 +1,68 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Arrow File metadata +/// +internal struct Footer : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static Footer GetRootAsFooter(ByteBuffer _bb) { return GetRootAsFooter(_bb, new Footer()); } + public static Footer GetRootAsFooter(ByteBuffer _bb, Footer obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Footer __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public MetadataVersion Version { get { int o = __p.__offset(4); return o != 0 ? (MetadataVersion)__p.bb.GetShort(o + __p.bb_pos) : MetadataVersion.V1; } } + public Schema? Schema { get { int o = __p.__offset(6); return o != 0 ? (Schema?)(new Schema()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + public Block? Dictionaries(int j) { int o = __p.__offset(8); return o != 0 ? (Block?)(new Block()).__assign(__p.__vector(o) + j * 24, __p.bb) : null; } + public int DictionariesLength { get { int o = __p.__offset(8); return o != 0 ? __p.__vector_len(o) : 0; } } + public Block? RecordBatches(int j) { int o = __p.__offset(10); return o != 0 ? (Block?)(new Block()).__assign(__p.__vector(o) + j * 24, __p.bb) : null; } + public int RecordBatchesLength { get { int o = __p.__offset(10); return o != 0 ? __p.__vector_len(o) : 0; } } + /// User-defined metadata + public KeyValue? CustomMetadata(int j) { int o = __p.__offset(12); return o != 0 ? (KeyValue?)(new KeyValue()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; } + public int CustomMetadataLength { get { int o = __p.__offset(12); return o != 0 ? __p.__vector_len(o) : 0; } } + + public static Offset