diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
commit | e6918187568dbd01842d8d1d2c808ce16a894239 (patch) | |
tree | 64f88b554b444a49f656b6c656111a145cbbaa28 /src/arrow/csharp/test | |
parent | Initial commit. (diff) | |
download | ceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip |
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/arrow/csharp/test')
60 files changed, 8460 insertions, 0 deletions
diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj new file mode 100644 index 000000000..e38d538af --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>netcoreapp3.1</TargetFramework> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="BenchmarkDotNet" Version="0.12.1" /> + <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\src\Apache.Arrow\Apache.Arrow.csproj" /> + <ProjectReference Include="..\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj" /> + </ItemGroup> + +</Project> diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs new file mode 100644 index 000000000..4e491a2a6 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Memory; +using Apache.Arrow.Tests; +using Apache.Arrow.Types; +using BenchmarkDotNet.Attributes; +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Apache.Arrow.Benchmarks +{ + //[EtwProfiler] - needs elevated privileges + [MemoryDiagnoser] + public class ArrowReaderBenchmark + { + [Params(10_000, 1_000_000)] + public int Count { get; set; } + + private MemoryStream _memoryStream; + private static readonly MemoryAllocator s_allocator = new TestMemoryAllocator(); + + [GlobalSetup] + public async Task GlobalSetup() + { + RecordBatch batch = TestData.CreateSampleRecordBatch(length: Count); + _memoryStream = new MemoryStream(); + + ArrowStreamWriter writer = new ArrowStreamWriter(_memoryStream, batch.Schema); + await writer.WriteRecordBatchAsync(batch); + } + + [IterationSetup] + public void Setup() + { + _memoryStream.Position = 0; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemoryStream() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemoryStream_ManagedMemory() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream, s_allocator); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemory() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream.GetBuffer()); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + private static double SumAllNumbers(RecordBatch recordBatch) + { + double sum = 0; + + for (int k = 0; k < recordBatch.ColumnCount; k++) + { + var array = recordBatch.Arrays.ElementAt(k); + switch (recordBatch.Schema.GetFieldByIndex(k).DataType.TypeId) + { + case ArrowTypeId.Int64: + Int64Array int64Array = (Int64Array)array; + sum += Sum(int64Array); + break; + case ArrowTypeId.Double: + DoubleArray doubleArray = (DoubleArray)array; + sum += Sum(doubleArray); + break; + case ArrowTypeId.Decimal128: + Decimal128Array decimalArray = (Decimal128Array)array; + sum += Sum(decimalArray); + break; + } + } + return sum; + } + + private static double Sum(DoubleArray doubleArray) + { + double sum = 0; + ReadOnlySpan<double> values = doubleArray.Values; + for (int valueIndex = 0; valueIndex < values.Length; valueIndex++) + { + sum += values[valueIndex]; + } + return sum; + } + + private static long Sum(Int64Array int64Array) + { + long sum = 0; + ReadOnlySpan<long> values = int64Array.Values; + for (int valueIndex = 0; valueIndex < values.Length; valueIndex++) + { + sum += values[valueIndex]; + } + return sum; + } + + private static double Sum(Decimal128Array decimal128Array) + { + double sum = 0; + for (int valueIndex = 0; valueIndex < decimal128Array.Length; valueIndex++) + { + sum += (double)decimal128Array.GetValue(valueIndex); + } + return sum; + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs new file mode 100644 index 000000000..c791c9969 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Tests; +using BenchmarkDotNet.Attributes; +using System.IO; +using System.Threading.Tasks; + +namespace Apache.Arrow.Benchmarks +{ + //[EtwProfiler] - needs elevated privileges + [MemoryDiagnoser] + public class ArrowWriterBenchmark + { + [Params(10_000, 1_000_000)] + public int BatchLength{ get; set; } + + //Max column set count is 15 before reaching 2gb limit of memory stream + [Params(10, 14)] + public int ColumnSetCount { get; set; } + + private MemoryStream _memoryStream; + private RecordBatch _batch; + + [GlobalSetup] + public void GlobalSetup() + { + _batch = TestData.CreateSampleRecordBatch(BatchLength, ColumnSetCount, false); + _memoryStream = new MemoryStream(); + } + + [IterationSetup] + public void Setup() + { + _memoryStream.Position = 0; + } + + [Benchmark] + public async Task WriteBatch() + { + ArrowStreamWriter writer = new ArrowStreamWriter(_memoryStream, _batch.Schema); + await writer.WriteRecordBatchAsync(_batch); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs new file mode 100644 index 000000000..0f1410fcb --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using BenchmarkDotNet.Running; + +namespace Apache.Arrow.Benchmarks +{ + public static class Program + { + public static void Main(string[] args) + { + BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .Run(args); + } + } +}
\ No newline at end of file diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj new file mode 100644 index 000000000..5214b3a2c --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk.Web"> + + <PropertyGroup> + <TargetFramework>netcoreapp3.1</TargetFramework> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Grpc.AspNetCore" Version="2.33.1" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\src\Apache.Arrow.Flight.AspNetCore\Apache.Arrow.Flight.AspNetCore.csproj" /> + </ItemGroup> + +</Project> diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Extensions/AsyncStreamExtensions.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Extensions/AsyncStreamExtensions.cs new file mode 100644 index 000000000..eeb13a8ca --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Extensions/AsyncStreamExtensions.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Grpc.Core.Utils +{ + public static class AsyncStreamExtensions + { + /// <summary> + /// Reads the entire stream and creates a list containing all the elements read. + /// </summary> + public static async Task<List<T>> ToListAsync<T>(this IAsyncStreamReader<T> streamReader) + where T : class + { + var result = new List<T>(); + while (await streamReader.MoveNext().ConfigureAwait(false)) + { + result.Add(streamReader.Current); + } + return result; + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightHolder.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightHolder.cs new file mode 100644 index 000000000..34a527018 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightHolder.cs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Google.Protobuf; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class FlightHolder + { + private readonly FlightDescriptor _flightDescriptor; + private readonly Schema _schema; + private readonly string _location; + + //Not thread safe, but only used in tests + private readonly List<RecordBatchWithMetadata> _recordBatches = new List<RecordBatchWithMetadata>(); + + public FlightHolder(FlightDescriptor flightDescriptor, Schema schema, string location) + { + _flightDescriptor = flightDescriptor; + _schema = schema; + _location = location; + } + + public void AddBatch(RecordBatchWithMetadata recordBatchWithMetadata) + { + //Should validate schema here + _recordBatches.Add(recordBatchWithMetadata); + } + + public IEnumerable<RecordBatchWithMetadata> GetRecordBatches() + { + return _recordBatches.ToList(); + } + + public FlightInfo GetFlightInfo() + { + return new FlightInfo(_schema, _flightDescriptor, new List<FlightEndpoint>() + { + new FlightEndpoint(new FlightTicket(_flightDescriptor.Paths.FirstOrDefault()), new List<FlightLocation>(){ + new FlightLocation(_location) + }) + }); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightStore.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightStore.cs new file mode 100644 index 000000000..fe53d88e3 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/FlightStore.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class FlightStore + { + public Dictionary<FlightDescriptor, FlightHolder> Flights { get; set; } = new Dictionary<FlightDescriptor, FlightHolder>(); + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Program.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Program.cs new file mode 100644 index 000000000..2c5c002b3 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Program.cs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Hosting; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + // Additional configuration is required to successfully run gRPC on macOS. + // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder + .ConfigureKestrel((context, options) => + { + if (context.HostingEnvironment.IsDevelopment()) + { + options.Listen(IPEndPoint.Parse("0.0.0.0:5001"), l => l.Protocols = HttpProtocols.Http2); + } + }) + .UseStartup<Startup>(); + }); + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Properties/launchSettings.json b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Properties/launchSettings.json new file mode 100644 index 000000000..50e6f3dd6 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "Apache.Arrow.Flight.TestWeb": { + "commandName": "Project", + "launchBrowser": false, + "applicationUrl": "https://localhost:5001", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/RecordBatchWithMetadata.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/RecordBatchWithMetadata.cs new file mode 100644 index 000000000..2a4d7e726 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/RecordBatchWithMetadata.cs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Protobuf; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class RecordBatchWithMetadata + { + public RecordBatch RecordBatch { get; } + public ByteString Metadata { get; } + + public RecordBatchWithMetadata(RecordBatch recordBatch, ByteString metadata = null) + { + RecordBatch = recordBatch; + Metadata = metadata; + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs new file mode 100644 index 000000000..97c1af2f0 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + services.AddGrpc() + .AddFlightServer<TestFlightServer>(); + + services.AddSingleton(new FlightStore()); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapFlightEndpoint(); + + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909"); + }); + }); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs new file mode 100644 index 000000000..ae6e2e4b0 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Server; +using Grpc.Core; +using Grpc.Core.Utils; + +namespace Apache.Arrow.Flight.TestWeb +{ + public class TestFlightServer : FlightServer + { + private readonly FlightStore _flightStore; + + public TestFlightServer(FlightStore flightStore) + { + _flightStore = flightStore; + } + + public override async Task DoAction(FlightAction request, IAsyncStreamWriter<FlightResult> responseStream, ServerCallContext context) + { + switch (request.Type) + { + case "test": + await responseStream.WriteAsync(new FlightResult("test data")); + break; + default: + throw new NotImplementedException(); + } + } + + public override async Task DoGet(FlightTicket ticket, FlightServerRecordBatchStreamWriter responseStream, ServerCallContext context) + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor(ticket.Ticket.ToStringUtf8()); + + if(_flightStore.Flights.TryGetValue(flightDescriptor, out var flightHolder)) + { + var batches = flightHolder.GetRecordBatches(); + + + foreach(var batch in batches) + { + await responseStream.WriteAsync(batch.RecordBatch, batch.Metadata); + } + } + } + + public override async Task DoPut(FlightServerRecordBatchStreamReader requestStream, IAsyncStreamWriter<FlightPutResult> responseStream, ServerCallContext context) + { + var flightDescriptor = await requestStream.FlightDescriptor; + + if(!_flightStore.Flights.TryGetValue(flightDescriptor, out var flightHolder)) + { + flightHolder = new FlightHolder(flightDescriptor, await requestStream.Schema, $"http://{context.Host}"); + _flightStore.Flights.Add(flightDescriptor, flightHolder); + } + + while (await requestStream.MoveNext()) + { + flightHolder.AddBatch(new RecordBatchWithMetadata(requestStream.Current, requestStream.ApplicationMetadata.FirstOrDefault())); + await responseStream.WriteAsync(FlightPutResult.Empty); + } + } + + public override Task<FlightInfo> GetFlightInfo(FlightDescriptor request, ServerCallContext context) + { + if(_flightStore.Flights.TryGetValue(request, out var flightHolder)) + { + return Task.FromResult(flightHolder.GetFlightInfo()); + } + throw new RpcException(new Status(StatusCode.NotFound, "Flight not found")); + } + + public override Task<Schema> GetSchema(FlightDescriptor request, ServerCallContext context) + { + if(_flightStore.Flights.TryGetValue(request, out var flightHolder)) + { + return Task.FromResult(flightHolder.GetFlightInfo().Schema); + } + throw new RpcException(new Status(StatusCode.NotFound, "Flight not found")); + } + + public override async Task ListActions(IAsyncStreamWriter<FlightActionType> responseStream, ServerCallContext context) + { + await responseStream.WriteAsync(new FlightActionType("get", "get a flight")); + await responseStream.WriteAsync(new FlightActionType("put", "add a flight")); + await responseStream.WriteAsync(new FlightActionType("delete", "delete a flight")); + await responseStream.WriteAsync(new FlightActionType("test", "test action")); + } + + public override async Task ListFlights(FlightCriteria request, IAsyncStreamWriter<FlightInfo> responseStream, ServerCallContext context) + { + var flightInfos = _flightStore.Flights.Select(x => x.Value.GetFlightInfo()).ToList(); + + foreach(var flightInfo in flightInfos) + { + await responseStream.WriteAsync(flightInfo); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.Development.json b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.Development.json new file mode 100644 index 000000000..fe20c40cc --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.Development.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Grpc": "Information", + "Microsoft": "Information" + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.json b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.json new file mode 100644 index 000000000..1f292413b --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.TestWeb/appsettings.json @@ -0,0 +1,15 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*", + "Kestrel": { + "EndpointDefaults": { + "Protocols": "Http2" + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj new file mode 100644 index 000000000..31efc526e --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -0,0 +1,21 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>netcoreapp3.1</TargetFramework> + + <IsPackable>false</IsPackable> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" /> + <PackageReference Include="xunit" Version="2.4.0" /> + <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" /> + <PackageReference Include="coverlet.collector" Version="1.2.0" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\Apache.Arrow.Flight.TestWeb\Apache.Arrow.Flight.TestWeb.csproj" /> + <ProjectReference Include="..\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj" /> + </ItemGroup> + +</Project> diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightInfoComparer.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightInfoComparer.cs new file mode 100644 index 000000000..b92e5c4cc --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightInfoComparer.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Tests; +using Xunit; + +namespace Apache.Arrow.Flight.Tests +{ + public static class FlightInfoComparer + { + public static void Compare(FlightInfo expected, FlightInfo actual) + { + //Check endpoints + Assert.Equal(expected.Endpoints, actual.Endpoints); + + //Check flight descriptor + Assert.Equal(expected.Descriptor, actual.Descriptor); + + //Check schema + SchemaComparer.Compare(expected.Schema, actual.Schema); + + Assert.Equal(expected.TotalBytes, actual.TotalBytes); + + Assert.Equal(expected.TotalRecords, actual.TotalRecords); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightTests.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightTests.cs new file mode 100644 index 000000000..79025a217 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/FlightTests.cs @@ -0,0 +1,316 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Apache.Arrow.Flight.Client; +using Apache.Arrow.Flight.TestWeb; +using Apache.Arrow.Tests; +using Google.Protobuf; +using Grpc.Core.Utils; +using Xunit; + +namespace Apache.Arrow.Flight.Tests +{ + public class FlightTests : IDisposable + { + readonly TestWebFactory _testWebFactory; + readonly FlightClient _flightClient; + readonly FlightStore _flightStore; + public FlightTests() + { + _flightStore = new FlightStore(); + _testWebFactory = new TestWebFactory(_flightStore); + _flightClient = new FlightClient(_testWebFactory.GetChannel()); + } + + public void Dispose() + { + _testWebFactory.Dispose(); + } + + private RecordBatch CreateTestBatch(int startValue, int length) + { + var batchBuilder = new RecordBatch.Builder(); + Int32Array.Builder builder = new Int32Array.Builder(); + for (int i = 0; i < length; i++) + { + builder.Append(startValue + i); + } + batchBuilder.Append("test", true, builder.Build()); + return batchBuilder.Build(); + } + + + private IEnumerable<RecordBatchWithMetadata> GetStoreBatch(FlightDescriptor flightDescriptor) + { + Assert.Contains(flightDescriptor, (IReadOnlyDictionary<FlightDescriptor, FlightHolder>)_flightStore.Flights); + + var flightHolder = _flightStore.Flights[flightDescriptor]; + return flightHolder.GetRecordBatches(); + } + + private FlightInfo GivenStoreBatches(FlightDescriptor flightDescriptor, params RecordBatchWithMetadata[] batches) + { + var initialBatch = batches.FirstOrDefault(); + + var flightHolder = new FlightHolder(flightDescriptor, initialBatch.RecordBatch.Schema, _testWebFactory.GetAddress()); + + foreach(var batch in batches) + { + flightHolder.AddBatch(batch); + } + + _flightStore.Flights.Add(flightDescriptor, flightHolder); + + return flightHolder.GetFlightInfo(); + } + + [Fact] + public async Task TestPutSingleRecordBatch() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch = CreateTestBatch(0, 100); + + var putStream = _flightClient.StartPut(flightDescriptor); + await putStream.RequestStream.WriteAsync(expectedBatch); + await putStream.RequestStream.CompleteAsync(); + var putResults = await putStream.ResponseStream.ToListAsync(); + + Assert.Single(putResults); + + var actualBatches = GetStoreBatch(flightDescriptor); + Assert.Single(actualBatches); + + ArrowReaderVerifier.CompareBatches(expectedBatch, actualBatches.First().RecordBatch); + } + + [Fact] + public async Task TestPutTwoRecordBatches() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch1 = CreateTestBatch(0, 100); + var expectedBatch2 = CreateTestBatch(0, 100); + + var putStream = _flightClient.StartPut(flightDescriptor); + await putStream.RequestStream.WriteAsync(expectedBatch1); + await putStream.RequestStream.WriteAsync(expectedBatch2); + await putStream.RequestStream.CompleteAsync(); + var putResults = await putStream.ResponseStream.ToListAsync(); + + Assert.Equal(2, putResults.Count); + + var actualBatches = GetStoreBatch(flightDescriptor).ToList(); + Assert.Equal(2, actualBatches.Count); + + ArrowReaderVerifier.CompareBatches(expectedBatch1, actualBatches[0].RecordBatch); + ArrowReaderVerifier.CompareBatches(expectedBatch2, actualBatches[1].RecordBatch); + } + + [Fact] + public async Task TestGetSingleRecordBatch() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch = CreateTestBatch(0, 100); + + //Add batch to the in memory store + GivenStoreBatches(flightDescriptor, new RecordBatchWithMetadata(expectedBatch)); + + //Get the flight info for the ticket + var flightInfo = await _flightClient.GetInfo(flightDescriptor); + Assert.Single(flightInfo.Endpoints); + + var endpoint = flightInfo.Endpoints.FirstOrDefault(); + + var getStream = _flightClient.GetStream(endpoint.Ticket); + var resultList = await getStream.ResponseStream.ToListAsync(); + + Assert.Single(resultList); + ArrowReaderVerifier.CompareBatches(expectedBatch, resultList[0]); + } + + [Fact] + public async Task TestGetTwoRecordBatch() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch1 = CreateTestBatch(0, 100); + var expectedBatch2 = CreateTestBatch(100, 100); + + //Add batch to the in memory store + GivenStoreBatches(flightDescriptor, new RecordBatchWithMetadata(expectedBatch1), new RecordBatchWithMetadata(expectedBatch2)); + + //Get the flight info for the ticket + var flightInfo = await _flightClient.GetInfo(flightDescriptor); + Assert.Single(flightInfo.Endpoints); + + var endpoint = flightInfo.Endpoints.FirstOrDefault(); + + var getStream = _flightClient.GetStream(endpoint.Ticket); + var resultList = await getStream.ResponseStream.ToListAsync(); + + Assert.Equal(2, resultList.Count); + ArrowReaderVerifier.CompareBatches(expectedBatch1, resultList[0]); + ArrowReaderVerifier.CompareBatches(expectedBatch2, resultList[1]); + } + + [Fact] + public async Task TestGetFlightMetadata() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch1 = CreateTestBatch(0, 100); + + var expectedMetadata = ByteString.CopyFromUtf8("test metadata"); + var expectedMetadataList = new List<ByteString>() { expectedMetadata }; + + //Add batch to the in memory store + GivenStoreBatches(flightDescriptor, new RecordBatchWithMetadata(expectedBatch1, expectedMetadata)); + + //Get the flight info for the ticket + var flightInfo = await _flightClient.GetInfo(flightDescriptor); + Assert.Single(flightInfo.Endpoints); + + var endpoint = flightInfo.Endpoints.FirstOrDefault(); + + var getStream = _flightClient.GetStream(endpoint.Ticket); + + List<ByteString> actualMetadata = new List<ByteString>(); + while(await getStream.ResponseStream.MoveNext(default)) + { + actualMetadata.AddRange(getStream.ResponseStream.ApplicationMetadata); + } + + Assert.Equal(expectedMetadataList, actualMetadata); + } + + [Fact] + public async Task TestPutWithMetadata() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch = CreateTestBatch(0, 100); + var expectedMetadata = ByteString.CopyFromUtf8("test metadata"); + + var putStream = _flightClient.StartPut(flightDescriptor); + await putStream.RequestStream.WriteAsync(expectedBatch, expectedMetadata); + await putStream.RequestStream.CompleteAsync(); + var putResults = await putStream.ResponseStream.ToListAsync(); + + Assert.Single(putResults); + + var actualBatches = GetStoreBatch(flightDescriptor); + Assert.Single(actualBatches); + + ArrowReaderVerifier.CompareBatches(expectedBatch, actualBatches.First().RecordBatch); + Assert.Equal(expectedMetadata, actualBatches.First().Metadata); + } + + [Fact] + public async Task TestGetSchema() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch = CreateTestBatch(0, 100); + var expectedSchema = expectedBatch.Schema; + + GivenStoreBatches(flightDescriptor, new RecordBatchWithMetadata(expectedBatch)); + + var actualSchema = await _flightClient.GetSchema(flightDescriptor); + + SchemaComparer.Compare(expectedSchema, actualSchema); + } + + [Fact] + public async Task TestDoAction() + { + var expectedResult = new List<FlightResult>() + { + new FlightResult("test data") + }; + + var resultStream = _flightClient.DoAction(new FlightAction("test")); + var actualResult = await resultStream.ResponseStream.ToListAsync(); + + Assert.Equal(expectedResult, actualResult); + } + + [Fact] + public async Task TestListActions() + { + var expected = new List<FlightActionType>() + { + new FlightActionType("get", "get a flight"), + new FlightActionType("put", "add a flight"), + new FlightActionType("delete", "delete a flight"), + new FlightActionType("test", "test action") + }; + + var actual = await _flightClient.ListActions().ResponseStream.ToListAsync(); + + Assert.Equal(expected, actual); + } + + [Fact] + public async Task TestListFlights() + { + var flightDescriptor1 = FlightDescriptor.CreatePathDescriptor("test1"); + var flightDescriptor2 = FlightDescriptor.CreatePathDescriptor("test2"); + var expectedBatch = CreateTestBatch(0, 100); + + List<FlightInfo> expectedFlightInfo = new List<FlightInfo>(); + + expectedFlightInfo.Add(GivenStoreBatches(flightDescriptor1, new RecordBatchWithMetadata(expectedBatch))); + expectedFlightInfo.Add(GivenStoreBatches(flightDescriptor2, new RecordBatchWithMetadata(expectedBatch))); + + var listFlightStream = _flightClient.ListFlights(); + + var actualFlights = await listFlightStream.ResponseStream.ToListAsync(); + + for(int i = 0; i < expectedFlightInfo.Count; i++) + { + FlightInfoComparer.Compare(expectedFlightInfo[i], actualFlights[i]); + } + } + + [Fact] + public async Task TestGetBatchesWithAsyncEnumerable() + { + var flightDescriptor = FlightDescriptor.CreatePathDescriptor("test"); + var expectedBatch1 = CreateTestBatch(0, 100); + var expectedBatch2 = CreateTestBatch(100, 100); + + //Add batch to the in memory store + GivenStoreBatches(flightDescriptor, new RecordBatchWithMetadata(expectedBatch1), new RecordBatchWithMetadata(expectedBatch2)); + + //Get the flight info for the ticket + var flightInfo = await _flightClient.GetInfo(flightDescriptor); + Assert.Single(flightInfo.Endpoints); + + var endpoint = flightInfo.Endpoints.FirstOrDefault(); + + var getStream = _flightClient.GetStream(endpoint.Ticket); + + + List<RecordBatch> resultList = new List<RecordBatch>(); + await foreach(var recordBatch in getStream.ResponseStream) + { + resultList.Add(recordBatch); + } + + Assert.Equal(2, resultList.Count); + ArrowReaderVerifier.CompareBatches(expectedBatch1, resultList[0]); + ArrowReaderVerifier.CompareBatches(expectedBatch2, resultList[1]); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/TestWebFactory.cs b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/TestWebFactory.cs new file mode 100644 index 000000000..9e6ebc476 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Flight.Tests/TestWebFactory.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Text; +using Apache.Arrow.Flight.TestWeb; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace Apache.Arrow.Flight.Tests +{ + public class TestWebFactory : IDisposable + { + readonly IHost host; + + public TestWebFactory(FlightStore flightStore) + { + host = WebHostBuilder(flightStore).Build(); //Create the server + host.Start(); + AppContext.SetSwitch( + "System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + } + + private IHostBuilder WebHostBuilder(FlightStore flightStore) + { + return Host.CreateDefaultBuilder() + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder + .ConfigureKestrel(c => + { + c.Listen(IPEndPoint.Parse("0.0.0.0:5001"), l => l.Protocols = HttpProtocols.Http2); + }) + .UseStartup<Startup>() + .ConfigureServices(services => + { + services.AddSingleton(flightStore); + }); + }); + } + + public string GetAddress() + { + return "http://127.0.0.1:5001"; + } + + public GrpcChannel GetChannel() + { + return GrpcChannel.ForAddress(GetAddress()); + } + + public void Stop() + { + host.StopAsync().Wait(); + } + + public void Dispose() + { + Stop(); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj new file mode 100644 index 000000000..813734084 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Apache.Arrow.IntegrationTest.csproj @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>netcoreapp3.1</TargetFramework> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.21216.1" /> + <PackageReference Include="System.Text.Json" Version="5.0.2" /> + <ProjectReference Include="..\..\src\Apache.Arrow\Apache.Arrow.csproj" /> + <ProjectReference Include="..\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj" /> + </ItemGroup> + +</Project>
\ No newline at end of file diff --git a/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs new file mode 100644 index 000000000..d45662419 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs @@ -0,0 +1,609 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Numerics; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Apache.Arrow.Arrays; +using Apache.Arrow.Ipc; +using Apache.Arrow.Tests; +using Apache.Arrow.Types; + +namespace Apache.Arrow.IntegrationTest +{ + public class IntegrationCommand + { + public string Mode { get; set; } + public FileInfo JsonFileInfo { get; set; } + public FileInfo ArrowFileInfo { get; set; } + + public IntegrationCommand(string mode, FileInfo jsonFileInfo, FileInfo arrowFileInfo) + { + Mode = mode; + JsonFileInfo = jsonFileInfo; + ArrowFileInfo = arrowFileInfo; + } + + public async Task<int> Execute() + { + Func<Task<int>> commandDelegate = Mode switch + { + "validate" => Validate, + "json-to-arrow" => JsonToArrow, + "stream-to-file" => StreamToFile, + "file-to-stream" => FileToStream, + _ => () => + { + Console.WriteLine($"Mode '{Mode}' is not supported."); + return Task.FromResult(-1); + } + }; + return await commandDelegate(); + } + + private async Task<int> Validate() + { + JsonFile jsonFile = await ParseJsonFile(); + + using FileStream arrowFileStream = ArrowFileInfo.OpenRead(); + using ArrowFileReader reader = new ArrowFileReader(arrowFileStream); + int batchCount = await reader.RecordBatchCountAsync(); + + if (batchCount != jsonFile.Batches.Count) + { + Console.WriteLine($"Incorrect batch count. JsonFile: {jsonFile.Batches.Count}, ArrowFile: {batchCount}"); + return -1; + } + + Schema jsonFileSchema = CreateSchema(jsonFile.Schema); + Schema arrowFileSchema = reader.Schema; + + SchemaComparer.Compare(jsonFileSchema, arrowFileSchema); + + for (int i = 0; i < batchCount; i++) + { + RecordBatch arrowFileRecordBatch = reader.ReadNextRecordBatch(); + RecordBatch jsonFileRecordBatch = CreateRecordBatch(jsonFileSchema, jsonFile.Batches[i]); + + ArrowReaderVerifier.CompareBatches(jsonFileRecordBatch, arrowFileRecordBatch, strictCompare: false); + } + + // ensure there are no more batches in the file + if (reader.ReadNextRecordBatch() != null) + { + Console.WriteLine($"The ArrowFile has more RecordBatches than it should."); + return -1; + } + + return 0; + } + + private async Task<int> JsonToArrow() + { + JsonFile jsonFile = await ParseJsonFile(); + Schema schema = CreateSchema(jsonFile.Schema); + + using (FileStream fs = ArrowFileInfo.Create()) + { + ArrowFileWriter writer = new ArrowFileWriter(fs, schema); + await writer.WriteStartAsync(); + + foreach (var jsonRecordBatch in jsonFile.Batches) + { + RecordBatch batch = CreateRecordBatch(schema, jsonRecordBatch); + await writer.WriteRecordBatchAsync(batch); + } + await writer.WriteEndAsync(); + await fs.FlushAsync(); + } + + return 0; + } + + private RecordBatch CreateRecordBatch(Schema schema, JsonRecordBatch jsonRecordBatch) + { + if (schema.Fields.Count != jsonRecordBatch.Columns.Count) + { + throw new NotSupportedException($"jsonRecordBatch.Columns.Count '{jsonRecordBatch.Columns.Count}' doesn't match schema field count '{schema.Fields.Count}'"); + } + + List<IArrowArray> arrays = new List<IArrowArray>(jsonRecordBatch.Columns.Count); + for (int i = 0; i < jsonRecordBatch.Columns.Count; i++) + { + JsonFieldData data = jsonRecordBatch.Columns[i]; + Field field = schema.GetFieldByName(data.Name); + ArrayCreator creator = new ArrayCreator(data); + field.DataType.Accept(creator); + arrays.Add(creator.Array); + } + + return new RecordBatch(schema, arrays, jsonRecordBatch.Count); + } + + private static Schema CreateSchema(JsonSchema jsonSchema) + { + Schema.Builder builder = new Schema.Builder(); + for (int i = 0; i < jsonSchema.Fields.Count; i++) + { + builder.Field(f => CreateField(f, jsonSchema.Fields[i])); + } + return builder.Build(); + } + + private static void CreateField(Field.Builder builder, JsonField jsonField) + { + builder.Name(jsonField.Name) + .DataType(ToArrowType(jsonField.Type)) + .Nullable(jsonField.Nullable); + + if (jsonField.Metadata != null) + { + builder.Metadata(jsonField.Metadata); + } + } + + private static IArrowType ToArrowType(JsonArrowType type) + { + return type.Name switch + { + "bool" => BooleanType.Default, + "int" => ToIntArrowType(type), + "floatingpoint" => ToFloatingPointArrowType(type), + "decimal" => ToDecimalArrowType(type), + "binary" => BinaryType.Default, + "utf8" => StringType.Default, + "fixedsizebinary" => new FixedSizeBinaryType(type.ByteWidth), + "date" => ToDateArrowType(type), + "time" => ToTimeArrowType(type), + "timestamp" => ToTimestampArrowType(type), + _ => throw new NotSupportedException($"JsonArrowType not supported: {type.Name}") + }; + } + + private static IArrowType ToIntArrowType(JsonArrowType type) + { + return (type.BitWidth, type.IsSigned) switch + { + (8, true) => Int8Type.Default, + (8, false) => UInt8Type.Default, + (16, true) => Int16Type.Default, + (16, false) => UInt16Type.Default, + (32, true) => Int32Type.Default, + (32, false) => UInt32Type.Default, + (64, true) => Int64Type.Default, + (64, false) => UInt64Type.Default, + _ => throw new NotSupportedException($"Int type not supported: {type.BitWidth}, {type.IsSigned}") + }; + } + + private static IArrowType ToFloatingPointArrowType(JsonArrowType type) + { + return type.FloatingPointPrecision switch + { + "SINGLE" => FloatType.Default, + "DOUBLE" => DoubleType.Default, + _ => throw new NotSupportedException($"FloatingPoint type not supported: {type.FloatingPointPrecision}") + }; + } + + private static IArrowType ToDecimalArrowType(JsonArrowType type) + { + return type.BitWidth switch + { + 256 => new Decimal256Type(type.DecimalPrecision, type.Scale), + _ => new Decimal128Type(type.DecimalPrecision, type.Scale), + }; + } + + private static IArrowType ToDateArrowType(JsonArrowType type) + { + return type.Unit switch + { + "DAY" => Date32Type.Default, + "MILLISECOND" => Date64Type.Default, + _ => throw new NotSupportedException($"Date type not supported: {type.Unit}") + }; + } + + private static IArrowType ToTimeArrowType(JsonArrowType type) + { + return (type.Unit, type.BitWidth) switch + { + ("SECOND", 32) => new Time32Type(TimeUnit.Second), + ("SECOND", 64) => new Time64Type(TimeUnit.Second), + ("MILLISECOND", 32) => new Time32Type(TimeUnit.Millisecond), + ("MILLISECOND", 64) => new Time64Type(TimeUnit.Millisecond), + ("MICROSECOND", 32) => new Time32Type(TimeUnit.Microsecond), + ("MICROSECOND", 64) => new Time64Type(TimeUnit.Microsecond), + ("NANOSECOND", 32) => new Time32Type(TimeUnit.Nanosecond), + ("NANOSECOND", 64) => new Time64Type(TimeUnit.Nanosecond), + _ => throw new NotSupportedException($"Time type not supported: {type.Unit}, {type.BitWidth}") + }; + } + + private static IArrowType ToTimestampArrowType(JsonArrowType type) + { + return type.Unit switch + { + "SECOND" => new TimestampType(TimeUnit.Second, type.Timezone), + "MILLISECOND" => new TimestampType(TimeUnit.Millisecond, type.Timezone), + "MICROSECOND" => new TimestampType(TimeUnit.Microsecond, type.Timezone), + "NANOSECOND" => new TimestampType(TimeUnit.Nanosecond, type.Timezone), + _ => throw new NotSupportedException($"Time type not supported: {type.Unit}, {type.BitWidth}") + }; + } + + private class ArrayCreator : + IArrowTypeVisitor<BooleanType>, + IArrowTypeVisitor<Int8Type>, + IArrowTypeVisitor<Int16Type>, + IArrowTypeVisitor<Int32Type>, + IArrowTypeVisitor<Int64Type>, + IArrowTypeVisitor<UInt8Type>, + IArrowTypeVisitor<UInt16Type>, + IArrowTypeVisitor<UInt32Type>, + IArrowTypeVisitor<UInt64Type>, + IArrowTypeVisitor<FloatType>, + IArrowTypeVisitor<DoubleType>, + IArrowTypeVisitor<Decimal128Type>, + IArrowTypeVisitor<Decimal256Type>, + IArrowTypeVisitor<Date32Type>, + IArrowTypeVisitor<Date64Type>, + IArrowTypeVisitor<TimestampType>, + IArrowTypeVisitor<StringType>, + IArrowTypeVisitor<BinaryType>, + IArrowTypeVisitor<FixedSizeBinaryType>, + IArrowTypeVisitor<ListType>, + IArrowTypeVisitor<StructType> + { + private JsonFieldData JsonFieldData { get; } + public IArrowArray Array { get; private set; } + + public ArrayCreator(JsonFieldData jsonFieldData) + { + JsonFieldData = jsonFieldData; + } + + public void Visit(BooleanType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer.BitmapBuilder valueBuilder = new ArrowBuffer.BitmapBuilder(validityBuffer.Length); + + var json = JsonFieldData.Data.GetRawText(); + bool[] values = JsonSerializer.Deserialize<bool[]>(json); + + foreach (bool value in values) + { + valueBuilder.Append(value); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = new BooleanArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + public void Visit(Int8Type type) => GenerateArray<sbyte, Int8Array>((v, n, c, nc, o) => new Int8Array(v, n, c, nc, o)); + public void Visit(Int16Type type) => GenerateArray<short, Int16Array>((v, n, c, nc, o) => new Int16Array(v, n, c, nc, o)); + public void Visit(Int32Type type) => GenerateArray<int, Int32Array>((v, n, c, nc, o) => new Int32Array(v, n, c, nc, o)); + public void Visit(Int64Type type) => GenerateLongArray<long, Int64Array>((v, n, c, nc, o) => new Int64Array(v, n, c, nc, o), s => long.Parse(s)); + public void Visit(UInt8Type type) => GenerateArray<byte, UInt8Array>((v, n, c, nc, o) => new UInt8Array(v, n, c, nc, o)); + public void Visit(UInt16Type type) => GenerateArray<ushort, UInt16Array>((v, n, c, nc, o) => new UInt16Array(v, n, c, nc, o)); + public void Visit(UInt32Type type) => GenerateArray<uint, UInt32Array>((v, n, c, nc, o) => new UInt32Array(v, n, c, nc, o)); + public void Visit(UInt64Type type) => GenerateLongArray<ulong, UInt64Array>((v, n, c, nc, o) => new UInt64Array(v, n, c, nc, o), s => ulong.Parse(s)); + public void Visit(FloatType type) => GenerateArray<float, FloatArray>((v, n, c, nc, o) => new FloatArray(v, n, c, nc, o)); + public void Visit(DoubleType type) => GenerateArray<double, DoubleArray>((v, n, c, nc, o) => new DoubleArray(v, n, c, nc, o)); + + public void Visit(Decimal128Type type) + { + Array = new Decimal128Array(GetDecimalArrayData(type)); + } + + public void Visit(Decimal256Type type) + { + Array = new Decimal256Array(GetDecimalArrayData(type)); + } + + private ArrayData GetDecimalArrayData(FixedSizeBinaryType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json, s_options); + + Span<byte> buffer = stackalloc byte[type.ByteWidth]; + + ArrowBuffer.Builder<byte> valueBuilder = new ArrowBuffer.Builder<byte>(); + foreach (string value in values) + { + buffer.Fill(0); + + BigInteger bigInteger = BigInteger.Parse(value); + if (!bigInteger.TryWriteBytes(buffer, out int bytesWritten, false, !BitConverter.IsLittleEndian)) + { + throw new InvalidDataException($"Decimal data was too big to fit into {type.BitWidth} bits."); + } + + if (bigInteger.Sign == -1) + { + buffer.Slice(bytesWritten).Fill(255); + } + + valueBuilder.Append(buffer); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + return new ArrayData(type, JsonFieldData.Count, nullCount, 0, new[] { validityBuffer, valueBuffer }); + } + + public void Visit(Date32Type type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder<int> valueBuilder = new ArrowBuffer.Builder<int>(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + int[] values = JsonSerializer.Deserialize<int[]>(json, s_options); + + foreach (int value in values) + { + valueBuilder.Append(value); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = new Date32Array( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + public void Visit(Date64Type type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder<long> valueBuilder = new ArrowBuffer.Builder<long>(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json, s_options); + + foreach (string value in values) + { + valueBuilder.Append(long.Parse(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = new Date64Array( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + public void Visit(TimestampType type) + { + throw new NotImplementedException(); + } + + public void Visit(StringType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer offsetBuffer = GetOffsetBuffer(); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json, s_options); + + ArrowBuffer.Builder<byte> valueBuilder = new ArrowBuffer.Builder<byte>(); + foreach (string value in values) + { + valueBuilder.Append(Encoding.UTF8.GetBytes(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + Array = new StringArray(JsonFieldData.Count, offsetBuffer, valueBuffer, validityBuffer, nullCount); + } + + public void Visit(BinaryType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + ArrowBuffer offsetBuffer = GetOffsetBuffer(); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json, s_options); + + ArrowBuffer.Builder<byte> valueBuilder = new ArrowBuffer.Builder<byte>(); + foreach (string value in values) + { + valueBuilder.Append(ConvertHexStringToByteArray(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + ArrayData arrayData = new ArrayData(type, JsonFieldData.Count, nullCount, 0, new[] { validityBuffer, offsetBuffer, valueBuffer }); + Array = new BinaryArray(arrayData); + } + + public void Visit(FixedSizeBinaryType type) + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json, s_options); + + ArrowBuffer.Builder<byte> valueBuilder = new ArrowBuffer.Builder<byte>(); + foreach (string value in values) + { + valueBuilder.Append(ConvertHexStringToByteArray(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + ArrayData arrayData = new ArrayData(type, JsonFieldData.Count, nullCount, 0, new[] { validityBuffer, valueBuffer }); + Array = new FixedSizeBinaryArray(arrayData); + } + + public void Visit(ListType type) + { + throw new NotImplementedException(); + } + + public void Visit(StructType type) + { + throw new NotImplementedException(); + } + + private static byte[] ConvertHexStringToByteArray(string hexString) + { + byte[] data = new byte[hexString.Length / 2]; + for (int index = 0; index < data.Length; index++) + { + data[index] = byte.Parse(hexString.AsSpan(index * 2, 2), NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + return data; + } + + private static readonly JsonSerializerOptions s_options = new JsonSerializerOptions() + { + Converters = + { + new ByteArrayConverter() + } + }; + + private void GenerateArray<T, TArray>(Func<ArrowBuffer, ArrowBuffer, int, int, int, TArray> createArray) + where TArray : PrimitiveArray<T> + where T : struct + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder<T> valueBuilder = new ArrowBuffer.Builder<T>(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + T[] values = JsonSerializer.Deserialize<T[]>(json, s_options); + + foreach (T value in values) + { + valueBuilder.Append(value); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = createArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + private void GenerateLongArray<T, TArray>(Func<ArrowBuffer, ArrowBuffer, int, int, int, TArray> createArray, Func<string, T> parse) + where TArray : PrimitiveArray<T> + where T : struct + { + ArrowBuffer validityBuffer = GetValidityBuffer(out int nullCount); + + ArrowBuffer.Builder<T> valueBuilder = new ArrowBuffer.Builder<T>(JsonFieldData.Count); + var json = JsonFieldData.Data.GetRawText(); + string[] values = JsonSerializer.Deserialize<string[]>(json); + + foreach (string value in values) + { + valueBuilder.Append(parse(value)); + } + ArrowBuffer valueBuffer = valueBuilder.Build(); + + Array = createArray( + valueBuffer, validityBuffer, + JsonFieldData.Count, nullCount, 0); + } + + private ArrowBuffer GetOffsetBuffer() + { + ArrowBuffer.Builder<int> valueOffsets = new ArrowBuffer.Builder<int>(JsonFieldData.Offset.Length); + valueOffsets.AppendRange(JsonFieldData.Offset); + return valueOffsets.Build(default); + } + + private ArrowBuffer GetValidityBuffer(out int nullCount) + { + if (JsonFieldData.Validity == null) + { + nullCount = 0; + return ArrowBuffer.Empty; + } + + ArrowBuffer.BitmapBuilder validityBuilder = new ArrowBuffer.BitmapBuilder(JsonFieldData.Validity.Length); + validityBuilder.AppendRange(JsonFieldData.Validity); + + nullCount = validityBuilder.UnsetBitCount; + return validityBuilder.Build(); + } + + public void Visit(IArrowType type) + { + throw new NotImplementedException($"{type.Name} not implemented"); + } + } + + private async Task<int> StreamToFile() + { + using ArrowStreamReader reader = new ArrowStreamReader(Console.OpenStandardInput()); + + RecordBatch batch = await reader.ReadNextRecordBatchAsync(); + + using FileStream fileStream = ArrowFileInfo.OpenWrite(); + using ArrowFileWriter writer = new ArrowFileWriter(fileStream, reader.Schema); + await writer.WriteStartAsync(); + + while (batch != null) + { + await writer.WriteRecordBatchAsync(batch); + + batch = await reader.ReadNextRecordBatchAsync(); + } + + await writer.WriteEndAsync(); + + return 0; + } + + private async Task<int> FileToStream() + { + using FileStream fileStream = ArrowFileInfo.OpenRead(); + using ArrowFileReader fileReader = new ArrowFileReader(fileStream); + + // read the record batch count to initialize the Schema + await fileReader.RecordBatchCountAsync(); + + using ArrowStreamWriter writer = new ArrowStreamWriter(Console.OpenStandardOutput(), fileReader.Schema); + await writer.WriteStartAsync(); + + RecordBatch batch; + while ((batch = fileReader.ReadNextRecordBatch()) != null) + { + await writer.WriteRecordBatchAsync(batch); + } + + await writer.WriteEndAsync(); + + return 0; + } + + private async ValueTask<JsonFile> ParseJsonFile() + { + using var fileStream = JsonFileInfo.OpenRead(); + JsonSerializerOptions options = new JsonSerializerOptions() + { + PropertyNamingPolicy = JsonFileNamingPolicy.Instance, + }; + options.Converters.Add(new ValidityConverter()); + + return await JsonSerializer.DeserializeAsync<JsonFile>(fileStream, options); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs new file mode 100644 index 000000000..f074afc01 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Apache.Arrow.IntegrationTest +{ + public class JsonFile + { + public JsonSchema Schema { get; set; } + public List<JsonRecordBatch> Batches { get; set; } + //public List<DictionaryBatch> Dictionaries {get;set;} + } + + public class JsonSchema + { + public List<JsonField> Fields { get; set; } + public JsonMetadata Metadata { get; set; } + } + + public class JsonField + { + public string Name { get; set; } + public bool Nullable { get; set; } + public JsonArrowType Type { get; set; } + public List<JsonField> Children { get; set; } + public JsonDictionaryIndex Dictionary { get; set; } + public JsonMetadata Metadata { get; set; } + } + + public class JsonArrowType + { + public string Name { get; set; } + + // int fields + public int BitWidth { get; set; } + public bool IsSigned { get; set; } + + // floating point fields + [JsonIgnore] + public string FloatingPointPrecision => ExtensionData["precision"].GetString(); + + // decimal fields + [JsonIgnore] + public int DecimalPrecision => ExtensionData["precision"].GetInt32(); + public int Scale { get; set; } + + // date and time fields + public string Unit { get; set; } + // timestamp fields + public string Timezone { get; set; } + + // FixedSizeBinary fields + public int ByteWidth { get; set; } + + [JsonExtensionData] + public Dictionary<string, JsonElement> ExtensionData { get; set; } + } + + public class JsonDictionaryIndex + { + public int Id { get; set; } + public JsonArrowType Type { get; set; } + public bool IsOrdered { get; set; } + } + + public class JsonMetadata : List<KeyValuePair<string, string>> + { + } + + public class JsonRecordBatch + { + public int Count { get; set; } + public List<JsonFieldData> Columns { get; set; } + } + + public class JsonFieldData + { + public string Name { get; set; } + public int Count { get; set; } + public bool[] Validity { get; set; } + public int[] Offset { get; set; } + public int[] TypeId { get; set; } + public JsonElement Data { get; set; } + public List<JsonFieldData> Children { get; set; } + } + + internal sealed class ValidityConverter : JsonConverter<bool> + { + public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.True) return true; + if (reader.TokenType == JsonTokenType.False) return false; + + if (typeToConvert != typeof(bool) || reader.TokenType != JsonTokenType.Number) + { + throw new InvalidOperationException($"Unexpected bool data: {reader.TokenType}"); + } + + int value = reader.GetInt32(); + if (value == 0) return false; + if (value == 1) return true; + + throw new InvalidOperationException($"Unexpected bool value: {value}"); + } + + public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) => throw new NotImplementedException(); + } + + internal sealed class ByteArrayConverter : JsonConverter<byte[]> + { + public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new InvalidOperationException($"Unexpected byte[] token: {reader.TokenType}"); + } + + List<byte> values = new List<byte>(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + { + return values.ToArray(); + } + + if (reader.TokenType != JsonTokenType.Number) + { + throw new InvalidOperationException($"Unexpected byte token: {reader.TokenType}"); + } + + values.Add(reader.GetByte()); + } + + throw new InvalidOperationException("Unexpectedly reached the end of the reader"); + } + + public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) => throw new NotImplementedException(); + } + + internal sealed class JsonFileNamingPolicy : JsonNamingPolicy + { + public static JsonFileNamingPolicy Instance { get; } = new JsonFileNamingPolicy(); + + public override string ConvertName(string name) + { + if (name == "Validity") + { + return "VALIDITY"; + } + else if (name == "Offset") + { + return "OFFSET"; + } + else if (name == "TypeId") + { + return "TYPE_ID"; + } + else if (name == "Data") + { + return "DATA"; + } + else + { + return CamelCase.ConvertName(name); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Program.cs b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Program.cs new file mode 100644 index 000000000..243269386 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.IntegrationTest/Program.cs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Apache.Arrow.IntegrationTest +{ + public class Program + { + public static async Task<int> Main(string[] args) + { + var integrationTestCommand = new RootCommand + { + new Option<string>( + "--mode", + description: "Which command to run"), + new Option<FileInfo>( + new[] { "--json-file", "-j" }, + "The JSON file to interact with"), + new Option<FileInfo>( + new[] { "--arrow-file", "-a" }, + "The arrow file to interact with") + }; + + integrationTestCommand.Description = "Integration test app for Apache.Arrow .NET Library."; + + integrationTestCommand.Handler = CommandHandler.Create<string, FileInfo, FileInfo>(async (mode, j, a) => + { + var integrationCommand = new IntegrationCommand(mode, j, a); + await integrationCommand.Execute(); + }); + return await integrationTestCommand.InvokeAsync(args); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/src/arrow/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj new file mode 100644 index 000000000..a725fe57e --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>netcoreapp3.1</TargetFramework> + <AllowUnsafeBlocks>true</AllowUnsafeBlocks> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" /> + <PackageReference Include="xunit" Version="2.4.0" /> + <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\src\Apache.Arrow\Apache.Arrow.csproj" /> + </ItemGroup> + +</Project>
\ No newline at end of file diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs new file mode 100644 index 000000000..41078998b --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrayBuilderTests + { + // TODO: Test various builder invariants (Append, AppendRange, Clear, Resize, Reserve, etc) + + [Fact] + public void PrimitiveArrayBuildersProduceExpectedArray() + { + TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(10).Append(20).Append(30)); + TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(10).Append(20).Append(30)); + } + + [Fact] + public void PrimitiveArrayBuildersProduceExpectedArrayWithNulls() + { + TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09); + TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09); + TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09); + } + + [Fact] + public void BooleanArrayBuilderProducersExpectedArray() + { + TestArrayBuilder<BooleanArray, BooleanArray.Builder>(x => x.Append(true).Append(false).Append(true)); + TestArrayBuilder<BooleanArray, BooleanArray.Builder>(x => x.Append(true).AppendNull().Append(false).Append(true), 4, 1, 0x0D); + } + + [Fact] + public void StringArrayBuilderHandlesNullsAndEmptyStrings() + { + var stringArray = TestArrayBuilder<StringArray, StringArray.Builder>(x => x.Append("123").Append(null).AppendNull().Append(string.Empty), 4, 2, 0x09); + Assert.Equal("123", stringArray.GetString(0)); + Assert.Null(stringArray.GetString(1)); + Assert.Null(stringArray.GetString(2)); + Assert.Equal(string.Empty, stringArray.GetString(3)); + } + + + [Fact] + public void ListArrayBuilder() + { + var listBuilder = new ListArray.Builder(StringType.Default); + var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder; + Assert.NotNull(valueBuilder); + listBuilder.Append(); + valueBuilder.Append("1"); + listBuilder.AppendNull(); + listBuilder.Append(); + valueBuilder.Append("22").Append("33"); + listBuilder.Append(); + valueBuilder.Append("444").AppendNull().Append("555").Append("666"); + + var list = listBuilder.Build(); + + Assert.Equal( + new List<string> { "1" }, + ConvertStringArrayToList(list.GetSlicedValues(0) as StringArray)); + Assert.Null(list.GetSlicedValues(1)); + Assert.Equal( + new List<string> { "22", "33" }, + ConvertStringArrayToList(list.GetSlicedValues(2) as StringArray)); + Assert.Equal( + new List<string> { "444", null, "555", "666" }, + ConvertStringArrayToList(list.GetSlicedValues(3) as StringArray)); + + List<string> ConvertStringArrayToList(StringArray array) + { + var length = array.Length; + var resultList = new List<string>(length); + for (var index = 0; index < length; index++) + { + resultList.Add(array.GetString(index)); + } + return resultList; + } + } + + [Fact] + public void ListArrayBuilderValidityBuffer() + { + ListArray listArray = new ListArray.Builder(Int64Type.Default).Append().AppendNull().Build(); + Assert.False(listArray.IsValid(2)); + } + + [Fact] + public void NestedListArrayBuilder() + { + var childListType = new ListType(Int64Type.Default); + var parentListBuilder = new ListArray.Builder(childListType); + var childListBuilder = parentListBuilder.ValueBuilder as ListArray.Builder; + Assert.NotNull(childListBuilder); + var valueBuilder = childListBuilder.ValueBuilder as Int64Array.Builder; + Assert.NotNull(valueBuilder); + + parentListBuilder.Append(); + childListBuilder.Append(); + valueBuilder.Append(1); + childListBuilder.Append(); + valueBuilder.Append(2).Append(3); + parentListBuilder.Append(); + childListBuilder.Append(); + valueBuilder.Append(4).Append(5).Append(6).Append(7); + parentListBuilder.Append(); + childListBuilder.Append(); + valueBuilder.Append(8).Append(9).Append(10).Append(11).Append(12); + + var parentList = parentListBuilder.Build(); + + var childList1 = (ListArray)parentList.GetSlicedValues(0); + var childList2 = (ListArray)parentList.GetSlicedValues(1); + var childList3 = (ListArray)parentList.GetSlicedValues(2); + + Assert.Equal(2, childList1.Length); + Assert.Equal(1, childList2.Length); + Assert.Equal(1, childList3.Length); + Assert.Equal( + new List<long?> { 1 }, + ((Int64Array)childList1.GetSlicedValues(0)).ToList()); + Assert.Equal( + new List<long?> { 2, 3 }, + ((Int64Array)childList1.GetSlicedValues(1)).ToList()); + Assert.Equal( + new List<long?> { 4, 5, 6, 7 }, + ((Int64Array)childList2.GetSlicedValues(0)).ToList()); + Assert.Equal( + new List<long?> { 8, 9, 10, 11, 12 }, + ((Int64Array)childList3.GetSlicedValues(0)).ToList()); + } + + public class TimestampArrayBuilder + { + [Fact] + public void ProducesExpectedArray() + { + var now = DateTimeOffset.UtcNow.ToLocalTime(); + var timestampType = new TimestampType(TimeUnit.Nanosecond, TimeZoneInfo.Local); + var array = new TimestampArray.Builder(timestampType) + .Append(now) + .Build(); + + Assert.Equal(1, array.Length); + var value = array.GetTimestamp(0); + Assert.NotNull(value); + Assert.Equal(now, value.Value); + + timestampType = new TimestampType(TimeUnit.Microsecond, TimeZoneInfo.Local); + array = new TimestampArray.Builder(timestampType) + .Append(now) + .Build(); + + Assert.Equal(1, array.Length); + value = array.GetTimestamp(0); + Assert.NotNull(value); + Assert.Equal(now.Truncate(TimeSpan.FromTicks(10)), value.Value); + + timestampType = new TimestampType(TimeUnit.Millisecond, TimeZoneInfo.Local); + array = new TimestampArray.Builder(timestampType) + .Append(now) + .Build(); + + Assert.Equal(1, array.Length); + value = array.GetTimestamp(0); + Assert.NotNull(value); + Assert.Equal(now.Truncate(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond)), value.Value); + } + } + + private static TArray TestArrayBuilder<TArray, TArrayBuilder>(Action<TArrayBuilder> action, int expectedLength = 3, int expectedNullCount = 0, int expectedNulls = 0) + where TArray : IArrowArray + where TArrayBuilder : IArrowArrayBuilder<TArray>, new() + { + var builder = new TArrayBuilder(); + action(builder); + var array = builder.Build(default); + + Assert.IsAssignableFrom<TArray>(array); + Assert.NotNull(array); + Assert.Equal(expectedLength, array.Length); + Assert.Equal(expectedNullCount, array.NullCount); + if (expectedNulls != 0) + { + Assert.True(array.Data.Buffers[0].Span.Slice(0, 1).SequenceEqual(new ReadOnlySpan<byte>(BitConverter.GetBytes(expectedNulls).Take(1).ToArray()))); + } + return array; + } + + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayDataConcatenatorTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayDataConcatenatorTests.cs new file mode 100644 index 000000000..9f034b9d0 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayDataConcatenatorTests.cs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Apache.Arrow.Memory; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrayDataConcatenatorTests + { + [Fact] + public void TestNullOrEmpty() + { + Assert.Null(ArrayDataConcatenatorReflector.InvokeConcatenate(null)); + Assert.Null(ArrayDataConcatenatorReflector.InvokeConcatenate(new List<ArrayData>())); + } + + [Fact] + public void TestSingleElement() + { + Int32Array array = new Int32Array.Builder().Append(1).Append(2).Build(); + ArrayData actualArray = ArrayDataConcatenatorReflector.InvokeConcatenate(new[] { array.Data }); + ArrowReaderVerifier.CompareArrays(array, ArrowArrayFactory.BuildArray(actualArray)); + } + + private static class ArrayDataConcatenatorReflector + { + private static readonly MethodInfo s_concatenateInfo = typeof(ArrayData).Assembly.GetType("Apache.Arrow.ArrayDataConcatenator") + .GetMethod("Concatenate", BindingFlags.Static | BindingFlags.NonPublic); + + internal static ArrayData InvokeConcatenate(IReadOnlyList<ArrayData> arrayDataList, MemoryAllocator allocator = default) + { + return s_concatenateInfo.Invoke(null, new object[] { arrayDataList, allocator }) as ArrayData; + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs new file mode 100644 index 000000000..f75111b66 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrayTypeComparer : + IArrowTypeVisitor<TimestampType>, + IArrowTypeVisitor<Date32Type>, + IArrowTypeVisitor<Date64Type>, + IArrowTypeVisitor<Time32Type>, + IArrowTypeVisitor<Time64Type>, + IArrowTypeVisitor<FixedSizeBinaryType>, + IArrowTypeVisitor<ListType>, + IArrowTypeVisitor<StructType> + { + private readonly IArrowType _expectedType; + + public ArrayTypeComparer(IArrowType expectedType) + { + Debug.Assert(expectedType != null); + _expectedType = expectedType; + } + + public void Visit(TimestampType actualType) + { + Assert.IsAssignableFrom<TimestampType>(_expectedType); + + var expectedType = (TimestampType)_expectedType; + + Assert.Equal(expectedType.Timezone, actualType.Timezone); + Assert.Equal(expectedType.Unit, actualType.Unit); + } + + public void Visit(Date32Type actualType) + { + Assert.IsAssignableFrom<Date32Type>(_expectedType); + var expectedType = (Date32Type)_expectedType; + + Assert.Equal(expectedType.Unit, actualType.Unit); + } + + public void Visit(Date64Type actualType) + { + Assert.IsAssignableFrom<Date64Type>(_expectedType); + var expectedType = (Date64Type)_expectedType; + + Assert.Equal(expectedType.Unit, actualType.Unit); + } + + public void Visit(Time32Type actualType) + { + Assert.IsAssignableFrom<Time32Type>(_expectedType); + var expectedType = (Time32Type)_expectedType; + + Assert.Equal(expectedType.Unit, actualType.Unit); + } + + public void Visit(Time64Type actualType) + { + Assert.IsAssignableFrom<Time64Type>(_expectedType); + var expectedType = (Time64Type)_expectedType; + + Assert.Equal(expectedType.Unit, actualType.Unit); + } + + public void Visit(FixedSizeBinaryType actualType) + { + Assert.IsAssignableFrom<FixedSizeBinaryType>(_expectedType); + var expectedType = (FixedSizeBinaryType)_expectedType; + + Assert.Equal(expectedType.ByteWidth, actualType.ByteWidth); + } + + public void Visit(ListType actualType) + { + Assert.IsAssignableFrom<ListType>(_expectedType); + var expectedType = (ListType)_expectedType; + + CompareNested(expectedType, actualType); + } + + public void Visit(StructType actualType) + { + Assert.IsAssignableFrom<StructType>(_expectedType); + var expectedType = (StructType)_expectedType; + + CompareNested(expectedType, actualType); + } + + private static void CompareNested(NestedType expectedType, NestedType actualType) + { + Assert.Equal(expectedType.Fields.Count, actualType.Fields.Count); + + for (int i = 0; i < expectedType.Fields.Count; i++) + { + FieldComparer.Compare(expectedType.Fields[i], actualType.Fields[i]); + } + } + + public void Visit(IArrowType actualType) + { + Assert.IsAssignableFrom(actualType.GetType(), _expectedType); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs new file mode 100644 index 000000000..69894ab3c --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Reflection; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Tests +{ + static class ArrayArrayBuilderFactoryReflector + { + private static readonly MethodInfo s_buildInfo = typeof(ArrayData).Assembly.GetType("Apache.Arrow.ArrowArrayBuilderFactory") + .GetMethod("Build", BindingFlags.Static | BindingFlags.NonPublic); + + internal static IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>> InvokeBuild(IArrowType dataType) + { + return s_buildInfo.Invoke(null, new object[] { dataType }) as IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>; + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs new file mode 100644 index 000000000..6b3277ed5 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs @@ -0,0 +1,396 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowArrayConcatenatorTests + { + [Fact] + public void TestStandardCases() + { + foreach ((List<IArrowArray> testTargetArrayList, IArrowArray expectedArray) in GenerateTestData()) + { + IArrowArray actualArray = ArrowArrayConcatenatorReflector.InvokeConcatenate(testTargetArrayList); + ArrowReaderVerifier.CompareArrays(expectedArray, actualArray); + } + } + + [Fact] + public void TestNullOrEmpty() + { + Assert.Null(ArrowArrayConcatenatorReflector.InvokeConcatenate(null)); + Assert.Null(ArrowArrayConcatenatorReflector.InvokeConcatenate(new List<IArrowArray>())); + } + + [Fact] + public void TestSingleElement() + { + Int32Array array = new Int32Array.Builder().Append(1).Append(2).Build(); + IArrowArray actualArray = ArrowArrayConcatenatorReflector.InvokeConcatenate(new[] { array }); + ArrowReaderVerifier.CompareArrays(array, actualArray); + } + + private static IEnumerable<Tuple<List<IArrowArray>, IArrowArray>> GenerateTestData() + { + var targetTypes = new List<IArrowType>() { + BooleanType.Default, + Int8Type.Default, + Int16Type.Default, + Int32Type.Default, + Int64Type.Default, + UInt8Type.Default, + UInt16Type.Default, + UInt32Type.Default, + UInt64Type.Default, + FloatType.Default, + DoubleType.Default, + BinaryType.Default, + StringType.Default, + Date32Type.Default, + Date64Type.Default, + TimestampType.Default, + new Decimal128Type(14, 10), + new Decimal256Type(14,10), + new ListType(Int64Type.Default), + new StructType(new List<Field>{ + new Field.Builder().Name("Strings").DataType(StringType.Default).Nullable(true).Build(), + new Field.Builder().Name("Ints").DataType(Int32Type.Default).Nullable(true).Build() + }), + }; + + foreach (IArrowType type in targetTypes) + { + var creator = new TestDataGenerator(); + type.Accept(creator); + yield return Tuple.Create(creator.TestTargetArrayList, creator.ExpectedArray); + } + } + + private static class ArrowArrayConcatenatorReflector + { + private static readonly MethodInfo s_concatenateInfo = typeof(ArrayData).Assembly.GetType("Apache.Arrow.ArrowArrayConcatenator") + .GetMethod("Concatenate", BindingFlags.Static | BindingFlags.NonPublic); + + internal static IArrowArray InvokeConcatenate(IReadOnlyList<IArrowArray> arrowArrayList, MemoryAllocator allocator = default) + { + return s_concatenateInfo.Invoke(null, new object[] { arrowArrayList, allocator }) as IArrowArray; + } + } + + private class TestDataGenerator : + IArrowTypeVisitor<BooleanType>, + IArrowTypeVisitor<Int8Type>, + IArrowTypeVisitor<Int16Type>, + IArrowTypeVisitor<Int32Type>, + IArrowTypeVisitor<Int64Type>, + IArrowTypeVisitor<UInt8Type>, + IArrowTypeVisitor<UInt16Type>, + IArrowTypeVisitor<UInt32Type>, + IArrowTypeVisitor<UInt64Type>, + IArrowTypeVisitor<FloatType>, + IArrowTypeVisitor<DoubleType>, + IArrowTypeVisitor<BinaryType>, + IArrowTypeVisitor<StringType>, + IArrowTypeVisitor<Decimal128Type>, + IArrowTypeVisitor<Decimal256Type>, + IArrowTypeVisitor<Date32Type>, + IArrowTypeVisitor<Date64Type>, + IArrowTypeVisitor<TimestampType>, + IArrowTypeVisitor<ListType>, + IArrowTypeVisitor<StructType> + { + + private List<List<int?>> _baseData; + + private int _baseDataListCount; + + private int _baseDataTotalElementCount; + + public List<IArrowArray> TestTargetArrayList { get; } + public IArrowArray ExpectedArray { get; private set; } + + public TestDataGenerator() + { + _baseData = new List<List<int?>> { + new List<int?> { 1, 2, 3 }, + new List<int?> { 100, 101, null }, + new List<int?> { 11, null, 12 }, + }; + + _baseDataListCount = _baseData.Count; + _baseDataTotalElementCount = _baseData.Sum(_ => _.Count); + TestTargetArrayList = new List<IArrowArray>(_baseDataListCount); + } + + public void Visit(BooleanType type) => GenerateTestData<bool, BooleanArray, BooleanArray.Builder>(type, x => x % 2 == 0); + public void Visit(Int8Type type) => GenerateTestData<sbyte, Int8Array, Int8Array.Builder>(type, x => (sbyte)x); + public void Visit(Int16Type type) => GenerateTestData<short, Int16Array, Int16Array.Builder>(type, x => (short)x); + public void Visit(Int32Type type) => GenerateTestData<int, Int32Array, Int32Array.Builder>(type, x => x); + public void Visit(Int64Type type) => GenerateTestData<long, Int64Array, Int64Array.Builder>(type, x => x); + public void Visit(UInt8Type type) => GenerateTestData<byte, UInt8Array, UInt8Array.Builder>(type, x => (byte)x); + public void Visit(UInt16Type type) => GenerateTestData<ushort, UInt16Array, UInt16Array.Builder>(type, x => (ushort)x); + public void Visit(UInt32Type type) => GenerateTestData<uint, UInt32Array, UInt32Array.Builder>(type, x => (uint)x); + public void Visit(UInt64Type type) => GenerateTestData<ulong, UInt64Array, UInt64Array.Builder>(type, x => (ulong)x); + public void Visit(FloatType type) => GenerateTestData<float, FloatArray, FloatArray.Builder>(type, x => x); + public void Visit(DoubleType type) => GenerateTestData<double, DoubleArray, DoubleArray.Builder>(type, x => x); + public void Visit(Date32Type type) => GenerateTestData<DateTime, Date32Array, Date32Array.Builder>(type, x => DateTime.MinValue.AddDays(x)); + public void Visit(Date64Type type) => GenerateTestData<DateTime, Date64Array, Date64Array.Builder>(type, x => DateTime.MinValue.AddDays(x)); + + public void Visit(Decimal128Type type) + { + Decimal128Array.Builder resultBuilder = new Decimal128Array.Builder(type).Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + Decimal128Array.Builder builder = new Decimal128Array.Builder(type).Reserve(dataList.Count); + foreach (decimal? value in dataList) + { + if (value.HasValue) + { + builder.Append(value.Value); + resultBuilder.Append(value.Value); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + public void Visit(Decimal256Type type) + { + Decimal256Array.Builder resultBuilder = new Decimal256Array.Builder(type).Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + Decimal256Array.Builder builder = new Decimal256Array.Builder(type).Reserve(dataList.Count); + foreach (decimal? value in dataList) + { + if (value.HasValue) + { + builder.Append(value.Value); + resultBuilder.Append(value.Value); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + public void Visit(TimestampType type) + { + TimestampArray.Builder resultBuilder = new TimestampArray.Builder().Reserve(_baseDataTotalElementCount); + DateTimeOffset basis = DateTimeOffset.UtcNow; + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + TimestampArray.Builder builder = new TimestampArray.Builder().Reserve(dataList.Count); + foreach (int? value in dataList) + { + if (value.HasValue) + { + DateTimeOffset dateValue = basis.AddMilliseconds(value.Value); + builder.Append(dateValue); + resultBuilder.Append(dateValue); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + + public void Visit(BinaryType type) + { + BinaryArray.Builder resultBuilder = new BinaryArray.Builder().Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + BinaryArray.Builder builder = new BinaryArray.Builder().Reserve(dataList.Count); + + foreach (byte? value in dataList) + { + if (value.HasValue) + { + builder.Append(value.Value); + resultBuilder.Append(value.Value); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + public void Visit(StringType type) + { + StringArray.Builder resultBuilder = new StringArray.Builder().Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + StringArray.Builder builder = new StringArray.Builder().Reserve(dataList.Count); + + foreach (string value in dataList.Select(_ => _.ToString() ?? null)) + { + builder.Append(value); + resultBuilder.Append(value); + } + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + public void Visit(ListType type) + { + ListArray.Builder resultBuilder = new ListArray.Builder(type.ValueDataType).Reserve(_baseDataTotalElementCount); + //Todo : Support various types + Int64Array.Builder resultValueBuilder = (Int64Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + + ListArray.Builder builder = new ListArray.Builder(type.ValueField).Reserve(dataList.Count); + Int64Array.Builder valueBuilder = (Int64Array.Builder)builder.ValueBuilder.Reserve(dataList.Count); + + foreach (long? value in dataList) + { + if (value.HasValue) + { + builder.Append(); + resultBuilder.Append(); + + valueBuilder.Append(value.Value); + resultValueBuilder.Append(value.Value); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + + TestTargetArrayList.Add(builder.Build()); + } + + ExpectedArray = resultBuilder.Build(); + } + + public void Visit(StructType type) + { + // TODO: Make data from type fields. + + // The following can be improved with a Builder class for StructArray. + StringArray.Builder resultStringBuilder = new StringArray.Builder(); + Int32Array.Builder resultInt32Builder = new Int32Array.Builder(); + ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build(); + + for (int i = 0; i < 3; i++) + { + resultStringBuilder.Append("joe").AppendNull().AppendNull().Append("mark"); + resultInt32Builder.Append(1).Append(2).AppendNull().Append(4); + StringArray stringArray = new StringArray.Builder().Append("joe").AppendNull().AppendNull().Append("mark").Build(); + Int32Array intArray = new Int32Array.Builder().Append(1).Append(2).AppendNull().Append(4).Build(); + List<Array> arrays = new List<Array> + { + stringArray, + intArray + }; + + TestTargetArrayList.Add(new StructArray(type, 3, arrays, nullBitmapBuffer, 1)); + } + + StringArray resultStringArray = resultStringBuilder.Build(); + Int32Array resultInt32Array = resultInt32Builder.Build(); + + ExpectedArray = new StructArray(type, 3, new List<Array> { resultStringArray, resultInt32Array }, nullBitmapBuffer, 1); + } + + + public void Visit(IArrowType type) + { + throw new NotImplementedException(); + } + + private void GenerateTestData<T, TArray, TArrayBuilder>(IArrowType type, Func<int, T> generator) + where TArrayBuilder : IArrowArrayBuilder<T, TArray, TArrayBuilder> + where TArray : IArrowArray + { + var resultBuilder = (IArrowArrayBuilder<T, TArray, TArrayBuilder>)ArrayArrayBuilderFactoryReflector.InvokeBuild(type); + resultBuilder.Reserve(_baseDataTotalElementCount); + + for (int i = 0; i < _baseDataListCount; i++) + { + List<int?> dataList = _baseData[i]; + var builder = (IArrowArrayBuilder<T, TArray, TArrayBuilder>)ArrayArrayBuilderFactoryReflector.InvokeBuild(type); + builder.Reserve(dataList.Count); + + foreach (int? value in dataList) + { + if (value.HasValue) + { + builder.Append(generator(value.Value)); + resultBuilder.Append(generator(value.Value)); + } + else + { + builder.AppendNull(); + resultBuilder.AppendNull(); + } + } + TestTargetArrayList.Add(builder.Build(default)); + } + + ExpectedArray = resultBuilder.Build(default); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs new file mode 100644 index 000000000..18d405613 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowArrayTests.cs @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowArrayTests + { + + [Fact] + public void ThrowsWhenGetValueIndexOutOfBounds() + { + var array = new Int64Array.Builder().Append(1).Append(2).Build(); + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValue(-1)); + Assert.Equal(1, array.GetValue(0)); + Assert.Equal(2, array.GetValue(1)); + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValue(2)); + } + + [Fact] + public void ThrowsWhenGetValueAndOffsetIndexOutOfBounds() + { + var array = new BinaryArray.Builder().Append(1).Append(2).Build(); + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValueLength(-1)); + Assert.Equal(1, array.GetValueLength(0)); + Assert.Equal(1, array.GetValueLength(1)); + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValueLength(2)); + +#pragma warning disable 618 + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValueOffset(-1)); + Assert.Equal(0, array.GetValueOffset(0)); + Assert.Equal(1, array.GetValueOffset(1)); + Assert.Equal(2, array.GetValueOffset(2)); + Assert.Throws<ArgumentOutOfRangeException>(() => array.GetValueOffset(3)); +#pragma warning restore 618 + + Assert.Throws<IndexOutOfRangeException>(() => array.ValueOffsets[-1]); + Assert.Equal(0, array.ValueOffsets[0]); + Assert.Equal(1, array.ValueOffsets[1]); + Assert.Equal(2, array.ValueOffsets[2]); + Assert.Throws<IndexOutOfRangeException>(() => array.ValueOffsets[3]); + + } + + [Fact] + public void IsValidValue() + { + const int totalValueCount = 8; + const byte nullBitmap = 0b_11110011; + + var nullBitmapBuffer = new ArrowBuffer.Builder<byte>().Append(nullBitmap).Build(); + var valueBuffer = new ArrowBuffer.Builder<long>().Append(0).Append(1).Append(4).Append(5).Append(6).Append(7).Append(8).Build(); + + //Check all offset and length + for (var offset = 0; offset < totalValueCount; offset++) + { + var nullCount = totalValueCount - offset - BitUtility.CountBits(nullBitmapBuffer.Span, offset); + for (var length = 1; length + offset < totalValueCount; length++) + { + TestIsValid(valueBuffer, nullBitmapBuffer, length, nullCount, offset); + } + } + + void TestIsValid(ArrowBuffer valueBuf, ArrowBuffer nullBitmapBuf, int length, int nullCount, int offset) + { + var array = new Int64Array(valueBuf, nullBitmapBuf, length, nullCount, offset); + for (var i = 0; i < length; i++) + { + if (BitUtility.GetBit(nullBitmap, i + offset)) + { + Assert.True(array.IsValid(i)); + } + else + { + Assert.False(array.IsValid(i)); + } + } + } + } + + [Fact] + public void SliceArray() + { + TestSlice<Int32Array, Int32Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<Int8Array, Int8Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<Int16Array, Int16Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<Int64Array, Int64Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<UInt8Array, UInt8Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<UInt16Array, UInt16Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<UInt32Array, UInt32Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<UInt64Array, UInt64Array.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<FloatArray, FloatArray.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<DoubleArray, DoubleArray.Builder>(x => x.Append(10).Append(20).Append(30)); + TestSlice<Date32Array, Date32Array.Builder>(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).Append(new DateTime(2019, 1, 3))); + TestSlice<Date64Array, Date64Array.Builder>(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).Append(new DateTime(2019, 1, 3))); + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").Append("20").Append("30")); + } + + [Fact] + public void SlicePrimitiveArrayWithNulls() + { + TestSlice<Int32Array, Int32Array.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<Int8Array, Int8Array.Builder>(x => x.Append(10).AppendNull().Append(20).AppendNull().Append(30)); + TestSlice<Int16Array, Int16Array.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<Int64Array, Int64Array.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<UInt8Array, UInt8Array.Builder>(x => x.Append(10).Append(20).Append(30).AppendNull()); + TestSlice<UInt16Array, UInt16Array.Builder>(x => x.Append(10).Append(20).AppendNull().AppendNull().Append(30)); + TestSlice<UInt32Array, UInt32Array.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<UInt64Array, UInt64Array.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<FloatArray, FloatArray.Builder>(x => x.AppendNull().Append(10).Append(20).AppendNull().Append(30)); + TestSlice<DoubleArray, DoubleArray.Builder>(x => x.Append(10).Append(20).AppendNull().Append(30)); + TestSlice<Date32Array, Date32Array.Builder>(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).AppendNull().Append(new DateTime(2019, 1, 3))); + TestSlice<Date64Array, Date64Array.Builder>(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).AppendNull().Append(new DateTime(2019, 1, 3))); + } + + [Fact] + public void SliceBooleanArray() + { + TestSlice<BooleanArray, BooleanArray.Builder>(x => x.Append(true).Append(false).Append(true)); + TestSlice<BooleanArray, BooleanArray.Builder>(x => x.Append(true).Append(false).AppendNull().Append(true)); + } + + [Fact] + public void SliceStringArrayWithNullsAndEmptyStrings() + { + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").AppendNull().Append("30")); + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").Append(string.Empty).Append("30")); + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").Append(string.Empty).AppendNull().Append("30")); + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").AppendNull().Append(string.Empty).Append("30")); + TestSlice<StringArray, StringArray.Builder>(x => x.Append("10").AppendNull().Append(string.Empty).AppendNull().Append("30")); + } + + private static void TestSlice<TArray, TArrayBuilder>(Action<TArrayBuilder> action) + where TArray : IArrowArray + where TArrayBuilder : IArrowArrayBuilder<TArray>, new() + { + var builder = new TArrayBuilder(); + action(builder); + var baseArray = builder.Build(default) as Array; + Assert.NotNull(baseArray); + var totalLength = baseArray.Length; + var validator = new ArraySliceValidator(baseArray); + + //Check all offset and length + for (var offset = 0; offset < totalLength; offset++) + { + for (var length = 1; length + offset <= totalLength; length++) + { + var targetArray = baseArray.Slice(offset, length); + targetArray.Accept(validator); + } + } + } + + private class ArraySliceValidator : + IArrowArrayVisitor<Int8Array>, + IArrowArrayVisitor<Int16Array>, + IArrowArrayVisitor<Int32Array>, + IArrowArrayVisitor<Int64Array>, + IArrowArrayVisitor<UInt8Array>, + IArrowArrayVisitor<UInt16Array>, + IArrowArrayVisitor<UInt32Array>, + IArrowArrayVisitor<UInt64Array>, + IArrowArrayVisitor<Date32Array>, + IArrowArrayVisitor<Date64Array>, + IArrowArrayVisitor<FloatArray>, + IArrowArrayVisitor<DoubleArray>, + IArrowArrayVisitor<BooleanArray>, + IArrowArrayVisitor<StringArray> + { + private readonly IArrowArray _baseArray; + + public ArraySliceValidator(IArrowArray baseArray) + { + _baseArray = baseArray; + } + + public void Visit(Int8Array array) => ValidateArrays(array); + public void Visit(Int16Array array) => ValidateArrays(array); + public void Visit(Int32Array array) => ValidateArrays(array); + public void Visit(Int64Array array) => ValidateArrays(array); + public void Visit(UInt8Array array) => ValidateArrays(array); + public void Visit(UInt16Array array) => ValidateArrays(array); + public void Visit(UInt32Array array) => ValidateArrays(array); + public void Visit(UInt64Array array) => ValidateArrays(array); + + public void Visit(Date32Array array) + { + ValidateArrays(array); + Assert.IsAssignableFrom<Date32Array>(_baseArray); + var baseArray = (Date32Array)_baseArray; + + Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0)); + } + + public void Visit(Date64Array array) + { + ValidateArrays(array); + Assert.IsAssignableFrom<Date64Array>(_baseArray); + var baseArray = (Date64Array)_baseArray; + + Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0)); + } + + public void Visit(FloatArray array) => ValidateArrays(array); + public void Visit(DoubleArray array) => ValidateArrays(array); + public void Visit(StringArray array) => ValidateArrays(array); + public void Visit(BooleanArray array) => ValidateArrays(array); + + public void Visit(IArrowArray array) => throw new NotImplementedException(); + + private void ValidateArrays<T>(PrimitiveArray<T> slicedArray) + where T : struct, IEquatable<T> + { + Assert.IsAssignableFrom<PrimitiveArray<T>>(_baseArray); + var baseArray = (PrimitiveArray<T>)_baseArray; + + Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); + Assert.True( + baseArray.ValueBuffer.Span.CastTo<T>().Slice(slicedArray.Offset, slicedArray.Length) + .SequenceEqual(slicedArray.Values)); + + Assert.Equal(baseArray.GetValue(slicedArray.Offset), slicedArray.GetValue(0)); + } + + private void ValidateArrays(BooleanArray slicedArray) + { + Assert.IsAssignableFrom<BooleanArray>(_baseArray); + var baseArray = (BooleanArray)_baseArray; + + Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); + Assert.True(baseArray.Values.SequenceEqual(slicedArray.Values)); + + Assert.True( + baseArray.ValueBuffer.Span.Slice(0, (int) Math.Ceiling(slicedArray.Length / 8.0)) + .SequenceEqual(slicedArray.Values)); + + Assert.Equal(baseArray.GetValue(slicedArray.Offset), slicedArray.GetValue(0)); + +#pragma warning disable CS0618 + Assert.Equal(baseArray.GetBoolean(slicedArray.Offset), slicedArray.GetBoolean(0)); +#pragma warning restore CS0618 + } + + private void ValidateArrays(BinaryArray slicedArray) + { + Assert.IsAssignableFrom<BinaryArray>(_baseArray); + var baseArray = (BinaryArray)_baseArray; + + Assert.True(baseArray.Values.SequenceEqual(slicedArray.Values)); + Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); + Assert.True( + baseArray.ValueOffsetsBuffer.Span.CastTo<int>().Slice(slicedArray.Offset, slicedArray.Length + 1) + .SequenceEqual(slicedArray.ValueOffsets)); + + Assert.True(baseArray.GetBytes(slicedArray.Offset).SequenceEqual(slicedArray.GetBytes(0))); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBitmapBuilderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBitmapBuilderTests.cs new file mode 100644 index 000000000..3a9734e84 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBitmapBuilderTests.cs @@ -0,0 +1,493 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow.Tests +{ + using System; + using System.Linq; + using Xunit; + + /// <summary> + /// The <see cref="ArrowBufferBitmapBuilderTests"/> class provides unit tests for the + /// <see cref="ArrowBuffer.BitmapBuilder"/> class. + /// </summary> + public class ArrowBufferBitmapBuilderTests + { + public class Append + { + [Theory] + [InlineData(new bool[] {}, false, 1, 0, 1)] + [InlineData(new bool[] {}, true, 1, 1, 0)] + [InlineData(new[] { true, false }, true, 3, 2, 1)] + [InlineData(new[] { true, false }, false, 3, 1, 2)] + public void IncreasesLength( + bool[] initialContents, + bool valueToAppend, + int expectedLength, + int expectedSetBitCount, + int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(initialContents); + + // Act + var actualReturnValue = builder.Append(valueToAppend); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + Assert.True(builder.Capacity >= expectedLength); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + } + + [Theory] + [InlineData(new bool[] {}, false)] + [InlineData(new bool[] {}, true)] + [InlineData(new[] { true, false }, true)] + [InlineData(new[] { true, false }, false)] + public void AfterClearIncreasesLength(bool[] initialContentsToClear, bool valueToAppend) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(initialContentsToClear); + builder.Clear(); + + // Act + var actualReturnValue = builder.Append(valueToAppend); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(1, builder.Length); + Assert.True(builder.Capacity >= 1); + Assert.Equal(valueToAppend ? 1 : 0, builder.SetBitCount); + Assert.Equal(valueToAppend ? 0 : 1, builder.UnsetBitCount); + } + + [Fact] + public void IncreasesCapacityWhenRequired() + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + int initialCapacity = builder.Capacity; + builder.AppendRange(Enumerable.Repeat(true, initialCapacity)); // Fill to capacity. + + // Act + var actualReturnValue = builder.Append(true); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(initialCapacity + 1, builder.Length); + Assert.True(builder.Capacity >= initialCapacity + 1); + } + } + + public class AppendRange + { + [Theory] + [InlineData(new bool[] {}, new bool[] {}, 0, 0, 0)] + [InlineData(new bool[] {}, new[] { true, false }, 2, 1, 1)] + [InlineData(new[] { true, false }, new bool[] {}, 2, 1, 1)] + [InlineData(new[] { true, false }, new[] { true, false }, 4, 2, 2)] + public void IncreasesLength( + bool[] initialContents, + bool[] toAppend, + int expectedLength, + int expectedSetBitCount, + int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(initialContents); + + // Act + var actualReturnValue = builder.AppendRange(toAppend); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + Assert.True(builder.Capacity >= expectedLength); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + } + } + + public class Build + { + [Theory] + [InlineData(new bool[] { }, new byte[] { })] + [InlineData(new[] { true, false, true, false }, new byte[] { 0b00000101 })] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false, true, false }, + new byte[] { 0b01010101, 0b00000101 })] + public void AppendedRangeBitPacks(bool[] contents, byte[] expectedBytes) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(contents); + + // Act + var buf = builder.Build(); + + // Assert + AssertBuffer(expectedBytes, buf); + } + } + + public class Clear + { + [Theory] + [InlineData(10)] + [InlineData(100)] + public void ClearingSetsBitCountToZero(int numBitsBeforeClear) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + var data = Enumerable.Repeat(true, numBitsBeforeClear).Select(x => x).ToArray(); + builder.AppendRange(data); + + // Act + var actualReturnValue = builder.Clear(); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(0, builder.Length); + } + } + + public class Resize + { + [Theory] + [InlineData(new bool[] {}, 256, 0, 256)] + [InlineData(new[] { true, true, true, true}, 256, 4, 252)] + [InlineData(new[] { false, false, false, false}, 256, 0, 256)] + [InlineData(new[] { true, true, true, true}, 2, 2, 0)] + [InlineData(new[] { true, true, true, true}, 0, 0, 0)] + public void LengthHasExpectedValueAfterResize( + bool[] bits, int newSize, int expectedSetBitCount, int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(bits); + + // Act + var actualReturnValue = builder.Resize(newSize); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.True(builder.Capacity >= newSize); + Assert.Equal(newSize, builder.Length); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + } + + [Fact] + public void NegativeLengthThrows() + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.Append(false); + builder.Append(true); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Resize(-1)); + } + } + + public class Reserve + { + [Theory] + [InlineData(0, 0, 0)] + [InlineData(0, 0, 8)] + [InlineData(8, 8, 8)] + [InlineData(8, 8, 16)] + public void CapacityIncreased(int initialCapacity, int numBitsToAppend, int additionalCapacity) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(initialCapacity); + builder.AppendRange(Enumerable.Repeat(true, numBitsToAppend)); + + // Act + var actualReturnValue = builder.Reserve(additionalCapacity); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.True(builder.Capacity >= numBitsToAppend + additionalCapacity); + } + + [Fact] + public void NegtativeCapacityThrows() + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Reserve(-1)); + } + } + + public class Set + { + [Theory] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 2, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 3, + new byte[] { 0b01011101, 0b00000001 }, + 6, 4)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 8, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 9, + new byte[] { 0b01010101, 0b00000011 }, + 6, 4)] + public void OverloadWithNoValueParameterSetsAsExpected( + bool[] bits, int indexToSet, byte[] expectedBytes, + int expectedSetBitCount, int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(bits); + + // Act + var actualReturnValue = builder.Set(indexToSet); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + var buf = builder.Build(); + AssertBuffer(expectedBytes, buf); + } + + [Theory] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 2, true, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 2, false, + new byte[] { 0b01010001, 0b00000001 }, + 4, 6)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 3, true, + new byte[] { 0b01011101, 0b00000001 }, + 6, 4)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 3, false, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 8, true, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 8, false, + new byte[] { 0b01010101, 0b00000000 }, + 4, 6)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 9, true, + new byte[] { 0b01010101, 0b00000011 }, + 6, 4)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 9, false, + new byte[] { 0b01010101, 0b00000001 }, + 5, 5)] + public void OverloadWithValueParameterSetsAsExpected( + bool[] bits, int indexToSet, bool valueToSet, byte[] expectedBytes, + int expectedSetBitCount, int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(bits); + + // Act + var actualReturnValue = builder.Set(indexToSet, valueToSet); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + var buf = builder.Build(); + AssertBuffer(expectedBytes, buf); + } + + [Theory] + [InlineData(0, -1)] + [InlineData(0, 0)] + [InlineData(1, 1)] + [InlineData(10, 10)] + [InlineData(10, 11)] + public void BadIndexThrows(int numBitsToAppend, int indexToSet) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + var bits = Enumerable.Repeat(true, numBitsToAppend); + builder.AppendRange(bits); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Set(indexToSet)); + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Set(indexToSet, true)); + } + } + + public class Swap + { + [Theory] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 0, 2, + new byte[] { 0b01010101, 0b00000001 })] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 0, 3, + new byte[] { 0b01011100, 0b00000001 })] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 4, 8, + new byte[] { 0b01010101, 0b00000001 })] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 4, 9, + new byte[] { 0b01000101, 0b00000011 })] + public void SwapsAsExpected(bool[] bits, int firstIndex, int secondIndex, byte[] expectedBytes) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(bits); + + // Act + var actualReturnValue = builder.Swap(firstIndex, secondIndex); + + // Assert + Assert.Equal(builder, actualReturnValue); + var buf = builder.Build(); + AssertBuffer(expectedBytes, buf); + } + + [Theory] + [InlineData(0, -1, 0)] + [InlineData(0, 0, -1)] + [InlineData(0, 0, 0)] + [InlineData(1, 0, 1)] + [InlineData(1, 1, 0)] + [InlineData(1, 0, -1)] + [InlineData(1, -1, 0)] + [InlineData(1, 1, 1)] + [InlineData(10, 10, 0)] + [InlineData(10, 0, 10)] + [InlineData(10, 10, 10)] + [InlineData(10, 11, 0)] + [InlineData(10, 0, 11)] + [InlineData(10, 11, 11)] + public void BadIndicesThrows(int numBitsToAppend, int firstIndex, int secondIndex) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + var bits = Enumerable.Repeat(true, numBitsToAppend); + builder.AppendRange(bits); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Swap(firstIndex, secondIndex)); + } + } + + public class Toggle + { + [Theory] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 2, + new byte[] { 0b01010001, 0b00000001 }, + 4, 6)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 3, + new byte[] { 0b01011101, 0b00000001 }, + 6, 4)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 8, + new byte[] { 0b01010101, 0b00000000 }, + 4, 6)] + [InlineData( + new[] { true, false, true, false, true, false, true, false, true, false}, + 9, + new byte[] { 0b01010101, 0b00000011 }, + 6, 4)] + public void TogglesAsExpected( + bool[] bits, int indexToToggle, byte[] expectedBytes, + int expectedSetBitCount, int expectedUnsetBitCount) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + builder.AppendRange(bits); + + // Act + var actualReturnValue = builder.Toggle(indexToToggle); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedSetBitCount, builder.SetBitCount); + Assert.Equal(expectedUnsetBitCount, builder.UnsetBitCount); + var buf = builder.Build(); + AssertBuffer(expectedBytes, buf); + } + + [Theory] + [InlineData(0, -1)] + [InlineData(0, 0)] + [InlineData(1, 1)] + [InlineData(10, 10)] + [InlineData(10, 11)] + public void BadIndexThrows(int numBitsToAppend, int indexToToggle) + { + // Arrange + var builder = new ArrowBuffer.BitmapBuilder(); + var bits = Enumerable.Repeat(true, numBitsToAppend); + builder.AppendRange(bits); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Toggle(indexToToggle)); + } + } + + private static void AssertBuffer(byte[] expectedBytes, ArrowBuffer buf) + { + Assert.True(buf.Length >= expectedBytes.Length); + for (int i = 0; i < expectedBytes.Length; i++) + { + Assert.Equal(expectedBytes[i], buf.Span[i]); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBuilderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBuilderTests.cs new file mode 100644 index 000000000..495fc2e06 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferBuilderTests.cs @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowBufferBuilderTests + { + [Fact] + public void ThrowsWhenIndexOutOfBounds() + { + Assert.Throws<IndexOutOfRangeException>(() => + { + var builder = new ArrowBuffer.Builder<int>(); + builder.Span[100] = 100; + }); + } + + public class Append + { + [Fact] + public void DoesNotThrowWithNullParameters() + { + var builder = new ArrowBuffer.Builder<int>(); + + builder.AppendRange(null); + } + + [Fact] + public void CapacityOnlyGrowsWhenLengthWillExceedCapacity() + { + var builder = new ArrowBuffer.Builder<int>(1); + var capacity = builder.Capacity; + + builder.Append(1); + + Assert.Equal(capacity, builder.Capacity); + } + + [Fact] + public void CapacityGrowsAfterAppendWhenLengthExceedsCapacity() + { + var builder = new ArrowBuffer.Builder<int>(1); + var capacity = builder.Capacity; + + builder.Append(1); + builder.Append(2); + + Assert.True(builder.Capacity > capacity); + } + + [Fact] + public void CapacityGrowsAfterAppendSpan() + { + var builder = new ArrowBuffer.Builder<int>(1); + var capacity = builder.Capacity; + var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); + + builder.Append(data); + + Assert.True(builder.Capacity > capacity); + } + + [Fact] + public void LengthIncrementsAfterAppend() + { + var builder = new ArrowBuffer.Builder<int>(1); + var length = builder.Length; + + builder.Append(1); + + Assert.Equal(length + 1, builder.Length); + } + + [Fact] + public void LengthGrowsBySpanLength() + { + var builder = new ArrowBuffer.Builder<int>(1); + var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); + + builder.Append(data); + + Assert.Equal(10, builder.Length); + } + + [Fact] + public void BufferHasExpectedValues() + { + var builder = new ArrowBuffer.Builder<int>(1); + + builder.Append(10); + builder.Append(20); + + var buffer = builder.Build(); + var span = buffer.Span.CastTo<int>(); + + Assert.Equal(10, span[0]); + Assert.Equal(20, span[1]); + Assert.Equal(0, span[2]); + } + } + + public class AppendRange + { + [Fact] + public void CapacityGrowsAfterAppendEnumerable() + { + var builder = new ArrowBuffer.Builder<int>(1); + var capacity = builder.Capacity; + var data = Enumerable.Range(0, 10).Select(x => x); + + builder.AppendRange(data); + + Assert.True(builder.Capacity > capacity); + } + + [Fact] + public void LengthGrowsByEnumerableCount() + { + var builder = new ArrowBuffer.Builder<int>(1); + var length = builder.Length; + var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); + var count = data.Length; + + builder.AppendRange(data); + + Assert.Equal(length + count, builder.Length); + } + + [Fact] + public void BufferHasExpectedValues() + { + var builder = new ArrowBuffer.Builder<int>(1); + var data = Enumerable.Range(0, 10).Select(x => x).ToArray(); + + builder.AppendRange(data); + + var buffer = builder.Build(); + var span = buffer.Span.CastTo<int>(); + + for (var i = 0; i < 10; i++) + { + Assert.Equal(i, span[i]); + } + } + } + + public class Clear + { + [Theory] + [InlineData(10)] + [InlineData(100)] + public void SetsAllValuesToDefault(int sizeBeforeClear) + { + var builder = new ArrowBuffer.Builder<int>(1); + var data = Enumerable.Range(0, sizeBeforeClear).Select(x => x).ToArray(); + + builder.AppendRange(data); + builder.Clear(); + builder.Append(0); + + var buffer = builder.Build(); + // No matter the sizeBeforeClear, we only appended a single 0, + // so the buffer length should be the smallest possible. + Assert.Equal(64, buffer.Length); + + // check all 16 int elements are default + var zeros = Enumerable.Range(0, 16).Select(x => 0).ToArray(); + var values = buffer.Span.CastTo<int>().Slice(0, 16).ToArray(); + + Assert.True(zeros.SequenceEqual(values)); + } + } + + public class Resize + { + [Fact] + public void LengthHasExpectedValueAfterResize() + { + var builder = new ArrowBuffer.Builder<int>(); + builder.Resize(8); + + Assert.True(builder.Capacity >= 8); + Assert.Equal(8, builder.Length); + } + + [Fact] + public void NegativeLengthThrows() + { + // Arrange + var builder = new ArrowBuffer.Builder<int>(); + builder.Append(10); + builder.Append(20); + + // Act/Assert + Assert.Throws<ArgumentOutOfRangeException>(() => builder.Resize(-1)); + } + } + + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferTests.cs new file mode 100644 index 000000000..e6fa5256a --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowBufferTests.cs @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Tests.Fixtures; +using System; +using System.Runtime.InteropServices; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowBufferTests + { + public class Allocate : + IClassFixture<DefaultMemoryAllocatorFixture> + { + private readonly DefaultMemoryAllocatorFixture _memoryPoolFixture; + + public Allocate(DefaultMemoryAllocatorFixture memoryPoolFixture) + { + _memoryPoolFixture = memoryPoolFixture; + } + + /// <summary> + /// Ensure Arrow buffers are allocated in multiples of 64 bytes. + /// </summary> + /// <param name="size">number of bytes to allocate</param> + /// <param name="expectedCapacity">expected buffer capacity after allocation</param> + [Theory] + [InlineData(0, 0)] + [InlineData(1, 64)] + [InlineData(8, 64)] + [InlineData(9, 64)] + [InlineData(65, 128)] + public void AllocatesWithExpectedPadding(int size, int expectedCapacity) + { + var builder = new ArrowBuffer.Builder<byte>(size); + for (int i = 0; i < size; i++) + { + builder.Append(0); + } + var buffer = builder.Build(); + + Assert.Equal(expectedCapacity, buffer.Length); + } + + /// <summary> + /// Ensure allocated buffers are aligned to multiples of 64. + /// </summary> + [Theory] + [InlineData(1)] + [InlineData(8)] + [InlineData(64)] + [InlineData(128)] + public unsafe void AllocatesAlignedToMultipleOf64(int size) + { + var builder = new ArrowBuffer.Builder<byte>(size); + for (int i = 0; i < size; i++) + { + builder.Append(0); + } + var buffer = builder.Build(); + + fixed (byte* ptr = &buffer.Span.GetPinnableReference()) + { + Assert.True(new IntPtr(ptr).ToInt64() % 64 == 0); + } + } + + /// <summary> + /// Ensure padding in arrow buffers is initialized with zeroes. + /// </summary> + [Fact] + public void HasZeroPadding() + { + var buffer = new ArrowBuffer.Builder<byte>(10).Append(0).Build(); + + foreach (var b in buffer.Span) + { + Assert.Equal(0, b); + } + } + + } + + [Fact] + public void TestExternalMemoryWrappedAsArrowBuffer() + { + Memory<byte> memory = new byte[sizeof(int) * 3]; + Span<byte> spanOfBytes = memory.Span; + var span = spanOfBytes.CastTo<int>(); + span[0] = 0; + span[1] = 1; + span[2] = 2; + + ArrowBuffer buffer = new ArrowBuffer(memory); + Assert.Equal(2, buffer.Span.CastTo<int>()[2]); + + span[2] = 10; + Assert.Equal(10, buffer.Span.CastTo<int>()[2]); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileReaderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileReaderTests.cs new file mode 100644 index 000000000..f0876c8b1 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileReaderTests.cs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Memory; +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowFileReaderTests + { + [Fact] + public void Ctor_LeaveOpenDefault_StreamClosedOnDispose() + { + var stream = new MemoryStream(); + new ArrowFileReader(stream).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenFalse_StreamClosedOnDispose() + { + var stream = new MemoryStream(); + new ArrowFileReader(stream, leaveOpen: false).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenTrue_StreamValidOnDispose() + { + var stream = new MemoryStream(); + new ArrowFileReader(stream, leaveOpen: true).Dispose(); + Assert.Equal(0, stream.Position); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Ctor_MemoryPool_AllocatesFromPool(bool shouldLeaveOpen) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + + using (MemoryStream stream = new MemoryStream()) + { + ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + stream.Position = 0; + + var memoryPool = new TestMemoryAllocator(); + ArrowFileReader reader = new ArrowFileReader(stream, memoryPool, leaveOpen: shouldLeaveOpen); + reader.ReadNextRecordBatch(); + + Assert.Equal(1, memoryPool.Statistics.Allocations); + Assert.True(memoryPool.Statistics.BytesAllocated > 0); + + reader.Dispose(); + + if (shouldLeaveOpen) + { + Assert.True(stream.Position > 0); + } + else + { + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + } + } + + [Fact] + public async Task TestReadNextRecordBatch() + { + await TestReadRecordBatchHelper((reader, originalBatch) => + { + ArrowReaderVerifier.VerifyReader(reader, originalBatch); + return Task.CompletedTask; + }); + } + + [Fact] + public async Task TestReadNextRecordBatchAsync() + { + await TestReadRecordBatchHelper(ArrowReaderVerifier.VerifyReaderAsync); + } + + [Fact] + public async Task TestReadRecordBatchAsync() + { + await TestReadRecordBatchHelper(async (reader, originalBatch) => + { + RecordBatch readBatch = await reader.ReadRecordBatchAsync(0); + ArrowReaderVerifier.CompareBatches(originalBatch, readBatch); + + // You should be able to read the same record batch again + RecordBatch readBatch2 = await reader.ReadRecordBatchAsync(0); + ArrowReaderVerifier.CompareBatches(originalBatch, readBatch2); + }); + } + + private static async Task TestReadRecordBatchHelper( + Func<ArrowFileReader, RecordBatch, Task> verificationFunc) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + + using (MemoryStream stream = new MemoryStream()) + { + ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + stream.Position = 0; + + ArrowFileReader reader = new ArrowFileReader(stream); + await verificationFunc(reader, originalBatch); + } + } + + [Fact] + public async Task TestReadMultipleRecordBatchAsync() + { + RecordBatch originalBatch1 = TestData.CreateSampleRecordBatch(length: 100); + RecordBatch originalBatch2 = TestData.CreateSampleRecordBatch(length: 50); + + using (MemoryStream stream = new MemoryStream()) + { + ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch1.Schema); + await writer.WriteRecordBatchAsync(originalBatch1); + await writer.WriteRecordBatchAsync(originalBatch2); + await writer.WriteEndAsync(); + stream.Position = 0; + + ArrowFileReader reader = new ArrowFileReader(stream); + RecordBatch readBatch1 = await reader.ReadRecordBatchAsync(0); + ArrowReaderVerifier.CompareBatches(originalBatch1, readBatch1); + + RecordBatch readBatch2 = await reader.ReadRecordBatchAsync(1); + ArrowReaderVerifier.CompareBatches(originalBatch2, readBatch2); + + // now read the first again, for random access + RecordBatch readBatch3 = await reader.ReadRecordBatchAsync(0); + ArrowReaderVerifier.CompareBatches(originalBatch1, readBatch3); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs new file mode 100644 index 000000000..a310a3609 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowFileWriterTests.cs @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowFileWriterTests + { + [Fact] + public void Ctor_LeaveOpenDefault_StreamClosedOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowFileWriter(stream, originalBatch.Schema).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenFalse_StreamClosedOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowFileWriter(stream, originalBatch.Schema, leaveOpen: false).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenTrue_StreamValidOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowFileWriter(stream, originalBatch.Schema, leaveOpen: true).Dispose(); + Assert.Equal(0, stream.Position); + } + + /// <summary> + /// Tests that writing an arrow file will always align the Block lengths + /// to 8 bytes. There are asserts in both the reader and writer which will fail + /// if this isn't the case. + /// </summary> + /// <returns></returns> + [Fact] + public async Task WritesFooterAlignedMulitpleOf8() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + + var stream = new MemoryStream(); + var writer = new ArrowFileWriter( + stream, + originalBatch.Schema, + leaveOpen: true, + // use WriteLegacyIpcFormat, which only uses a 4-byte length prefix + // which causes the length prefix to not be 8-byte aligned by default + new IpcOptions() { WriteLegacyIpcFormat = true }); + + writer.WriteRecordBatch(originalBatch); + writer.WriteEnd(); + + stream.Position = 0; + + await ValidateRecordBatchFile(stream, originalBatch); + } + + /// <summary> + /// Tests that writing an arrow file will always align the Block lengths + /// to 8 bytes. There are asserts in both the reader and writer which will fail + /// if this isn't the case. + /// </summary> + /// <returns></returns> + [Fact] + public async Task WritesFooterAlignedMulitpleOf8Async() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + + var stream = new MemoryStream(); + var writer = new ArrowFileWriter( + stream, + originalBatch.Schema, + leaveOpen: true, + // use WriteLegacyIpcFormat, which only uses a 4-byte length prefix + // which causes the length prefix to not be 8-byte aligned by default + new IpcOptions() { WriteLegacyIpcFormat = true }); + + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + + stream.Position = 0; + + await ValidateRecordBatchFile(stream, originalBatch); + } + + private async Task ValidateRecordBatchFile(Stream stream, RecordBatch recordBatch) + { + var reader = new ArrowFileReader(stream); + int count = await reader.RecordBatchCountAsync(); + Assert.Equal(1, count); + RecordBatch readBatch = await reader.ReadRecordBatchAsync(0); + ArrowReaderVerifier.CompareBatches(recordBatch, readBatch); + } + + /// <summary> + /// Tests that writing an arrow file with no RecordBatches produces the correct + /// file. + /// </summary> + [Fact] + public async Task WritesEmptyFile() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1); + + var stream = new MemoryStream(); + var writer = new ArrowFileWriter(stream, originalBatch.Schema); + + writer.WriteStart(); + writer.WriteEnd(); + + stream.Position = 0; + + var reader = new ArrowFileReader(stream); + int count = await reader.RecordBatchCountAsync(); + Assert.Equal(0, count); + RecordBatch readBatch = reader.ReadNextRecordBatch(); + Assert.Null(readBatch); + SchemaComparer.Compare(originalBatch.Schema, reader.Schema); + } + + /// <summary> + /// Tests that writing an arrow file with no RecordBatches produces the correct + /// file when using WriteStartAsync and WriteEndAsync. + /// </summary> + [Fact] + public async Task WritesEmptyFileAsync() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1); + + var stream = new MemoryStream(); + var writer = new ArrowFileWriter(stream, originalBatch.Schema); + + await writer.WriteStartAsync(); + await writer.WriteEndAsync(); + + stream.Position = 0; + + var reader = new ArrowFileReader(stream); + int count = await reader.RecordBatchCountAsync(); + Assert.Equal(0, count); + RecordBatch readBatch = reader.ReadNextRecordBatch(); + Assert.Null(readBatch); + SchemaComparer.Compare(originalBatch.Schema, reader.Schema); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs new file mode 100644 index 000000000..a2c9a9ef7 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs @@ -0,0 +1,302 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using System; +using System.Linq; +using System.Threading.Tasks; +using Apache.Arrow.Arrays; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public static class ArrowReaderVerifier + { + public static void VerifyReader(ArrowStreamReader reader, RecordBatch originalBatch) + { + RecordBatch readBatch = reader.ReadNextRecordBatch(); + CompareBatches(originalBatch, readBatch); + + // There should only be one batch - calling ReadNextRecordBatch again should return null. + Assert.Null(reader.ReadNextRecordBatch()); + Assert.Null(reader.ReadNextRecordBatch()); + } + + public static async Task VerifyReaderAsync(ArrowStreamReader reader, RecordBatch originalBatch) + { + RecordBatch readBatch = await reader.ReadNextRecordBatchAsync(); + CompareBatches(originalBatch, readBatch); + + // There should only be one batch - calling ReadNextRecordBatchAsync again should return null. + Assert.Null(await reader.ReadNextRecordBatchAsync()); + Assert.Null(await reader.ReadNextRecordBatchAsync()); + } + + public static void CompareBatches(RecordBatch expectedBatch, RecordBatch actualBatch, bool strictCompare = true) + { + SchemaComparer.Compare(expectedBatch.Schema, actualBatch.Schema); + Assert.Equal(expectedBatch.Length, actualBatch.Length); + Assert.Equal(expectedBatch.ColumnCount, actualBatch.ColumnCount); + + for (int i = 0; i < expectedBatch.ColumnCount; i++) + { + IArrowArray expectedArray = expectedBatch.Arrays.ElementAt(i); + IArrowArray actualArray = actualBatch.Arrays.ElementAt(i); + + CompareArrays(expectedArray, actualArray, strictCompare); + } + } + + public static void CompareArrays(IArrowArray expectedArray, IArrowArray actualArray, bool strictCompare = true) + { + actualArray.Accept(new ArrayComparer(expectedArray, strictCompare)); + } + + private class ArrayComparer : + IArrowArrayVisitor<Int8Array>, + IArrowArrayVisitor<Int16Array>, + IArrowArrayVisitor<Int32Array>, + IArrowArrayVisitor<Int64Array>, + IArrowArrayVisitor<UInt8Array>, + IArrowArrayVisitor<UInt16Array>, + IArrowArrayVisitor<UInt32Array>, + IArrowArrayVisitor<UInt64Array>, + IArrowArrayVisitor<FloatArray>, + IArrowArrayVisitor<DoubleArray>, + IArrowArrayVisitor<BooleanArray>, + IArrowArrayVisitor<TimestampArray>, + IArrowArrayVisitor<Date32Array>, + IArrowArrayVisitor<Date64Array>, + IArrowArrayVisitor<ListArray>, + IArrowArrayVisitor<StringArray>, + IArrowArrayVisitor<FixedSizeBinaryArray>, + IArrowArrayVisitor<BinaryArray>, + IArrowArrayVisitor<StructArray>, + IArrowArrayVisitor<Decimal128Array>, + IArrowArrayVisitor<Decimal256Array>, + IArrowArrayVisitor<DictionaryArray> + { + private readonly IArrowArray _expectedArray; + private readonly ArrayTypeComparer _arrayTypeComparer; + private readonly bool _strictCompare; + + public ArrayComparer(IArrowArray expectedArray, bool strictCompare) + { + _expectedArray = expectedArray; + _arrayTypeComparer = new ArrayTypeComparer(expectedArray.Data.DataType); + _strictCompare = strictCompare; + } + + public void Visit(Int8Array array) => CompareArrays(array); + public void Visit(Int16Array array) => CompareArrays(array); + public void Visit(Int32Array array) => CompareArrays(array); + public void Visit(Int64Array array) => CompareArrays(array); + public void Visit(UInt8Array array) => CompareArrays(array); + public void Visit(UInt16Array array) => CompareArrays(array); + public void Visit(UInt32Array array) => CompareArrays(array); + public void Visit(UInt64Array array) => CompareArrays(array); + public void Visit(FloatArray array) => CompareArrays(array); + public void Visit(DoubleArray array) => CompareArrays(array); + public void Visit(BooleanArray array) => CompareArrays(array); + public void Visit(TimestampArray array) => CompareArrays(array); + public void Visit(Date32Array array) => CompareArrays(array); + public void Visit(Date64Array array) => CompareArrays(array); + public void Visit(ListArray array) => CompareArrays(array); + public void Visit(FixedSizeBinaryArray array) => CompareArrays(array); + public void Visit(Decimal128Array array) => CompareArrays(array); + public void Visit(Decimal256Array array) => CompareArrays(array); + public void Visit(StringArray array) => CompareBinaryArrays<StringArray>(array); + public void Visit(BinaryArray array) => CompareBinaryArrays<BinaryArray>(array); + + public void Visit(StructArray array) + { + Assert.IsAssignableFrom<StructArray>(_expectedArray); + StructArray expectedArray = (StructArray)_expectedArray; + + Assert.Equal(expectedArray.Length, array.Length); + Assert.Equal(expectedArray.NullCount, array.NullCount); + Assert.Equal(expectedArray.Offset, array.Offset); + Assert.Equal(expectedArray.Data.Children.Length, array.Data.Children.Length); + Assert.Equal(expectedArray.Fields.Count, array.Fields.Count); + + for (int i = 0; i < array.Fields.Count; i++) + { + array.Fields[i].Accept(new ArrayComparer(expectedArray.Fields[i], _strictCompare)); + } + } + + public void Visit(DictionaryArray array) + { + Assert.IsAssignableFrom<DictionaryArray>(_expectedArray); + DictionaryArray expectedArray = (DictionaryArray)_expectedArray; + var indicesComparer = new ArrayComparer(expectedArray.Indices, _strictCompare); + var dictionaryComparer = new ArrayComparer(expectedArray.Dictionary, _strictCompare); + array.Indices.Accept(indicesComparer); + array.Dictionary.Accept(dictionaryComparer); + } + + public void Visit(IArrowArray array) => throw new NotImplementedException(); + + private void CompareBinaryArrays<T>(BinaryArray actualArray) + where T : IArrowArray + { + Assert.IsAssignableFrom<T>(_expectedArray); + Assert.IsAssignableFrom<T>(actualArray); + + var expectedArray = (BinaryArray)_expectedArray; + + actualArray.Data.DataType.Accept(_arrayTypeComparer); + + Assert.Equal(expectedArray.Length, actualArray.Length); + Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); + + if (_strictCompare) + { + Assert.True(expectedArray.ValueOffsetsBuffer.Span.SequenceEqual(actualArray.ValueOffsetsBuffer.Span)); + Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length))); + } + else + { + for (int i = 0; i < expectedArray.Length; i++) + { + Assert.True( + expectedArray.GetBytes(i).SequenceEqual(actualArray.GetBytes(i)), + $"BinaryArray values do not match at index {i}."); + } + } + } + + private void CompareArrays(FixedSizeBinaryArray actualArray) + { + Assert.IsAssignableFrom<FixedSizeBinaryArray>(_expectedArray); + Assert.IsAssignableFrom<FixedSizeBinaryArray>(actualArray); + + var expectedArray = (FixedSizeBinaryArray)_expectedArray; + + actualArray.Data.DataType.Accept(_arrayTypeComparer); + + Assert.Equal(expectedArray.Length, actualArray.Length); + Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); + + if (_strictCompare) + { + Assert.True(expectedArray.ValueBuffer.Span.Slice(0, expectedArray.Length).SequenceEqual(actualArray.ValueBuffer.Span.Slice(0, actualArray.Length))); + } + else + { + for (int i = 0; i < expectedArray.Length; i++) + { + Assert.True( + expectedArray.GetBytes(i).SequenceEqual(actualArray.GetBytes(i)), + $"FixedSizeBinaryArray values do not match at index {i}."); + } + } + } + + private void CompareArrays<T>(PrimitiveArray<T> actualArray) + where T : struct, IEquatable<T> + { + Assert.IsAssignableFrom<PrimitiveArray<T>>(_expectedArray); + PrimitiveArray<T> expectedArray = (PrimitiveArray<T>)_expectedArray; + + actualArray.Data.DataType.Accept(_arrayTypeComparer); + + Assert.Equal(expectedArray.Length, actualArray.Length); + Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); + + if (_strictCompare) + { + Assert.True(expectedArray.Values.Slice(0, expectedArray.Length).SequenceEqual(actualArray.Values.Slice(0, actualArray.Length))); + } + else + { + for (int i = 0; i < expectedArray.Length; i++) + { + Assert.Equal(expectedArray.GetValue(i), actualArray.GetValue(i)); + } + } + } + + private void CompareArrays(BooleanArray actualArray) + { + Assert.IsAssignableFrom<BooleanArray>(_expectedArray); + BooleanArray expectedArray = (BooleanArray)_expectedArray; + + actualArray.Data.DataType.Accept(_arrayTypeComparer); + + Assert.Equal(expectedArray.Length, actualArray.Length); + Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); + + if (_strictCompare) + { + int booleanByteCount = BitUtility.ByteCount(expectedArray.Length); + Assert.True(expectedArray.Values.Slice(0, booleanByteCount).SequenceEqual(actualArray.Values.Slice(0, booleanByteCount))); + } + else + { + for (int i = 0; i < expectedArray.Length; i++) + { + Assert.Equal(expectedArray.GetValue(i), actualArray.GetValue(i)); + } + } + } + + private void CompareArrays(ListArray actualArray) + { + Assert.IsAssignableFrom<ListArray>(_expectedArray); + ListArray expectedArray = (ListArray)_expectedArray; + + actualArray.Data.DataType.Accept(_arrayTypeComparer); + + Assert.Equal(expectedArray.Length, actualArray.Length); + Assert.Equal(expectedArray.NullCount, actualArray.NullCount); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + + CompareValidityBuffer(expectedArray.NullCount, _expectedArray.Length, expectedArray.NullBitmapBuffer, actualArray.NullBitmapBuffer); + Assert.True(expectedArray.ValueOffsetsBuffer.Span.SequenceEqual(actualArray.ValueOffsetsBuffer.Span)); + + actualArray.Values.Accept(new ArrayComparer(expectedArray.Values, _strictCompare)); + } + + private void CompareValidityBuffer(int nullCount, int arrayLength, ArrowBuffer expectedValidityBuffer, ArrowBuffer actualValidityBuffer) + { + if (_strictCompare) + { + Assert.True(expectedValidityBuffer.Span.SequenceEqual(actualValidityBuffer.Span)); + } + else if (nullCount != 0) + { + int validityBitmapByteCount = BitUtility.ByteCount(arrayLength); + Assert.True( + expectedValidityBuffer.Span.Slice(0, validityBitmapByteCount).SequenceEqual(actualValidityBuffer.Span.Slice(0, validityBitmapByteCount)), + "Validity buffers do not match."); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs new file mode 100644 index 000000000..973fc6a0a --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Memory; +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowStreamReaderTests + { + [Fact] + public void Ctor_LeaveOpenDefault_StreamClosedOnDispose() + { + var stream = new MemoryStream(); + new ArrowStreamReader(stream).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenFalse_StreamClosedOnDispose() + { + var stream = new MemoryStream(); + new ArrowStreamReader(stream, leaveOpen: false).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenTrue_StreamValidOnDispose() + { + var stream = new MemoryStream(); + new ArrowStreamReader(stream, leaveOpen: true).Dispose(); + Assert.Equal(0, stream.Position); + } + + [Theory] + [InlineData(true, true, 2)] + [InlineData(true, false, 1)] + [InlineData(false, true, 2)] + [InlineData(false, false, 1)] + public async Task Ctor_MemoryPool_AllocatesFromPool(bool shouldLeaveOpen, bool createDictionaryArray, int expectedAllocations) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + + using (MemoryStream stream = new MemoryStream()) + { + ArrowStreamWriter writer = new ArrowStreamWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + + stream.Position = 0; + + var memoryPool = new TestMemoryAllocator(); + ArrowStreamReader reader = new ArrowStreamReader(stream, memoryPool, shouldLeaveOpen); + reader.ReadNextRecordBatch(); + + Assert.Equal(expectedAllocations, memoryPool.Statistics.Allocations); + Assert.True(memoryPool.Statistics.BytesAllocated > 0); + + reader.Dispose(); + + if (shouldLeaveOpen) + { + Assert.True(stream.Position > 0); + } + else + { + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReadRecordBatch_Memory(bool writeEnd) + { + await TestReaderFromMemory((reader, originalBatch) => + { + ArrowReaderVerifier.VerifyReader(reader, originalBatch); + return Task.CompletedTask; + }, writeEnd); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReadRecordBatchAsync_Memory(bool writeEnd) + { + await TestReaderFromMemory(ArrowReaderVerifier.VerifyReaderAsync, writeEnd); + } + + private static async Task TestReaderFromMemory( + Func<ArrowStreamReader, RecordBatch, Task> verificationFunc, + bool writeEnd) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + + byte[] buffer; + using (MemoryStream stream = new MemoryStream()) + { + ArrowStreamWriter writer = new ArrowStreamWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + if (writeEnd) + { + await writer.WriteEndAsync(); + } + buffer = stream.GetBuffer(); + } + + ArrowStreamReader reader = new ArrowStreamReader(buffer); + await verificationFunc(reader, originalBatch); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadRecordBatch_Stream(bool writeEnd, bool createDictionaryArray) + { + await TestReaderFromStream((reader, originalBatch) => + { + ArrowReaderVerifier.VerifyReader(reader, originalBatch); + return Task.CompletedTask; + }, writeEnd, createDictionaryArray); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadRecordBatchAsync_Stream(bool writeEnd, bool createDictionaryArray) + { + await TestReaderFromStream(ArrowReaderVerifier.VerifyReaderAsync, writeEnd, createDictionaryArray); + } + + private static async Task TestReaderFromStream( + Func<ArrowStreamReader, RecordBatch, Task> verificationFunc, + bool writeEnd, bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + + using (MemoryStream stream = new MemoryStream()) + { + ArrowStreamWriter writer = new ArrowStreamWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + if (writeEnd) + { + await writer.WriteEndAsync(); + } + + stream.Position = 0; + + ArrowStreamReader reader = new ArrowStreamReader(stream); + await verificationFunc(reader, originalBatch); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReadRecordBatch_PartialReadStream(bool createDictionaryArray) + { + await TestReaderFromPartialReadStream((reader, originalBatch) => + { + ArrowReaderVerifier.VerifyReader(reader, originalBatch); + return Task.CompletedTask; + }, createDictionaryArray); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReadRecordBatchAsync_PartialReadStream(bool createDictionaryArray) + { + await TestReaderFromPartialReadStream(ArrowReaderVerifier.VerifyReaderAsync, createDictionaryArray); + } + + /// <summary> + /// Verifies that the stream reader reads multiple times when a stream + /// only returns a subset of the data from each Read. + /// </summary> + private static async Task TestReaderFromPartialReadStream(Func<ArrowStreamReader, RecordBatch, Task> verificationFunc, bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + + using (PartialReadStream stream = new PartialReadStream()) + { + ArrowStreamWriter writer = new ArrowStreamWriter(stream, originalBatch.Schema); + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + + stream.Position = 0; + + ArrowStreamReader reader = new ArrowStreamReader(stream); + await verificationFunc(reader, originalBatch); + } + } + + /// <summary> + /// A stream class that only returns a part of the data at a time. + /// </summary> + private class PartialReadStream : MemoryStream + { + // by default return 20 bytes at a time + public int PartialReadLength { get; set; } = 20; + + public override int Read(Span<byte> destination) + { + if (destination.Length > PartialReadLength) + { + destination = destination.Slice(0, PartialReadLength); + } + + return base.Read(destination); + } + + public override ValueTask<int> ReadAsync(Memory<byte> destination, CancellationToken cancellationToken = default) + { + if (destination.Length > PartialReadLength) + { + destination = destination.Slice(0, PartialReadLength); + } + + return base.ReadAsync(destination, cancellationToken); + } + } + } +} + diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs new file mode 100644 index 000000000..4932217b1 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ArrowStreamWriterTests.cs @@ -0,0 +1,682 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ArrowStreamWriterTests + { + [Fact] + public void Ctor_LeaveOpenDefault_StreamClosedOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowStreamWriter(stream, originalBatch.Schema).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenFalse_StreamClosedOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: false).Dispose(); + Assert.Throws<ObjectDisposedException>(() => stream.Position); + } + + [Fact] + public void Ctor_LeaveOpenTrue_StreamValidOnDispose() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); + var stream = new MemoryStream(); + new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true).Dispose(); + Assert.Equal(0, stream.Position); + } + + [Theory] + [InlineData(true, 32153)] + [InlineData(false, 32154)] + public void CanWriteToNetworkStream(bool createDictionaryArray, int port) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + + TcpListener listener = new TcpListener(IPAddress.Loopback, port); + listener.Start(); + + using (TcpClient sender = new TcpClient()) + { + sender.Connect(IPAddress.Loopback, port); + NetworkStream stream = sender.GetStream(); + + using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema)) + { + writer.WriteRecordBatch(originalBatch); + writer.WriteEnd(); + + stream.Flush(); + } + } + + using (TcpClient receiver = listener.AcceptTcpClient()) + { + NetworkStream stream = receiver.GetStream(); + using (var reader = new ArrowStreamReader(stream)) + { + RecordBatch newBatch = reader.ReadNextRecordBatch(); + ArrowReaderVerifier.CompareBatches(originalBatch, newBatch); + } + } + } + + [Theory] + [InlineData(true, 32155)] + [InlineData(false, 32156)] + public async Task CanWriteToNetworkStreamAsync(bool createDictionaryArray, int port) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + + TcpListener listener = new TcpListener(IPAddress.Loopback, port); + listener.Start(); + + using (TcpClient sender = new TcpClient()) + { + sender.Connect(IPAddress.Loopback, port); + NetworkStream stream = sender.GetStream(); + + using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema)) + { + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + + stream.Flush(); + } + } + + using (TcpClient receiver = listener.AcceptTcpClient()) + { + NetworkStream stream = receiver.GetStream(); + using (var reader = new ArrowStreamReader(stream)) + { + RecordBatch newBatch = reader.ReadNextRecordBatch(); + ArrowReaderVerifier.CompareBatches(originalBatch, newBatch); + } + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void WriteEmptyBatch(bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 0, createDictionaryArray: createDictionaryArray); + + TestRoundTripRecordBatch(originalBatch); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task WriteEmptyBatchAsync(bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 0, createDictionaryArray: createDictionaryArray); + + await TestRoundTripRecordBatchAsync(originalBatch); + } + + [Fact] + public void WriteBatchWithNulls() + { + RecordBatch originalBatch = new RecordBatch.Builder() + .Append("Column1", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10)))) + .Append("Column2", true, new Int32Array( + valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(), + length: 10, + nullCount: 2, + offset: 0)) + .Append("Column3", true, new Int32Array( + valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0x00).Append(0x00).Build(), + length: 10, + nullCount: 10, + offset: 0)) + .Append("NullableBooleanColumn", true, new BooleanArray( + valueBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xed).Append(0xff).Build(), + length: 10, + nullCount: 3, + offset: 0)) + .Build(); + + TestRoundTripRecordBatch(originalBatch); + } + + [Fact] + public async Task WriteBatchWithNullsAsync() + { + RecordBatch originalBatch = new RecordBatch.Builder() + .Append("Column1", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10)))) + .Append("Column2", true, new Int32Array( + valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(), + length: 10, + nullCount: 2, + offset: 0)) + .Append("Column3", true, new Int32Array( + valueBuffer: new ArrowBuffer.Builder<int>().AppendRange(Enumerable.Range(0, 10)).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0x00).Append(0x00).Build(), + length: 10, + nullCount: 10, + offset: 0)) + .Append("NullableBooleanColumn", true, new BooleanArray( + valueBuffer: new ArrowBuffer.Builder<byte>().Append(0xfd).Append(0xff).Build(), + nullBitmapBuffer: new ArrowBuffer.Builder<byte>().Append(0xed).Append(0xff).Build(), + length: 10, + nullCount: 3, + offset: 0)) + .Build(); + + await TestRoundTripRecordBatchAsync(originalBatch); + } + + private static void TestRoundTripRecordBatches(List<RecordBatch> originalBatches, IpcOptions options = null) + { + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, originalBatches[0].Schema, leaveOpen: true, options)) + { + foreach (RecordBatch originalBatch in originalBatches) + { + writer.WriteRecordBatch(originalBatch); + } + writer.WriteEnd(); + } + + stream.Position = 0; + + using (var reader = new ArrowStreamReader(stream)) + { + foreach (RecordBatch originalBatch in originalBatches) + { + RecordBatch newBatch = reader.ReadNextRecordBatch(); + ArrowReaderVerifier.CompareBatches(originalBatch, newBatch); + } + } + } + } + + private static async Task TestRoundTripRecordBatchesAsync(List<RecordBatch> originalBatches, IpcOptions options = null) + { + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, originalBatches[0].Schema, leaveOpen: true, options)) + { + foreach (RecordBatch originalBatch in originalBatches) + { + await writer.WriteRecordBatchAsync(originalBatch); + } + await writer.WriteEndAsync(); + } + + stream.Position = 0; + + using (var reader = new ArrowStreamReader(stream)) + { + foreach (RecordBatch originalBatch in originalBatches) + { + RecordBatch newBatch = reader.ReadNextRecordBatch(); + ArrowReaderVerifier.CompareBatches(originalBatch, newBatch); + } + } + } + } + + private static void TestRoundTripRecordBatch(RecordBatch originalBatch, IpcOptions options = null) + { + TestRoundTripRecordBatches(new List<RecordBatch> { originalBatch }, options); + } + + private static async Task TestRoundTripRecordBatchAsync(RecordBatch originalBatch, IpcOptions options = null) + { + await TestRoundTripRecordBatchesAsync(new List<RecordBatch> { originalBatch }, options); + } + + [Fact] + public void WriteBatchWithCorrectPadding() + { + byte value1 = 0x04; + byte value2 = 0x14; + var batch = new RecordBatch( + new Schema.Builder() + .Field(f => f.Name("age").DataType(Int32Type.Default)) + .Field(f => f.Name("characterCount").DataType(Int32Type.Default)) + .Build(), + new IArrowArray[] + { + new Int32Array( + new ArrowBuffer(new byte[] { value1, value1, 0x00, 0x00 }), + ArrowBuffer.Empty, + length: 1, + nullCount: 0, + offset: 0), + new Int32Array( + new ArrowBuffer(new byte[] { value2, value2, 0x00, 0x00 }), + ArrowBuffer.Empty, + length: 1, + nullCount: 0, + offset: 0) + }, + length: 1); + + TestRoundTripRecordBatch(batch); + + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, batch.Schema, leaveOpen: true)) + { + writer.WriteRecordBatch(batch); + writer.WriteEnd(); + } + + byte[] writtenBytes = stream.ToArray(); + + // ensure that the data buffers at the end are 8-byte aligned + Assert.Equal(value1, writtenBytes[writtenBytes.Length - 24]); + Assert.Equal(value1, writtenBytes[writtenBytes.Length - 23]); + for (int i = 22; i > 16; i--) + { + Assert.Equal(0, writtenBytes[writtenBytes.Length - i]); + } + + Assert.Equal(value2, writtenBytes[writtenBytes.Length - 16]); + Assert.Equal(value2, writtenBytes[writtenBytes.Length - 15]); + for (int i = 14; i > 8; i--) + { + Assert.Equal(0, writtenBytes[writtenBytes.Length - i]); + } + + // verify the EOS is written correctly + for (int i = 8; i > 4; i--) + { + Assert.Equal(0xFF, writtenBytes[writtenBytes.Length - i]); + } + for (int i = 4; i > 0; i--) + { + Assert.Equal(0x00, writtenBytes[writtenBytes.Length - i]); + } + } + } + + [Fact] + public async Task WriteBatchWithCorrectPaddingAsync() + { + byte value1 = 0x04; + byte value2 = 0x14; + var batch = new RecordBatch( + new Schema.Builder() + .Field(f => f.Name("age").DataType(Int32Type.Default)) + .Field(f => f.Name("characterCount").DataType(Int32Type.Default)) + .Build(), + new IArrowArray[] + { + new Int32Array( + new ArrowBuffer(new byte[] { value1, value1, 0x00, 0x00 }), + ArrowBuffer.Empty, + length: 1, + nullCount: 0, + offset: 0), + new Int32Array( + new ArrowBuffer(new byte[] { value2, value2, 0x00, 0x00 }), + ArrowBuffer.Empty, + length: 1, + nullCount: 0, + offset: 0) + }, + length: 1); + + await TestRoundTripRecordBatchAsync(batch); + + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, batch.Schema, leaveOpen: true)) + { + await writer.WriteRecordBatchAsync(batch); + await writer.WriteEndAsync(); + } + + byte[] writtenBytes = stream.ToArray(); + + // ensure that the data buffers at the end are 8-byte aligned + Assert.Equal(value1, writtenBytes[writtenBytes.Length - 24]); + Assert.Equal(value1, writtenBytes[writtenBytes.Length - 23]); + for (int i = 22; i > 16; i--) + { + Assert.Equal(0, writtenBytes[writtenBytes.Length - i]); + } + + Assert.Equal(value2, writtenBytes[writtenBytes.Length - 16]); + Assert.Equal(value2, writtenBytes[writtenBytes.Length - 15]); + for (int i = 14; i > 8; i--) + { + Assert.Equal(0, writtenBytes[writtenBytes.Length - i]); + } + + // verify the EOS is written correctly + for (int i = 8; i > 4; i--) + { + Assert.Equal(0xFF, writtenBytes[writtenBytes.Length - i]); + } + for (int i = 4; i > 0; i--) + { + Assert.Equal(0x00, writtenBytes[writtenBytes.Length - i]); + } + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void LegacyIpcFormatRoundTrips(bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + TestRoundTripRecordBatch(originalBatch, new IpcOptions() { WriteLegacyIpcFormat = true }); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task LegacyIpcFormatRoundTripsAsync(bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + await TestRoundTripRecordBatchAsync(originalBatch, new IpcOptions() { WriteLegacyIpcFormat = true }); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void WriteLegacyIpcFormat(bool writeLegacyIpcFormat, bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + var options = new IpcOptions() { WriteLegacyIpcFormat = writeLegacyIpcFormat }; + + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true, options)) + { + writer.WriteRecordBatch(originalBatch); + writer.WriteEnd(); + } + + stream.Position = 0; + + // ensure the continuation is written correctly + byte[] buffer = stream.ToArray(); + int messageLength = BinaryPrimitives.ReadInt32LittleEndian(buffer); + int endOfBuffer1 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 8)); + int endOfBuffer2 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 4)); + if (writeLegacyIpcFormat) + { + // the legacy IPC format doesn't have a continuation token at the start + Assert.NotEqual(-1, messageLength); + Assert.NotEqual(-1, endOfBuffer1); + } + else + { + // the latest IPC format has a continuation token at the start + Assert.Equal(-1, messageLength); + Assert.Equal(-1, endOfBuffer1); + } + + Assert.Equal(0, endOfBuffer2); + } + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task WriteLegacyIpcFormatAsync(bool writeLegacyIpcFormat, bool createDictionaryArray) + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100, createDictionaryArray: createDictionaryArray); + var options = new IpcOptions() { WriteLegacyIpcFormat = writeLegacyIpcFormat }; + + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true, options)) + { + await writer.WriteRecordBatchAsync(originalBatch); + await writer.WriteEndAsync(); + } + + stream.Position = 0; + + // ensure the continuation is written correctly + byte[] buffer = stream.ToArray(); + int messageLength = BinaryPrimitives.ReadInt32LittleEndian(buffer); + int endOfBuffer1 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 8)); + int endOfBuffer2 = BinaryPrimitives.ReadInt32LittleEndian(buffer.AsSpan(buffer.Length - 4)); + if (writeLegacyIpcFormat) + { + // the legacy IPC format doesn't have a continuation token at the start + Assert.NotEqual(-1, messageLength); + Assert.NotEqual(-1, endOfBuffer1); + } + else + { + // the latest IPC format has a continuation token at the start + Assert.Equal(-1, messageLength); + Assert.Equal(-1, endOfBuffer1); + } + + Assert.Equal(0, endOfBuffer2); + } + } + + [Fact] + public void WritesMetadataCorrectly() + { + Schema.Builder schemaBuilder = new Schema.Builder() + .Metadata("index", "1, 2, 3, 4, 5") + .Metadata("reverseIndex", "5, 4, 3, 2, 1") + .Field(f => f + .Name("IntCol") + .DataType(UInt32Type.Default) + .Metadata("custom1", "false") + .Metadata("custom2", "true")) + .Field(f => f + .Name("StringCol") + .DataType(StringType.Default) + .Metadata("custom2", "false") + .Metadata("custom3", "4")) + .Field(f => f + .Name("StructCol") + .DataType(new StructType(new[] { + new Field("Inner1", FloatType.Default, nullable: false), + new Field("Inner2", DoubleType.Default, nullable: true, new Dictionary<string, string>() { { "customInner", "1" }, { "customInner2", "3" } }) + })) + .Metadata("custom4", "6.4") + .Metadata("custom1", "true")); + + var schema = schemaBuilder.Build(); + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(schema, length: 10); + + TestRoundTripRecordBatch(originalBatch); + } + + [Fact] + public async Task WriteMultipleDictionaryArraysAsync() + { + List<RecordBatch> originalRecordBatches = CreateMultipleDictionaryArraysTestData(); + await TestRoundTripRecordBatchesAsync(originalRecordBatches); + } + + [Fact] + public void WriteMultipleDictionaryArrays() + { + List<RecordBatch> originalRecordBatches = CreateMultipleDictionaryArraysTestData(); + TestRoundTripRecordBatches(originalRecordBatches); + } + + private List<RecordBatch> CreateMultipleDictionaryArraysTestData() + { + var dictionaryData = new List<string> { "a", "b", "c" }; + int length = dictionaryData.Count; + + var schemaForSimpleCase = new Schema(new List<Field> { + new Field("int8", Int8Type.Default, true), + new Field("uint8", UInt8Type.Default, true), + new Field("int16", Int16Type.Default, true), + new Field("uint16", UInt16Type.Default, true), + new Field("int32", Int32Type.Default, true), + new Field("uint32", UInt32Type.Default, true), + new Field("int64", Int64Type.Default, true), + new Field("uint64", UInt64Type.Default, true) + }, null); + + StringArray dictionary = new StringArray.Builder().AppendRange(dictionaryData).Build(); + IEnumerable<IArrowArray> indicesArraysForSimpleCase = TestData.CreateArrays(schemaForSimpleCase, length); + + var fields = new List<Field>(capacity: length + 1); + var testTargetArrays = new List<IArrowArray>(capacity: length + 1); + + foreach (IArrowArray indices in indicesArraysForSimpleCase) + { + var dictionaryArray = new DictionaryArray( + new DictionaryType(indices.Data.DataType, StringType.Default, false), + indices, dictionary); + testTargetArrays.Add(dictionaryArray); + fields.Add(new Field($"dictionaryField_{indices.Data.DataType.Name}", dictionaryArray.Data.DataType, false)); + } + + (Field dictionaryTypeListArrayField, ListArray dictionaryTypeListArray) = CreateDictionaryTypeListArrayTestData(dictionary); + + fields.Add(dictionaryTypeListArrayField); + testTargetArrays.Add(dictionaryTypeListArray); + + (Field listTypeDictionaryArrayField, DictionaryArray listTypeDictionaryArray) = CreateListTypeDictionaryArrayTestData(dictionaryData); + + fields.Add(listTypeDictionaryArrayField); + testTargetArrays.Add(listTypeDictionaryArray); + + var schema = new Schema(fields, null); + + return new List<RecordBatch> { + new RecordBatch(schema, testTargetArrays, length), + new RecordBatch(schema, testTargetArrays, length), + }; + } + + private Tuple<Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary) + { + Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build(); + + //DictionaryArray has no Builder for now, so creating ListArray directly. + var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false); + var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary); + + var valueOffsetsBufferBuilder = new ArrowBuffer.Builder<int>(); + var validityBufferBuilder = new ArrowBuffer.BitmapBuilder(); + + foreach (int i in Enumerable.Range(0, dictionary.Length + 1)) + { + valueOffsetsBufferBuilder.Append(i); + validityBufferBuilder.Append(true); + } + + var dictionaryField = new Field("dictionaryField_list", dictionaryType, false); + var listType = new ListType(dictionaryField); + var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build()); + + return Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray); + } + + private Tuple<Field, DictionaryArray> CreateListTypeDictionaryArrayTestData(List<string> dictionaryDataBase) + { + var listBuilder = new ListArray.Builder(StringType.Default); + var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder; + + foreach(string data in dictionaryDataBase) { + listBuilder.Append(); + valueBuilder.Append(data); + } + + ListArray dictionary = listBuilder.Build(); + Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build(); + var dictionaryArrayType = new DictionaryType(Int32Type.Default, dictionary.Data.DataType, false); + var dictionaryArray = new DictionaryArray(dictionaryArrayType, indiceArray, dictionary); + + return Tuple.Create(new Field($"dictionaryField_{dictionaryArray.Data.DataType.Name}", dictionaryArrayType, false), dictionaryArray); + } + + /// <summary> + /// Tests that writing an arrow stream with no RecordBatches produces the correct result. + /// </summary> + [Fact] + public void WritesEmptyFile() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1); + + var stream = new MemoryStream(); + var writer = new ArrowStreamWriter(stream, originalBatch.Schema); + + writer.WriteStart(); + writer.WriteEnd(); + + stream.Position = 0; + + var reader = new ArrowStreamReader(stream); + RecordBatch readBatch = reader.ReadNextRecordBatch(); + Assert.Null(readBatch); + SchemaComparer.Compare(originalBatch.Schema, reader.Schema); + } + + /// <summary> + /// Tests that writing an arrow stream with no RecordBatches produces the correct + /// result when using WriteStartAsync and WriteEndAsync. + /// </summary> + [Fact] + public async Task WritesEmptyFileAsync() + { + RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1); + + var stream = new MemoryStream(); + var writer = new ArrowStreamWriter(stream, originalBatch.Schema); + + await writer.WriteStartAsync(); + await writer.WriteEndAsync(); + + stream.Position = 0; + + var reader = new ArrowStreamReader(stream); + RecordBatch readBatch = reader.ReadNextRecordBatch(); + Assert.Null(readBatch); + SchemaComparer.Compare(originalBatch.Schema, reader.Schema); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/BinaryArrayBuilderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/BinaryArrayBuilderTests.cs new file mode 100644 index 000000000..7f45ce857 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/BinaryArrayBuilderTests.cs @@ -0,0 +1,489 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Apache.Arrow.Memory; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class BinaryArrayBuilderTests + { + private static readonly MemoryAllocator _allocator = new NativeMemoryAllocator(); + + // Various example byte arrays for use in testing. + private static readonly byte[] _exampleNull = null; + private static readonly byte[] _exampleEmpty = { }; + private static readonly byte[] _exampleNonEmpty1 = { 10, 20, 30, 40 }; + private static readonly byte[] _exampleNonEmpty2 = { 50, 60, 70, 80 }; + private static readonly byte[] _exampleNonEmpty3 = { 90 }; + + // Base set of single bytes that may be used to append to a builder in testing. + private static readonly byte[] _singleBytesToAppend = { 0, 123, 127, 255 }; + + // Base set of byte arrays that may be used to append to a builder in testing. + private static readonly byte[][] _byteArraysToAppend = + { + _exampleNull, + _exampleEmpty, + _exampleNonEmpty2, + _exampleNonEmpty3, + }; + + // Base set of multiple byte arrays that may be used to append to a builder in testing. + private static readonly byte[][][] _byteArrayArraysToAppend = + { + new byte[][] { }, + new[] { _exampleNull }, + new[] { _exampleEmpty }, + new[] { _exampleNonEmpty2 }, + new[] { _exampleNonEmpty2, _exampleNonEmpty3 }, + new[] { _exampleNonEmpty2, _exampleEmpty, _exampleNull }, + }; + + // Base set of byte arrays that can be used as "initial contents" of any builder under test. + private static readonly byte[][][] _initialContentsSet = + { + new byte[][] { }, + new[] { _exampleNull }, + new[] { _exampleEmpty }, + new[] { _exampleNonEmpty1 }, + new[] { _exampleNonEmpty1, _exampleNonEmpty3 }, + new[] { _exampleNonEmpty1, _exampleEmpty, _exampleNull }, + }; + + public class Append + { + public static IEnumerable<object[]> _appendSingleByteTestData = + from initialContents in _initialContentsSet + from singleByte in _singleBytesToAppend + select new object[] { initialContents, singleByte }; + + [Theory] + [MemberData(nameof(_appendSingleByteTestData))] + public void AppendSingleByte(byte[][] initialContents, byte singleByte) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + int expectedLength = initialLength + 1; + var expectedArrayContents = initialContents.Append(new[] { singleByte }); + + // Act + var actualReturnValue = builder.Append(singleByte); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + [Theory] + [MemberData(nameof(_appendSingleByteTestData))] + public void AppendSingleByteAfterClear(byte[][] initialContents, byte singleByte) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var expectedArrayContents = new[] { new[] { singleByte } }; + + // Act + var actualReturnValue = builder.Append(singleByte); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(1, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + public static readonly IEnumerable<object[]> _appendNullTestData = + from initialContents in _initialContentsSet + select new object[] { initialContents }; + + [Theory] + [MemberData(nameof(_appendNullTestData))] + public void AppendNull(byte[][] initialContents) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + int expectedLength = initialLength + 1; + var expectedArrayContents = initialContents.Append(null); + + // Act + var actualReturnValue = builder.AppendNull(); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + [Theory] + [MemberData(nameof(_appendNullTestData))] + public void AppendNullAfterClear(byte[][] initialContents) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var expectedArrayContents = new byte[][] { null }; + + // Act + var actualReturnValue = builder.AppendNull(); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(1, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + public static readonly IEnumerable<object[]> _appendNonNullByteArrayTestData = + from initialContents in _initialContentsSet + from bytes in _byteArraysToAppend + where bytes != null + select new object[] { initialContents, bytes }; + + [Theory] + [MemberData(nameof(_appendNonNullByteArrayTestData))] + public void AppendReadOnlySpan(byte[][] initialContents, byte[] bytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + var span = (ReadOnlySpan<byte>)bytes; + int expectedLength = initialLength + 1; + var expectedArrayContents = initialContents.Append(bytes); + + // Act + var actualReturnValue = builder.Append(span); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + [Theory] + [MemberData(nameof(_appendNonNullByteArrayTestData))] + public void AppendReadOnlySpanAfterClear(byte[][] initialContents, byte[] bytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var span = (ReadOnlySpan<byte>)bytes; + var expectedArrayContents = new[] { bytes }; + + // Act + var actualReturnValue = builder.Append(span); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(1, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + public static readonly IEnumerable<object[]> _appendByteArrayTestData = + from initialContents in _initialContentsSet + from bytes in _byteArraysToAppend + select new object[] { initialContents, bytes }; + + [Theory] + [MemberData(nameof(_appendByteArrayTestData))] + public void AppendEnumerable(byte[][] initialContents, byte[] bytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + int expectedLength = initialLength + 1; + var enumerable = (IEnumerable<byte>)bytes; + var expectedArrayContents = initialContents.Append(bytes); + + // Act + var actualReturnValue = builder.Append(enumerable); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + [Theory] + [MemberData(nameof(_appendByteArrayTestData))] + public void AppendEnumerableAfterClear(byte[][] initialContents, byte[] bytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var enumerable = (IEnumerable<byte>)bytes; + var expectedArrayContents = new[] { bytes }; + + // Act + var actualReturnValue = builder.Append(enumerable); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(1, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + } + + public class AppendRange + { + public static readonly IEnumerable<object[]> _appendRangeSingleBytesTestData = + from initialContents in _initialContentsSet + select new object[] { initialContents, _singleBytesToAppend }; + + [Theory] + [MemberData(nameof(_appendRangeSingleBytesTestData))] + public void AppendRangeSingleBytes(byte[][] initialContents, byte[] singleBytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + int expectedNewLength = initialLength + singleBytes.Length; + var expectedArrayContents = initialContents.Concat(singleBytes.Select(b => new[] { b })); + + // Act + var actualReturnValue = builder.AppendRange(singleBytes); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedNewLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + + } + + [Theory] + [MemberData(nameof(_appendRangeSingleBytesTestData))] + public void AppendRangeSingleBytesAfterClear(byte[][] initialContents, byte[] singleBytes) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var expectedArrayContents = singleBytes.Select(b => new[] { b }); + + // Act + var actualReturnValue = builder.AppendRange(singleBytes); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(singleBytes.Length, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + public static readonly IEnumerable<object[]> _appendRangeByteArraysTestData = + from initialContents in _initialContentsSet + from byteArrays in _byteArrayArraysToAppend + select new object[] { initialContents, byteArrays }; + + [Theory] + [MemberData(nameof(_appendRangeByteArraysTestData))] + public void AppendRangeArrays(byte[][] initialContents, byte[][] byteArrays) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + int initialLength = builder.Length; + int expectedNewLength = initialLength + byteArrays.Length; + var expectedArrayContents = initialContents.Concat(byteArrays); + + // Act + var actualReturnValue = builder.AppendRange(byteArrays); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(expectedNewLength, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + + [Theory] + [MemberData(nameof(_appendRangeByteArraysTestData))] + public void AppendRangeArraysAfterClear(byte[][] initialContents, byte[][] byteArrays) + { + // Arrange + var builder = new BinaryArray.Builder(); + if (initialContents.Length > 0) + builder.AppendRange(initialContents); + builder.Clear(); + var expectedArrayContents = byteArrays; + + // Act + var actualReturnValue = builder.AppendRange(byteArrays); + + // Assert + Assert.Equal(builder, actualReturnValue); + Assert.Equal(byteArrays.Length, builder.Length); + var actualArray = builder.Build(_allocator); + AssertArrayContents(expectedArrayContents, actualArray); + } + } + + public class Clear + { + [Fact] + public void ClearEmpty() + { + // Arrange + var builder = new BinaryArray.Builder(); + + // Act + var actualReturnValue = builder.Clear(); + + // Assert + Assert.NotNull(actualReturnValue); + Assert.Equal(builder, actualReturnValue); + Assert.Equal(0, builder.Length); + var array = builder.Build(_allocator); + Assert.Equal(0, array.Length); + } + + public static readonly IEnumerable<object[]> _testData = + from byteArrays in _byteArrayArraysToAppend + select new object[] { byteArrays }; + + [Theory] + [MemberData(nameof(_testData))] + public void ClearNonEmpty(byte[][] byteArrays) + { + // Arrange + var builder = new BinaryArray.Builder(); + builder.AppendRange(byteArrays); + + // Act + var actualReturnValue = builder.Clear(); + + // Assert + Assert.NotNull(actualReturnValue); + Assert.Equal(builder, actualReturnValue); + Assert.Equal(0, builder.Length); + var array = builder.Build(_allocator); + Assert.Equal(0, array.Length); + } + } + + public class Build + { + [Fact] + public void BuildImmediately() + { + // Arrange + var builder = new BinaryArray.Builder(); + + // Act + var array = builder.Build(_allocator); + + // Assert + Assert.Equal(0, array.Length); + } + + public static readonly IEnumerable<object[]> _testData = + from ba1 in _initialContentsSet + from ba2 in _byteArrayArraysToAppend + select new object[] { ba1.Concat(ba2) }; + + [Theory] + [MemberData(nameof(_testData))] + public void AppendThenBuild(byte[][] byteArrays) + { + // Arrange + var builder = new BinaryArray.Builder(); + foreach (var byteArray in byteArrays) + { + // Test the type of byte array to ensure each Append() overload is exercised. + if (byteArray == null) + { + builder.AppendNull(); + } + else if (byteArray.Length == 1) + { + builder.Append(byteArray[0]); + } + else + { + builder.Append((ReadOnlySpan<byte>)byteArray); + } + } + + // Act + var array = builder.Build(_allocator); + + // Assert + AssertArrayContents(byteArrays, array); + } + + [Theory] + [MemberData(nameof(_testData))] + public void BuildMultipleTimes(byte[][] byteArrays) + { + // Arrange + var builder = new BinaryArray.Builder(); + builder.AppendRange(byteArrays); + builder.Build(_allocator); + + // Act + var array = builder.Build(_allocator); + + // Assert + AssertArrayContents(byteArrays, array); + } + } + + private static void AssertArrayContents(IEnumerable<byte[]> expectedContents, BinaryArray array) + { + var expectedContentsArr = expectedContents.ToArray(); + Assert.Equal(expectedContentsArr.Length, array.Length); + for (int i = 0; i < array.Length; i++) + { + var expectedArray = expectedContentsArr[i]; + var actualArray = array.IsNull(i) ? null : array.GetBytes(i).ToArray(); + Assert.Equal(expectedArray, actualArray); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/BitUtilityTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/BitUtilityTests.cs new file mode 100644 index 000000000..5e18716a0 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/BitUtilityTests.cs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class BitUtilityTests + { + public class ByteCount + { + [Theory] + [InlineData(0, 0)] + [InlineData(1, 1)] + [InlineData(8, 1)] + [InlineData(9, 2)] + [InlineData(32, 4)] + public void HasExpectedResult(int n, int expected) + { + var count = BitUtility.ByteCount(n); + Assert.Equal(expected, count); + } + } + + public class CountBits + { + [Theory] + [InlineData(new byte[] { 0b00000000 }, 0)] + [InlineData(new byte[] { 0b00000001 }, 1)] + [InlineData(new byte[] { 0b11111111 }, 8)] + [InlineData(new byte[] { 0b01001001, 0b01010010 }, 6)] + public void CountsAllOneBits(byte[] data, int expectedCount) + { + Assert.Equal(expectedCount, + BitUtility.CountBits(data)); + } + + [Theory] + [InlineData(new byte[] { 0b11111111 }, 0, 8)] + [InlineData(new byte[] { 0b11111111 }, 3, 5)] + [InlineData(new byte[] { 0b11111111, 0b11111111 }, 9, 7)] + [InlineData(new byte[] { 0b11111111 }, -1, 0)] + public void CountsAllOneBitsFromAnOffset(byte[] data, int offset, int expectedCount) + { + Assert.Equal(expectedCount, + BitUtility.CountBits(data, offset)); + } + + [Theory] + [InlineData(new byte[] { 0b11111111 }, 0, 8, 8)] + [InlineData(new byte[] { 0b11111111 }, 0, 4, 4)] + [InlineData(new byte[] { 0b11111111 }, 3, 2, 2)] + [InlineData(new byte[] { 0b11111111 }, 3, 5, 5)] + [InlineData(new byte[] { 0b11111111, 0b11111111 }, 9, 7, 7)] + [InlineData(new byte[] { 0b11111111, 0b11111111 }, 7, 2, 2)] + [InlineData(new byte[] { 0b11111111, 0b11111111, 0b11111111 }, 0, 24, 24)] + [InlineData(new byte[] { 0b11111111, 0b11111111, 0b11111111 }, 8, 16, 16)] + [InlineData(new byte[] { 0b11111111, 0b11111111, 0b11111111 }, 0, 16, 16)] + [InlineData(new byte[] { 0b11111111, 0b11111111, 0b11111111 }, 3, 18, 18)] + [InlineData(new byte[] { 0b11111111 }, -1, 0, 0)] + public void CountsAllOneBitsFromOffsetWithinLength(byte[] data, int offset, int length, int expectedCount) + { + var actualCount = BitUtility.CountBits(data, offset, length); + Assert.Equal(expectedCount, actualCount); + } + + [Fact] + public void CountsZeroBitsWhenDataIsEmpty() + { + Assert.Equal(0, + BitUtility.CountBits(null)); + } + } + + public class GetBit + { + [Theory] + [InlineData(new byte[] { 0b01001001 }, 0, true)] + [InlineData(new byte[] { 0b01001001 }, 1, false)] + [InlineData(new byte[] { 0b01001001 }, 2, false)] + [InlineData(new byte[] { 0b01001001 }, 3, true)] + [InlineData(new byte[] { 0b01001001 }, 4, false)] + [InlineData(new byte[] { 0b01001001 }, 5, false)] + [InlineData(new byte[] { 0b01001001 }, 6, true)] + [InlineData(new byte[] { 0b01001001 }, 7, false)] + [InlineData(new byte[] { 0b01001001, 0b01010010 }, 8, false)] + [InlineData(new byte[] { 0b01001001, 0b01010010 }, 14, true)] + public void GetsCorrectBitForIndex(byte[] data, int index, bool expectedValue) + { + Assert.Equal(expectedValue, + BitUtility.GetBit(data, index)); + } + + [Theory] + [InlineData(null, 0)] + [InlineData(new byte[] { 0b00000000 }, -1)] + public void ThrowsWhenBitIndexOutOfRange(byte[] data, int index) + { + Assert.Throws<IndexOutOfRangeException>(() => + BitUtility.GetBit(data, index)); + } + } + + public class SetBit + { + [Theory] + [InlineData(new byte[] { 0b00000000 }, 0, new byte[] { 0b00000001 })] + [InlineData(new byte[] { 0b00000000 }, 2, new byte[] { 0b00000100 })] + [InlineData(new byte[] { 0b00000000 }, 7, new byte[] { 0b10000000 })] + [InlineData(new byte[] { 0b00000000, 0b00000000 }, 8, new byte[] { 0b00000000, 0b00000001 })] + [InlineData(new byte[] { 0b00000000, 0b00000000 }, 15, new byte[] { 0b00000000, 0b10000000 })] + public void SetsBitAtIndex(byte[] data, int index, byte[] expectedValue) + { + BitUtility.SetBit(data, index); + Assert.Equal(expectedValue, data); + } + } + + public class ClearBit + { + [Theory] + [InlineData(new byte[] { 0b00000001 }, 0, new byte[] { 0b00000000 })] + [InlineData(new byte[] { 0b00000010 }, 1, new byte[] { 0b00000000 })] + [InlineData(new byte[] { 0b10000001 }, 7, new byte[] { 0b00000001 })] + [InlineData(new byte[] { 0b11111111, 0b11111111 }, 15, new byte[] { 0b11111111, 0b01111111 })] + public void ClearsBitAtIndex(byte[] data, int index, byte[] expectedValue) + { + BitUtility.ClearBit(data, index); + Assert.Equal(expectedValue, data); + } + } + + public class RoundUpToMultipleOf64 + { + [Theory] + [InlineData(0, 0)] + [InlineData(1, 64)] + [InlineData(63, 64)] + [InlineData(64, 64)] + [InlineData(65, 128)] + [InlineData(129, 192)] + public void ReturnsNextMultiple(int size, int expectedSize) + { + Assert.Equal(expectedSize, + BitUtility.RoundUpToMultipleOf64(size)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void ReturnsZeroWhenSizeIsLessThanOrEqualToZero(int size) + { + Assert.Equal(0, + BitUtility.RoundUpToMultipleOf64(size)); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/BooleanArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/BooleanArrayTests.cs new file mode 100644 index 000000000..efac07dba --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/BooleanArrayTests.cs @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class BooleanArrayTests + { + public class Builder + { + public class Append + { + [Theory] + [InlineData(1)] + [InlineData(3)] + public void IncrementsLength(int count) + { + var builder = new BooleanArray.Builder(); + + for (var i = 0; i < count; i++) + { + builder.Append(true); + } + + var array = builder.Build(); + + Assert.Equal(count, array.Length); + } + + [Fact] + public void AppendsExpectedBit() + { + var array1 = new BooleanArray.Builder() + .Append(false) + .Build(); + + Assert.False(array1.GetValue(0).Value); + + var array2 = new BooleanArray.Builder() + .Append(true) + .Build(); + + Assert.True(array2.GetValue(0).Value); + } + } + + public class Clear + { + [Fact] + public void SetsAllBitsToDefault() + { + var array = new BooleanArray.Builder() + .Resize(8) + .Set(0, true) + .Set(7, true) + .Clear() + .Build(); + + for (var i = 0; i < array.Length; i++) + { + Assert.False(array.GetValue(i).Value); + } + } + } + + public class Toggle + { + [Theory] + [InlineData(8, 1)] + [InlineData(16, 13)] + public void TogglesExpectedBitToFalse(int length, int index) + { + var array = new BooleanArray.Builder() + .Resize(length) + .Set(index, true) + .Toggle(index) + .Build(); + + Assert.False(array.GetValue(index).Value); + } + + [Theory] + [InlineData(8, 1)] + [InlineData(16, 13)] + public void TogglesExpectedBitToTreu(int length, int index) + { + var array = new BooleanArray.Builder() + .Resize(length) + .Set(index, false) + .Toggle(index) + .Build(); + + Assert.True(array.GetValue(index).Value); + } + + [Fact] + public void ThrowsWhenIndexOutOfRange() + { + Assert.Throws<ArgumentOutOfRangeException>(() => + { + var builder = new BooleanArray.Builder(); + builder.Toggle(8); + }); + } + } + + public class Swap + { + [Fact] + public void SwapsExpectedBits() + { + var array = new BooleanArray.Builder() + .AppendRange(Enumerable.Repeat(false, 8)) + .Set(0, true) + .Swap(0, 7) + .Build(); + + Assert.True(array.GetValue(0).HasValue); + Assert.False(array.GetValue(0).Value); + Assert.True(array.GetValue(7).HasValue); + Assert.True(array.GetValue(7).Value); + #pragma warning disable CS0618 + Assert.False(array.GetBoolean(0)); + Assert.True(array.GetBoolean(7)); + #pragma warning restore CS0618 + } + + [Fact] + public void ThrowsWhenIndexOutOfRange() + { + Assert.Throws<ArgumentOutOfRangeException>(() => + { + var builder = new BooleanArray.Builder(); + builder.Swap(0, 1); + }); + } + } + + public class Set + { + [Theory] + [InlineData(8, 0)] + [InlineData(8, 4)] + [InlineData(8, 7)] + [InlineData(16, 8)] + [InlineData(16, 15)] + public void SetsExpectedBitToTrue(int length, int index) + { + var array = new BooleanArray.Builder() + .Resize(length) + .Set(index, true) + .Build(); + + Assert.True(array.GetValue(index).Value); + } + + [Theory] + [InlineData(8, 0)] + [InlineData(8, 4)] + [InlineData(8, 7)] + [InlineData(16, 8)] + [InlineData(16, 15)] + public void SetsExpectedBitsToFalse(int length, int index) + { + var array = new BooleanArray.Builder() + .Resize(length) + .Set(index, false) + .Build(); + + Assert.False(array.GetValue(index).Value); + } + + [Theory] + [InlineData(4)] + public void UnsetBitsAreUnchanged(int index) + { + var array = new BooleanArray.Builder() + .AppendRange(Enumerable.Repeat(false, 8)) + .Set(index, true) + .Build(); + + for (var i = 0; i < 8; i++) + { + if (i != index) + { + Assert.True(array.GetValue(i).HasValue); + Assert.False(array.GetValue(i).Value); + #pragma warning disable CS0618 + Assert.False(array.GetBoolean(i)); + #pragma warning restore CS0618 + } + } + } + + [Fact] + public void ThrowsWhenIndexOutOfRange() + { + Assert.Throws<ArgumentOutOfRangeException>(() => + { + var builder = new BooleanArray.Builder(); + builder.Set(builder.Length, false); + }); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/ColumnTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/ColumnTests.cs new file mode 100644 index 000000000..b90c68162 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/ColumnTests.cs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Linq; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class ColumnTests + { + public static Array MakeIntArray(int length) + { + // The following should be improved once the ArrayBuilder PR goes in + var intBuilder = new ArrowBuffer.Builder<int>(); + intBuilder.AppendRange(Enumerable.Range(0, length).Select(x => x)); + ArrowBuffer buffer = intBuilder.Build(); + ArrayData intData = new ArrayData(Int32Type.Default, length, 0, 0, new[] { ArrowBuffer.Empty, buffer }); + Array intArray = ArrowArrayFactory.BuildArray(intData) as Array; + return intArray; + } + + [Fact] + public void TestColumn() + { + Array intArray = MakeIntArray(10); + Array intArrayCopy = MakeIntArray(10); + + Field field = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Column column = new Column(field, new[] { intArray, intArrayCopy }); + + Assert.True(column.Name == field.Name); + Assert.True(column.Field == field); + Assert.Equal(20, column.Length); + Assert.Equal(0, column.NullCount); + Assert.Equal(field.DataType, column.Type); + + Column slice5 = column.Slice(0, 5); + Assert.Equal(5, slice5.Length); + Column sliceFull = column.Slice(2); + Assert.Equal(18, sliceFull.Length); + Column sliceMore = column.Slice(0, 25); + Assert.Equal(20, sliceMore.Length); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs new file mode 100644 index 000000000..0d6aad96e --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Date32ArrayTests.cs @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class Date32ArrayTests + { + public static IEnumerable<object[]> GetDatesData() => + TestDateAndTimeData.ExampleDates.Select(d => new object[] { d }); + + public static IEnumerable<object[]> GetDateTimesData() => + TestDateAndTimeData.ExampleDateTimes.Select(dt => new object[] { dt }); + + public static IEnumerable<object[]> GetDateTimeOffsetsData() => + TestDateAndTimeData.ExampleDateTimeOffsets.Select(dto => new object[] { dto }); + + public class AppendNull + { + [Fact] + public void AppendThenGetGivesNull() + { + // Arrange + var builder = new Date32Array.Builder(); + + // Act + builder = builder.AppendNull(); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Null(array.GetDateTime(0)); + Assert.Null(array.GetDateTimeOffset(0)); + Assert.Null(array.GetValue(0)); + } + } + + public class AppendDateTime + { + [Theory] + [MemberData(nameof(GetDatesData), MemberType = typeof(Date32ArrayTests))] + public void AppendDateGivesSameDate(DateTime date) + { + // Arrange + var builder = new Date32Array.Builder(); + var expectedDateTime = date; + var expectedDateTimeOffset = + new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Unspecified), TimeSpan.Zero); + int expectedValue = (int)date.Subtract(new DateTime(1970, 1, 1)).TotalDays; + + // Act + builder = builder.Append(date); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + } + + [Theory] + [MemberData(nameof(GetDateTimesData), MemberType = typeof(Date32ArrayTests))] + public void AppendWithTimeGivesSameWithTimeIgnored(DateTime dateTime) + { + // Arrange + var builder = new Date32Array.Builder(); + var expectedDateTime = dateTime.Date; + var expectedDateTimeOffset = + new DateTimeOffset(DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified), TimeSpan.Zero); + int expectedValue = (int)dateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays; + + // Act + builder = builder.Append(dateTime); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + } + } + + public class AppendDateTimeOffset + { + [Theory] + [MemberData(nameof(GetDateTimeOffsetsData), MemberType = typeof(Date32ArrayTests))] + public void AppendGivesUtcDate(DateTimeOffset dateTimeOffset) + { + // Arrange + var builder = new Date32Array.Builder(); + var expectedDateTime = dateTimeOffset.UtcDateTime.Date; + var expectedDateTimeOffset = new DateTimeOffset(dateTimeOffset.UtcDateTime.Date, TimeSpan.Zero); + int expectedValue = (int)dateTimeOffset.UtcDateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays; + + // Act + builder = builder.Append(dateTimeOffset); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs new file mode 100644 index 000000000..65cffc84e --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Date64ArrayTests.cs @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class Date64ArrayTests + { + private const long MillisecondsPerDay = 86400000; + + public static IEnumerable<object[]> GetDatesData() => + TestDateAndTimeData.ExampleDates.Select(d => new object[] { d }); + + public static IEnumerable<object[]> GetDateTimesData() => + TestDateAndTimeData.ExampleDateTimes.Select(dt => new object[] { dt }); + + public static IEnumerable<object[]> GetDateTimeOffsetsData() => + TestDateAndTimeData.ExampleDateTimeOffsets.Select(dto => new object[] { dto }); + + public class AppendNull + { + [Fact] + public void AppendThenGetGivesNull() + { + // Arrange + var builder = new Date64Array.Builder(); + + // Act + builder = builder.AppendNull(); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Null(array.GetDateTime(0)); + Assert.Null(array.GetDateTimeOffset(0)); + Assert.Null(array.GetValue(0)); + } + } + + public class AppendDateTime + { + [Theory] + [MemberData(nameof(GetDatesData), MemberType = typeof(Date64ArrayTests))] + public void AppendDateGivesSameDate(DateTime date) + { + // Arrange + var builder = new Date64Array.Builder(); + var expectedDateTime = date; + var expectedDateTimeOffset = + new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Unspecified), TimeSpan.Zero); + long expectedValue = (long)date.Subtract(new DateTime(1970, 1, 1)).TotalDays * MillisecondsPerDay; + + // Act + builder = builder.Append(date); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay); + } + + [Theory] + [MemberData(nameof(GetDateTimesData), MemberType = typeof(Date64ArrayTests))] + public void AppendWithTimeGivesSameWithTimeIgnored(DateTime dateTime) + { + // Arrange + var builder = new Date64Array.Builder(); + var expectedDateTime = dateTime.Date; + var expectedDateTimeOffset = + new DateTimeOffset(DateTime.SpecifyKind(dateTime.Date, DateTimeKind.Unspecified), TimeSpan.Zero); + long expectedValue = + (long)dateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays * MillisecondsPerDay; + + // Act + builder = builder.Append(dateTime); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay); + } + } + + public class AppendDateTimeOffset + { + [Theory] + [MemberData(nameof(GetDateTimeOffsetsData), MemberType = typeof(Date64ArrayTests))] + public void AppendGivesUtcDate(DateTimeOffset dateTimeOffset) + { + // Arrange + var builder = new Date64Array.Builder(); + var expectedDateTime = dateTimeOffset.UtcDateTime.Date; + var expectedDateTimeOffset = new DateTimeOffset(dateTimeOffset.UtcDateTime.Date, TimeSpan.Zero); + long expectedValue = + (long)dateTimeOffset.UtcDateTime.Date.Subtract(new DateTime(1970, 1, 1)).TotalDays * + MillisecondsPerDay; + + // Act + builder = builder.Append(dateTimeOffset); + + // Assert + var array = builder.Build(); + Assert.Equal(1, array.Length); + Assert.Equal(expectedDateTime, array.GetDateTime(0)); + Assert.Equal(expectedDateTimeOffset, array.GetDateTimeOffset(0)); + Assert.Equal(expectedValue, array.GetValue(0)); + Assert.Equal(0, array.GetValue(0).Value % MillisecondsPerDay); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal128ArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal128ArrayTests.cs new file mode 100644 index 000000000..68f8ee02b --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal128ArrayTests.cs @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class Decimal128ArrayTests + { + public class Builder + { + public class AppendNull + { + [Fact] + public void AppendThenGetGivesNull() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(8,2)); + + // Act + + builder = builder.AppendNull(); + builder = builder.AppendNull(); + builder = builder.AppendNull(); + // Assert + var array = builder.Build(); + + Assert.Equal(3, array.Length); + Assert.Equal(array.Data.Buffers[1].Length, array.ByteWidth * 3); + Assert.Null(array.GetValue(0)); + Assert.Null(array.GetValue(1)); + Assert.Null(array.GetValue(2)); + } + } + + public class Append + { + [Theory] + [InlineData(200)] + public void AppendDecimal(int count) + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(14, 10)); + + // Act + decimal?[] testData = new decimal?[count]; + for (int i = 0; i < count; i++) + { + if (i == count - 2) + { + builder.AppendNull(); + testData[i] = null; + continue; + } + decimal rnd = i * (decimal)Math.Round(new Random().NextDouble(),10); + testData[i] = rnd; + builder.Append(rnd); + } + + // Assert + var array = builder.Build(); + Assert.Equal(count, array.Length); + for (int i = 0; i < count; i++) + { + Assert.Equal(testData[i], array.GetValue(i)); + } + } + + [Fact] + public void AppendLargeDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(26, 2)); + decimal large = 999999999999909999999999.80M; + // Act + builder.Append(large); + builder.Append(-large); + + // Assert + var array = builder.Build(); + Assert.Equal(large, array.GetValue(0)); + Assert.Equal(-large, array.GetValue(1)); + } + + [Fact] + public void AppendFractionalDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(26, 20)); + decimal fraction = 0.99999999999990999992M; + // Act + builder.Append(fraction); + builder.Append(-fraction); + + // Assert + var array = builder.Build(); + Assert.Equal(fraction, array.GetValue(0)); + Assert.Equal(-fraction, array.GetValue(1)); + } + + [Fact] + public void AppendRangeDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)); + var range = new decimal[] {2.123M, 1.5984M, -0.0000001M, 9878987987987987.1235407M}; + + // Act + builder.AppendRange(range); + builder.AppendNull(); + + // Assert + var array = builder.Build(); + for(int i = 0; i < range.Length; i ++) + { + Assert.Equal(range[i], array.GetValue(i)); + } + + Assert.Null( array.GetValue(range.Length)); + } + + [Fact] + public void AppendClearAppendDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)); + + // Act + builder.Append(1); + builder.Clear(); + builder.Append(10); + + // Assert + var array = builder.Build(); + Assert.Equal(10, array.GetValue(0)); + } + + [Fact] + public void AppendInvalidPrecisionAndScaleDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(2, 1)); + + // Assert + Assert.Throws<OverflowException>(() => builder.Append(100)); + Assert.Throws<OverflowException>(() => builder.Append(0.01M)); + builder.Append(-9.9M); + builder.Append(0); + builder.Append(9.9M); + } + } + + public class Set + { + [Fact] + public void SetDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)) + .Resize(1); + + // Act + builder.Set(0, 50.123456M); + builder.Set(0, 1.01M); + + // Assert + var array = builder.Build(); + Assert.Equal(1.01M, array.GetValue(0)); + } + + [Fact] + public void SetNull() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)) + .Resize(1); + + // Act + builder.Set(0, 50.123456M); + builder.SetNull(0); + + // Assert + var array = builder.Build(); + Assert.Null(array.GetValue(0)); + } + } + + public class Swap + { + [Fact] + public void SetDecimal() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)); + + // Act + builder.Append(123.45M); + builder.Append(678.9M); + builder.Swap(0, 1); + + // Assert + var array = builder.Build(); + Assert.Equal(678.9M, array.GetValue(0)); + Assert.Equal(123.45M, array.GetValue(1)); + } + + [Fact] + public void SwapNull() + { + // Arrange + var builder = new Decimal128Array.Builder(new Decimal128Type(24, 8)); + + // Act + builder.Append(123.456M); + builder.AppendNull(); + builder.Swap(0, 1); + + // Assert + var array = builder.Build(); + Assert.Null(array.GetValue(0)); + Assert.Equal(123.456M, array.GetValue(1)); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal256ArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal256ArrayTests.cs new file mode 100644 index 000000000..35b68823d --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Decimal256ArrayTests.cs @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class Decimal256ArrayTests + { + public class Builder + { + public class AppendNull + { + [Fact] + public void AppendThenGetGivesNull() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(8,2)); + + // Act + + builder = builder.AppendNull(); + builder = builder.AppendNull(); + builder = builder.AppendNull(); + // Assert + var array = builder.Build(); + + Assert.Equal(3, array.Length); + Assert.Equal(array.Data.Buffers[1].Length, array.ByteWidth * 3); + Assert.Null(array.GetValue(0)); + Assert.Null(array.GetValue(1)); + Assert.Null(array.GetValue(2)); + } + } + + public class Append + { + [Theory] + [InlineData(200)] + public void AppendDecimal(int count) + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(14, 10)); + + // Act + decimal?[] testData = new decimal?[count]; + for (int i = 0; i < count; i++) + { + if (i == count - 2) + { + builder.AppendNull(); + testData[i] = null; + continue; + } + decimal rnd = i * (decimal)Math.Round(new Random().NextDouble(),10); + testData[i] = rnd; + builder.Append(rnd); + } + + // Assert + var array = builder.Build(); + Assert.Equal(count, array.Length); + for (int i = 0; i < count; i++) + { + Assert.Equal(testData[i], array.GetValue(i)); + } + } + + [Fact] + public void AppendLargeDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(26, 2)); + decimal large = 999999999999909999999999.80M; + // Act + builder.Append(large); + builder.Append(-large); + + // Assert + var array = builder.Build(); + Assert.Equal(large, array.GetValue(0)); + Assert.Equal(-large, array.GetValue(1)); + } + + [Fact] + public void AppendFractionalDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(26, 20)); + decimal fraction = 0.99999999999990999992M; + // Act + builder.Append(fraction); + builder.Append(-fraction); + + // Assert + var array = builder.Build(); + Assert.Equal(fraction, array.GetValue(0)); + Assert.Equal(-fraction, array.GetValue(1)); + } + + [Fact] + public void AppendRangeDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)); + var range = new decimal[] {2.123M, 1.5984M, -0.0000001M, 9878987987987987.1235407M}; + + // Act + builder.AppendRange(range); + builder.AppendNull(); + + // Assert + var array = builder.Build(); + for(int i = 0; i < range.Length; i ++) + { + Assert.Equal(range[i], array.GetValue(i)); + } + + Assert.Null( array.GetValue(range.Length)); + } + + [Fact] + public void AppendClearAppendDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)); + + // Act + builder.Append(1); + builder.Clear(); + builder.Append(10); + + // Assert + var array = builder.Build(); + Assert.Equal(10, array.GetValue(0)); + } + + [Fact] + public void AppendInvalidPrecisionAndScaleDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(2, 1)); + + // Assert + Assert.Throws<OverflowException>(() => builder.Append(100)); + Assert.Throws<OverflowException>(() => builder.Append(0.01M)); + builder.Append(-9.9M); + builder.Append(0); + builder.Append(9.9M); + } + } + + public class Set + { + [Fact] + public void SetDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)) + .Resize(1); + + // Act + builder.Set(0, 50.123456M); + builder.Set(0, 1.01M); + + // Assert + var array = builder.Build(); + Assert.Equal(1.01M, array.GetValue(0)); + } + + [Fact] + public void SetNull() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)) + .Resize(1); + + // Act + builder.Set(0, 50.123456M); + builder.SetNull(0); + + // Assert + var array = builder.Build(); + Assert.Null(array.GetValue(0)); + } + } + + public class Swap + { + [Fact] + public void SetDecimal() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)); + + // Act + builder.Append(123.45M); + builder.Append(678.9M); + builder.Swap(0, 1); + + // Assert + var array = builder.Build(); + Assert.Equal(678.9M, array.GetValue(0)); + Assert.Equal(123.45M, array.GetValue(1)); + } + + [Fact] + public void SwapNull() + { + // Arrange + var builder = new Decimal256Array.Builder(new Decimal256Type(24, 8)); + + // Act + builder.Append(123.456M); + builder.AppendNull(); + builder.Swap(0, 1); + + // Assert + var array = builder.Build(); + Assert.Null(array.GetValue(0)); + Assert.Equal(123.456M, array.GetValue(1)); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/DecimalUtilityTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/DecimalUtilityTests.cs new file mode 100644 index 000000000..d235524d9 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/DecimalUtilityTests.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class DecimalUtilityTests + { + public class Overflow + { + [Theory] + [InlineData(100.123, 10, 4, false)] + [InlineData(100.123, 6, 4, false)] + [InlineData(100.123, 3, 3, true)] + [InlineData(100.123, 10, 2, true)] + [InlineData(100.123, 5, 2, true)] + [InlineData(100.123, 5, 3, true)] + [InlineData(100.123, 6, 3, false)] + public void HasExpectedResultOrThrows(decimal d, int precision , int scale, bool shouldThrow) + { + var builder = new Decimal128Array.Builder(new Decimal128Type(precision, scale)); + + if (shouldThrow) + { + Assert.Throws<OverflowException>(() => builder.Append(d)); + } + else + { + builder.Append(d); + var result = builder.Build(new TestMemoryAllocator()); + Assert.Equal(d, result.GetValue(0)); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/DictionaryArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/DictionaryArrayTests.cs new file mode 100644 index 000000000..da678563c --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/DictionaryArrayTests.cs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class DictionaryArrayTests + { + [Fact] + public void CreateTest() + { + (StringArray originalDictionary, Int32Array originalIndicesArray, DictionaryArray dictionaryArray) = + CreateSimpleTestData(); + + Assert.Equal(dictionaryArray.Dictionary, originalDictionary); + Assert.Equal(dictionaryArray.Indices, originalIndicesArray); + } + + [Fact] + public void SliceTest() + { + (StringArray originalDictionary, Int32Array originalIndicesArray, DictionaryArray dictionaryArray) = + CreateSimpleTestData(); + + int batchLength = originalIndicesArray.Length; + for (int offset = 0; offset < batchLength; offset++) + { + for (int length = 1; offset + length <= batchLength; length++) + { + var sliced = dictionaryArray.Slice(offset, length) as DictionaryArray; + var actualSlicedDictionary = sliced.Dictionary as StringArray; + var actualSlicedIndicesArray = sliced.Indices as Int32Array; + + var expectedSlicedIndicesArray = originalIndicesArray.Slice(offset, length) as Int32Array; + + //Dictionary is not sliced. + Assert.Equal(originalDictionary.Data, actualSlicedDictionary.Data); + Assert.Equal(expectedSlicedIndicesArray.ToList(), actualSlicedIndicesArray.ToList()); + } + } + } + + private Tuple<StringArray, Int32Array, DictionaryArray> CreateSimpleTestData() + { + StringArray originalDictionary = new StringArray.Builder().AppendRange(new[] { "a", "b", "c" }).Build(); + Int32Array originalIndicesArray = new Int32Array.Builder().AppendRange(new[] { 0, 0, 1, 1, 2, 2 }).Build(); + var dictionaryArray = new DictionaryArray(new DictionaryType(Int32Type.Default, StringType.Default, false), originalIndicesArray, originalDictionary); + + return Tuple.Create(originalDictionary, originalIndicesArray, dictionaryArray); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Extensions/DateTimeOffsetExtensions.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Extensions/DateTimeOffsetExtensions.cs new file mode 100644 index 000000000..4375c39cd --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Extensions/DateTimeOffsetExtensions.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Apache.Arrow.Tests +{ + public static class DateTimeOffsetExtensions + { + public static DateTimeOffset Truncate(this DateTimeOffset dateTimeOffset, TimeSpan offset) + { + if (offset == TimeSpan.Zero) + { + return dateTimeOffset; + } + + if (dateTimeOffset == DateTimeOffset.MinValue || dateTimeOffset == DateTimeOffset.MaxValue) + { + return dateTimeOffset; + } + + return dateTimeOffset.AddTicks(-(dateTimeOffset.Ticks % offset.Ticks)); + } + + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/FieldComparer.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/FieldComparer.cs new file mode 100644 index 000000000..d7dcc398f --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/FieldComparer.cs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public static class FieldComparer + { + public static void Compare(Field expected, Field actual) + { + if (ReferenceEquals(expected, actual)) + { + return; + } + + Assert.Equal(expected.Name, actual.Name); + Assert.Equal(expected.IsNullable, actual.IsNullable); + + Assert.Equal(expected.HasMetadata, actual.HasMetadata); + if (expected.HasMetadata) + { + Assert.Equal(expected.Metadata.Keys.Count(), actual.Metadata.Keys.Count()); + Assert.True(expected.Metadata.Keys.All(k => actual.Metadata.ContainsKey(k) && expected.Metadata[k] == actual.Metadata[k])); + Assert.True(actual.Metadata.Keys.All(k => expected.Metadata.ContainsKey(k) && actual.Metadata[k] == expected.Metadata[k])); + } + + actual.DataType.Accept(new ArrayTypeComparer(expected.DataType)); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/Fixtures/DefaultMemoryAllocatorFixture.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/Fixtures/DefaultMemoryAllocatorFixture.cs new file mode 100644 index 000000000..276caf1ba --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/Fixtures/DefaultMemoryAllocatorFixture.cs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; + +namespace Apache.Arrow.Tests.Fixtures +{ + public class DefaultMemoryAllocatorFixture + { + public MemoryAllocator MemoryAllocator { get; } + + public DefaultMemoryAllocatorFixture() + { + const int alignment = 64; + + MemoryAllocator = new NativeMemoryAllocator(alignment); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs new file mode 100644 index 000000000..6ddbcd204 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaBuilderTests.cs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Sdk; + +namespace Apache.Arrow.Tests +{ + public class SchemaBuilderTests + { + public class Build + { + [Fact] + public void FieldsAreNullableByDefault() + { + var b = new Schema.Builder(); + + var schema = new Schema.Builder() + .Field(f => f.Name("f0").DataType(Int32Type.Default)) + .Build(); + + Assert.True(schema.Fields["f0"].IsNullable); + } + + [Fact] + public void FieldsHaveNullTypeByDefault() + { + var schema = new Schema.Builder() + .Field(f => f.Name("f0")) + .Build(); + + Assert.True(schema.Fields["f0"].DataType.GetType() == typeof(NullType)); + } + + [Fact] + public void FieldNameIsRequired() + { + Assert.Throws<ArgumentNullException>(() => + { + var schema = new Schema.Builder() + .Field(f => f.DataType(Int32Type.Default)) + .Build(); + }); + } + + [Fact] + public void GetFieldIndex() + { + var schema = new Schema.Builder() + .Field(f => f.Name("f0").DataType(Int32Type.Default)) + .Field(f => f.Name("f1").DataType(Int8Type.Default)) + .Build(); + Assert.True(schema.GetFieldIndex("f0") == 0 && schema.GetFieldIndex("f1") == 1); + } + + + [Fact] + public void GetFieldByName() + { + Field f0 = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f1 = new Field.Builder().Name("f1").DataType(Int8Type.Default).Build(); + + var schema = new Schema.Builder() + .Field(f0) + .Field(f1) + .Build(); + Assert.True(schema.GetFieldByName("f0") == f0 && schema.GetFieldByName("f1") == f1); + } + + [Fact] + public void MetadataConstruction() + { + + var metadata0 = new Dictionary<string, string> { { "foo", "bar" }, { "bizz", "buzz" } }; + var metadata1 = new Dictionary<string, string> { { "foo", "bar" } }; + var metadata0Copy = new Dictionary<string, string>(metadata0); + var metadata1Copy = new Dictionary<string, string>(metadata1); + Field f0 = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f1 = new Field.Builder().Name("f1").DataType(UInt8Type.Default).Nullable(false).Build(); + Field f2 = new Field.Builder().Name("f2").DataType(StringType.Default).Build(); + Field f3 = new Field.Builder().Name("f2").DataType(StringType.Default).Metadata(metadata1Copy).Build(); + + var schema0 = new Schema.Builder() + .Field(f0) + .Field(f1) + .Field(f2) + .Metadata(metadata0) + .Build(); + var schema1 = new Schema.Builder() + .Field(f0) + .Field(f1) + .Field(f2) + .Metadata(metadata1) + .Build(); + var schema2 = new Schema.Builder() + .Field(f0) + .Field(f1) + .Field(f2) + .Metadata(metadata0Copy) + .Build(); + var schema3 = new Schema.Builder() + .Field(f0) + .Field(f1) + .Field(f3) + .Metadata(metadata0Copy) + .Build(); + + Assert.True(metadata0.Keys.SequenceEqual(schema0.Metadata.Keys) && metadata0.Values.SequenceEqual(schema0.Metadata.Values)); + Assert.True(metadata1.Keys.SequenceEqual(schema1.Metadata.Keys) && metadata1.Values.SequenceEqual(schema1.Metadata.Values)); + Assert.True(metadata0.Keys.SequenceEqual(schema2.Metadata.Keys) && metadata0.Values.SequenceEqual(schema2.Metadata.Values)); + SchemaComparer.Compare(schema0, schema2); + Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema0, schema1)); + Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema2, schema1)); + Assert.Throws<EqualException>(() => SchemaComparer.Compare(schema2, schema3)); + } + + [Theory] + [MemberData(nameof(SampleSchema1))] + public void FieldsHaveExpectedValues(string name, IArrowType type, bool nullable) + { + var schema = new Schema.Builder() + .Field(f => f.Name(name).DataType(type).Nullable(nullable)) + .Build(); + + var field = schema.Fields[name]; + + Assert.Equal(name, field.Name); + Assert.Equal(type.Name, field.DataType.Name); + Assert.Equal(nullable, field.IsNullable); + } + + public static IEnumerable<object[]> SampleSchema1() + { + yield return new object[] {"f0", Int32Type.Default, true}; + yield return new object[] {"f1", DoubleType.Default, true}; + yield return new object[] {"f2", Int64Type.Default, false}; + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs new file mode 100644 index 000000000..3546d5e0c --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/SchemaComparer.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Linq; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public static class SchemaComparer + { + public static void Compare(Schema expected, Schema actual) + { + if (ReferenceEquals(expected, actual)) + { + return; + } + + Assert.Equal(expected.HasMetadata, actual.HasMetadata); + if (expected.HasMetadata) + { + Assert.Equal(expected.Metadata.Keys.Count(), actual.Metadata.Keys.Count()); + Assert.True(expected.Metadata.Keys.All(k => actual.Metadata.ContainsKey(k) && expected.Metadata[k] == actual.Metadata[k])); + Assert.True(actual.Metadata.Keys.All(k => expected.Metadata.ContainsKey(k) && actual.Metadata[k] == expected.Metadata[k])); + } + + Assert.Equal(expected.Fields.Count, actual.Fields.Count); + Assert.True(expected.Fields.Keys.All(k => actual.Fields.ContainsKey(k))); + foreach (string name in expected.Fields.Keys) + { + FieldComparer.Compare(expected.Fields[name], actual.Fields[name]); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs new file mode 100644 index 000000000..e2d0fa851 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/StructArrayTests.cs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using System.Collections.Generic; +using System.IO; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class StructArrayTests + { + [Fact] + public void TestStructArray() + { + // The following can be improved with a Builder class for StructArray. + List<Field> fields = new List<Field>(); + Field.Builder fieldBuilder = new Field.Builder(); + fields.Add(fieldBuilder.Name("Strings").DataType(StringType.Default).Nullable(true).Build()); + fieldBuilder = new Field.Builder(); + fields.Add(fieldBuilder.Name("Ints").DataType(Int32Type.Default).Nullable(true).Build()); + StructType structType = new StructType(fields); + + StringArray.Builder stringBuilder = new StringArray.Builder(); + StringArray stringArray = stringBuilder.Append("joe").AppendNull().AppendNull().Append("mark").Build(); + Int32Array.Builder intBuilder = new Int32Array.Builder(); + Int32Array intArray = intBuilder.Append(1).Append(2).AppendNull().Append(4).Build(); + List<Array> arrays = new List<Array>(); + arrays.Add(stringArray); + arrays.Add(intArray); + + ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder(); + var nullBitmapBuffer = nullBitmap.Append(true).Append(true).Append(false).Append(true).Build(); + StructArray structs = new StructArray(structType, 4, arrays, nullBitmapBuffer, 1); + + Assert.Equal(4, structs.Length); + Assert.Equal(1, structs.NullCount); + ArrayData[] childArrays = structs.Data.Children; // Data for StringArray and Int32Array + Assert.Equal(2, childArrays.Length); + for (int i = 0; i < childArrays.Length; i++) + { + ArrayData arrayData = childArrays[i]; + Assert.Null(arrayData.Children); + if (i == 0) + { + Assert.Equal(ArrowTypeId.String, arrayData.DataType.TypeId); + Array array = new StringArray(arrayData); + StringArray structStringArray = array as StringArray; + Assert.NotNull(structStringArray); + Assert.Equal(structs.Length, structStringArray.Length); + Assert.Equal(stringArray.Length, structStringArray.Length); + Assert.Equal(stringArray.NullCount, structStringArray.NullCount); + for (int j = 0; j < stringArray.Length; j++) + { + Assert.Equal(stringArray.GetString(j), structStringArray.GetString(j)); + } + } + if (i == 1) + { + Assert.Equal(ArrowTypeId.Int32, arrayData.DataType.TypeId); + Array array = new Int32Array(arrayData); + Int32Array structIntArray = array as Int32Array; + Assert.NotNull(structIntArray); + Assert.Equal(structs.Length, structIntArray.Length); + Assert.Equal(intArray.Length, structIntArray.Length); + Assert.Equal(intArray.NullCount, structIntArray.NullCount); + for (int j = 0; j < intArray.Length; j++) + { + Assert.Equal(intArray.GetValue(j), structIntArray.GetValue(j)); + } + } + } + } + + [Fact] + public void TestListOfStructArray() + { + Schema.Builder builder = new Schema.Builder(); + Field structField = new Field( + "struct", + new StructType( + new[] + { + new Field("name", StringType.Default, nullable: false), + new Field("age", Int64Type.Default, nullable: false), + }), + nullable: false); + + Field listField = new Field("listOfStructs", new ListType(structField), nullable: false); + builder.Field(listField); + Schema schema = builder.Build(); + + StringArray stringArray = new StringArray.Builder() + .Append("joe").AppendNull().AppendNull().Append("mark").Append("abe").Append("phil").Build(); + Int64Array intArray = new Int64Array.Builder() + .Append(1).Append(2).AppendNull().Append(4).Append(10).Append(55).Build(); + + ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder() + .Append(true).Append(true).Append(false).Append(true).Append(true).Append(true).Build(); + + StructArray structs = new StructArray(structField.DataType, 6, new IArrowArray[] { stringArray, intArray }, nullBitmapBuffer, nullCount: 1); + + ArrowBuffer offsetsBuffer = new ArrowBuffer.Builder<int>() + .Append(0).Append(2).Append(5).Append(6).Build(); + ListArray listArray = new ListArray(listField.DataType, 3, offsetsBuffer, structs, ArrowBuffer.Empty); + + RecordBatch batch = new RecordBatch(schema, new[] { listArray }, 3); + TestRoundTripRecordBatch(batch); + } + + private static void TestRoundTripRecordBatch(RecordBatch originalBatch) + { + using (MemoryStream stream = new MemoryStream()) + { + using (var writer = new ArrowStreamWriter(stream, originalBatch.Schema, leaveOpen: true)) + { + writer.WriteRecordBatch(originalBatch); + writer.WriteEnd(); + } + + stream.Position = 0; + + using (var reader = new ArrowStreamReader(stream)) + { + RecordBatch newBatch = reader.ReadNextRecordBatch(); + ArrowReaderVerifier.CompareBatches(originalBatch, newBatch); + } + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/TableTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/TableTests.cs new file mode 100644 index 000000000..b919bf3b6 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/TableTests.cs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class TableTests + { + public static Table MakeTableWithOneColumnOfTwoIntArrays(int lengthOfEachArray) + { + Array intArray = ColumnTests.MakeIntArray(lengthOfEachArray); + Array intArrayCopy = ColumnTests.MakeIntArray(lengthOfEachArray); + + Field field = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Schema s0 = new Schema.Builder().Field(field).Build(); + + Column column = new Column(field, new List<Array> { intArray, intArrayCopy }); + Table table = new Table(s0, new List<Column> { column }); + return table; + } + + [Fact] + public void TestEmptyTable() + { + Table table = new Table(); + Assert.Equal(0, table.ColumnCount); + Assert.Equal(0, table.RowCount); + } + + [Fact] + public void TestTableBasics() + { + Table table = MakeTableWithOneColumnOfTwoIntArrays(10); + Assert.Equal(20, table.RowCount); + Assert.Equal(1, table.ColumnCount); + } + + [Fact] + public void TestTableAddRemoveAndSetColumn() + { + Table table = MakeTableWithOneColumnOfTwoIntArrays(10); + + Array nonEqualLengthIntArray = ColumnTests.MakeIntArray(10); + Field field1 = new Field.Builder().Name("f1").DataType(Int32Type.Default).Build(); + Column nonEqualLengthColumn = new Column(field1, new[] { nonEqualLengthIntArray}); + Assert.Throws<ArgumentException>(() => table.InsertColumn(-1, nonEqualLengthColumn)); + Assert.Throws<ArgumentException>(() => table.InsertColumn(1, nonEqualLengthColumn)); + + Array equalLengthIntArray = ColumnTests.MakeIntArray(20); + Field field2 = new Field.Builder().Name("f2").DataType(Int32Type.Default).Build(); + Column equalLengthColumn = new Column(field2, new[] { equalLengthIntArray}); + Column existingColumn = table.Column(0); + + Table newTable = table.InsertColumn(0, equalLengthColumn); + Assert.Equal(2, newTable.ColumnCount); + Assert.True(newTable.Column(0) == equalLengthColumn); + Assert.True(newTable.Column(1) == existingColumn); + + newTable = newTable.RemoveColumn(1); + Assert.Equal(1, newTable.ColumnCount); + Assert.True(newTable.Column(0) == equalLengthColumn); + + newTable = table.SetColumn(0, existingColumn); + Assert.True(newTable.Column(0) == existingColumn); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/TestData.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/TestData.cs new file mode 100644 index 000000000..9b6d0cf8b --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/TestData.cs @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow.Tests +{ + public static class TestData + { + public static RecordBatch CreateSampleRecordBatch(int length, bool createDictionaryArray = false) + { + return CreateSampleRecordBatch(length, columnSetCount: 1, createDictionaryArray); + } + + public static RecordBatch CreateSampleRecordBatch(int length, int columnSetCount, bool createAdvancedTypeArrays) + { + Schema.Builder builder = new Schema.Builder(); + for (int i = 0; i < columnSetCount; i++) + { + builder.Field(CreateField(new ListType(Int64Type.Default), i)); + builder.Field(CreateField(BooleanType.Default, i)); + builder.Field(CreateField(UInt8Type.Default, i)); + builder.Field(CreateField(Int8Type.Default, i)); + builder.Field(CreateField(UInt16Type.Default, i)); + builder.Field(CreateField(Int16Type.Default, i)); + builder.Field(CreateField(UInt32Type.Default, i)); + builder.Field(CreateField(Int32Type.Default, i)); + builder.Field(CreateField(UInt64Type.Default, i)); + builder.Field(CreateField(Int64Type.Default, i)); + builder.Field(CreateField(FloatType.Default, i)); + builder.Field(CreateField(DoubleType.Default, i)); + builder.Field(CreateField(Date32Type.Default, i)); + builder.Field(CreateField(Date64Type.Default, i)); + builder.Field(CreateField(TimestampType.Default, i)); + builder.Field(CreateField(StringType.Default, i)); + builder.Field(CreateField(new StructType(new List<Field> { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }), i)); + builder.Field(CreateField(new Decimal128Type(10, 6), i)); + builder.Field(CreateField(new Decimal256Type(16, 8), i)); + + if (createAdvancedTypeArrays) + { + builder.Field(CreateField(new DictionaryType(Int32Type.Default, StringType.Default, false), i)); + builder.Field(CreateField(new FixedSizeBinaryType(16), i)); + } + + //builder.Field(CreateField(HalfFloatType.Default)); + //builder.Field(CreateField(StringType.Default)); + //builder.Field(CreateField(Time32Type.Default)); + //builder.Field(CreateField(Time64Type.Default)); + } + + Schema schema = builder.Build(); + + return CreateSampleRecordBatch(schema, length); + } + + public static RecordBatch CreateSampleRecordBatch(Schema schema, int length) + { + IEnumerable<IArrowArray> arrays = CreateArrays(schema, length); + + return new RecordBatch(schema, arrays, length); + } + + private static Field CreateField(ArrowType type, int iteration) + { + return new Field(type.Name + iteration, type, nullable: false); + } + + public static IEnumerable<IArrowArray> CreateArrays(Schema schema, int length) + { + int fieldCount = schema.Fields.Count; + List<IArrowArray> arrays = new List<IArrowArray>(fieldCount); + for (int i = 0; i < fieldCount; i++) + { + Field field = schema.GetFieldByIndex(i); + arrays.Add(CreateArray(field, length)); + } + return arrays; + } + + private static IArrowArray CreateArray(Field field, int length) + { + var creator = new ArrayCreator(length); + + field.DataType.Accept(creator); + + return creator.Array; + } + + private class ArrayCreator : + IArrowTypeVisitor<BooleanType>, + IArrowTypeVisitor<Date32Type>, + IArrowTypeVisitor<Date64Type>, + IArrowTypeVisitor<Int8Type>, + IArrowTypeVisitor<Int16Type>, + IArrowTypeVisitor<Int32Type>, + IArrowTypeVisitor<Int64Type>, + IArrowTypeVisitor<UInt8Type>, + IArrowTypeVisitor<UInt16Type>, + IArrowTypeVisitor<UInt32Type>, + IArrowTypeVisitor<UInt64Type>, + IArrowTypeVisitor<FloatType>, + IArrowTypeVisitor<DoubleType>, + IArrowTypeVisitor<TimestampType>, + IArrowTypeVisitor<StringType>, + IArrowTypeVisitor<ListType>, + IArrowTypeVisitor<StructType>, + IArrowTypeVisitor<Decimal128Type>, + IArrowTypeVisitor<Decimal256Type>, + IArrowTypeVisitor<DictionaryType>, + IArrowTypeVisitor<FixedSizeBinaryType> + { + private int Length { get; } + public IArrowArray Array { get; private set; } + + public ArrayCreator(int length) + { + Length = length; + } + + public void Visit(BooleanType type) => GenerateArray(new BooleanArray.Builder(), x => x % 2 == 0); + public void Visit(Int8Type type) => GenerateArray(new Int8Array.Builder(), x => (sbyte)x); + public void Visit(Int16Type type) => GenerateArray(new Int16Array.Builder(), x => (short)x); + public void Visit(Int32Type type) => GenerateArray(new Int32Array.Builder(), x => x); + public void Visit(Int64Type type) => GenerateArray(new Int64Array.Builder(), x => x); + public void Visit(UInt8Type type) => GenerateArray(new UInt8Array.Builder(), x => (byte)x); + public void Visit(UInt16Type type) => GenerateArray(new UInt16Array.Builder(), x => (ushort)x); + public void Visit(UInt32Type type) => GenerateArray(new UInt32Array.Builder(), x => (uint)x); + public void Visit(UInt64Type type) => GenerateArray(new UInt64Array.Builder(), x => (ulong)x); + public void Visit(FloatType type) => GenerateArray(new FloatArray.Builder(), x => ((float)x / Length)); + public void Visit(DoubleType type) => GenerateArray(new DoubleArray.Builder(), x => ((double)x / Length)); + public void Visit(Decimal128Type type) + { + var builder = new Decimal128Array.Builder(type).Reserve(Length); + + for (var i = 0; i < Length; i++) + { + builder.Append((decimal)i / Length); + } + + Array = builder.Build(); + } + + public void Visit(Decimal256Type type) + { + var builder = new Decimal256Array.Builder(type).Reserve(Length); + + for (var i = 0; i < Length; i++) + { + builder.Append((decimal)i / Length); + } + + Array = builder.Build(); + } + + public void Visit(Date32Type type) + { + var builder = new Date32Array.Builder().Reserve(Length); + + // Length can be greater than the number of days since DateTime.MinValue. + // Set a cap for how many days can be subtracted from now. + int maxDays = Math.Min(Length, 100_000); + var basis = DateTimeOffset.UtcNow.AddDays(-maxDays); + + for (var i = 0; i < Length; i++) + { + builder.Append(basis.AddDays(i % maxDays)); + } + + Array = builder.Build(); + } + + public void Visit(Date64Type type) + { + var builder = new Date64Array.Builder().Reserve(Length); + var basis = DateTimeOffset.UtcNow.AddSeconds(-Length); + + for (var i = 0; i < Length; i++) + { + builder.Append(basis.AddSeconds(i)); + } + + Array = builder.Build(); + } + + public void Visit(TimestampType type) + { + var builder = new TimestampArray.Builder().Reserve(Length); + var basis = DateTimeOffset.UtcNow.AddMilliseconds(-Length); + + for (var i = 0; i < Length; i++) + { + builder.Append(basis.AddMilliseconds(i)); + } + + Array = builder.Build(); + } + + public void Visit(StringType type) + { + var str = "hello"; + var builder = new StringArray.Builder(); + + for (var i = 0; i < Length; i++) + { + builder.Append(str); + } + + Array = builder.Build(); + } + + public void Visit(ListType type) + { + var builder = new ListArray.Builder(type.ValueField).Reserve(Length); + + //Todo : Support various types + var valueBuilder = (Int64Array.Builder)builder.ValueBuilder.Reserve(Length + 1); + + for (var i = 0; i < Length; i++) + { + builder.Append(); + valueBuilder.Append(i); + } + //Add a value to check if Values.Length can exceed ListArray.Length + valueBuilder.Append(0); + + Array = builder.Build(); + } + + public void Visit(StructType type) + { + IArrowArray[] childArrays = new IArrowArray[type.Fields.Count]; + for (int i = 0; i < childArrays.Length; i++) + { + childArrays[i] = CreateArray(type.Fields[i], Length); + } + + ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder(); + for (int i = 0; i < Length; i++) + { + nullBitmap.Append(true); + } + + Array = new StructArray(type, Length, childArrays, nullBitmap.Build()); + } + + public void Visit(DictionaryType type) + { + Int32Array.Builder indicesBuilder = new Int32Array.Builder().Reserve(Length); + StringArray.Builder valueBuilder = new StringArray.Builder().Reserve(Length); + + for (int i = 0; i < Length; i++) + { + indicesBuilder.Append(i); + valueBuilder.Append($"{i}"); + } + + Array = new DictionaryArray(type, indicesBuilder.Build(), valueBuilder.Build()); + } + + public void Visit(FixedSizeBinaryType type) + { + ArrowBuffer.Builder<byte> valueBuilder = new ArrowBuffer.Builder<byte>(); + + int valueSize = type.BitWidth; + for (int i = 0; i < Length; i++) + { + valueBuilder.Append(Enumerable.Repeat((byte)i, valueSize).ToArray()); + } + + ArrowBuffer validityBuffer = ArrowBuffer.Empty; + ArrowBuffer valueBuffer = valueBuilder.Build(default); + + ArrayData arrayData = new ArrayData(type, Length, 0, 0, new[] { validityBuffer, valueBuffer }); + Array = new FixedSizeBinaryArray(arrayData); + } + + private void GenerateArray<T, TArray, TArrayBuilder>(IArrowArrayBuilder<T, TArray, TArrayBuilder> builder, Func<int, T> generator) + where TArrayBuilder : IArrowArrayBuilder<T, TArray, TArrayBuilder> + where TArray : IArrowArray + where T : struct + { + for (var i = 0; i < Length; i++) + { + if (i == Length - 2) + { + builder.AppendNull(); + } + else + { + var value = generator(i); + builder.Append(value); + } + } + + Array = builder.Build(default); + } + + public void Visit(IArrowType type) + { + throw new NotImplementedException(); + } + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs new file mode 100644 index 000000000..1f2eae45b --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/TestDateAndTimeData.cs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow.Tests +{ + /// <summary> + /// The <see cref="TestDateAndTimeData"/> class holds example dates and times useful for testing. + /// </summary> + internal static class TestDateAndTimeData + { + private static readonly DateTime _earliestDate = new DateTime(1, 1, 1); + private static readonly DateTime _latestDate = new DateTime(9999, 12, 31); + + private static readonly DateTime[] _exampleDates = + { + _earliestDate, new DateTime(1969, 12, 31), new DateTime(1970, 1, 1), new DateTime(1970, 1, 2), + new DateTime(1972, 6, 30), new DateTime(2015, 6, 30), new DateTime(2016, 12, 31), new DateTime(2020, 2, 29), + new DateTime(2020, 7, 1), _latestDate, + }; + + private static readonly TimeSpan[] _exampleTimes = + { + new TimeSpan(0, 0, 1), new TimeSpan(12, 0, 0), new TimeSpan(23, 59, 59), + }; + + private static readonly DateTimeKind[] _exampleKinds = + { + DateTimeKind.Local, DateTimeKind.Unspecified, DateTimeKind.Utc, + }; + + private static readonly TimeSpan[] _exampleOffsets = + { + TimeSpan.FromHours(-2), + TimeSpan.Zero, + TimeSpan.FromHours(2), + }; + + /// <summary> + /// Gets a collection of example dates (i.e. with a zero time component), of all different kinds. + /// </summary> + public static IEnumerable<DateTime> ExampleDates => + from date in _exampleDates + from kind in _exampleKinds + select DateTime.SpecifyKind(date, kind); + + /// <summary> + /// Gets a collection of example date/times, of all different kinds. + /// </summary> + public static IEnumerable<DateTime> ExampleDateTimes => + from date in _exampleDates + from time in _exampleTimes + from kind in _exampleKinds + select DateTime.SpecifyKind(date.Add(time), kind); + + /// <summary> + /// Gets a collection of example date time offsets. + /// </summary> + /// <returns></returns> + public static IEnumerable<DateTimeOffset> ExampleDateTimeOffsets => + from date in _exampleDates + from time in _exampleTimes + from offset in _exampleOffsets + where !(date == _earliestDate && offset.Ticks > 0) + where !(date == _latestDate && offset.Ticks < 0) + select new DateTimeOffset(date.Add(time), offset); + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/TestMemoryAllocator.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/TestMemoryAllocator.cs new file mode 100644 index 000000000..e0e36af17 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/TestMemoryAllocator.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System.Buffers; + +namespace Apache.Arrow.Tests +{ + public class TestMemoryAllocator : MemoryAllocator + { + protected override IMemoryOwner<byte> AllocateInternal(int length, out int bytesAllocated) + { + bytesAllocated = length; + return MemoryPool<byte>.Shared.Rent(length); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Tests/TypeTests.cs b/src/arrow/csharp/test/Apache.Arrow.Tests/TypeTests.cs new file mode 100644 index 000000000..c279d6984 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Tests/TypeTests.cs @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Sdk; + +namespace Apache.Arrow.Tests +{ + public class TypeTests + { + [Fact] + public void Basics() + { + Field.Builder fb = new Field.Builder(); + Field f0_nullable = fb.Name("f0").DataType(Int32Type.Default).Build(); + Field f0_nonnullable = fb.Name("f0").DataType(Int32Type.Default).Nullable(false).Build(); + + Assert.True(f0_nullable.Name == "f0"); + Assert.True(f0_nullable.DataType.Name == Int32Type.Default.Name); + + Assert.True(f0_nullable.IsNullable); + Assert.False(f0_nonnullable.IsNullable); + } + + [Fact] + public void Equality() + { + Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f0_nonnullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Nullable(false).Build(); + Field f0_other = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f0_with_meta = new Field.Builder().Name("f0").DataType(Int32Type.Default).Nullable(true).Metadata("a", "1").Metadata("b", "2").Build(); + + FieldComparer.Compare(f0_nullable, f0_other); + Assert.Throws<EqualException>(() => FieldComparer.Compare(f0_nullable, f0_nonnullable)); + Assert.Throws<EqualException>(() => FieldComparer.Compare(f0_nullable, f0_with_meta)); + } + + [Fact] + public void TestMetadataConstruction() + { + var metadata = new Dictionary<string, string> { { "foo", "bar" }, { "bizz", "buzz" } }; + var metadata1 = new Dictionary<string, string>(metadata); + Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Metadata(metadata).Build(); + Field f1_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Metadata(metadata1).Build(); + Assert.True(metadata.Keys.SequenceEqual(f0_nullable.Metadata.Keys) && metadata.Values.SequenceEqual(f0_nullable.Metadata.Values)); + FieldComparer.Compare(f0_nullable, f1_nullable); + } + + [Fact] + public void TestStructBasics() + { + + Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f1_nullable = new Field.Builder().Name("f1").DataType(StringType.Default).Build(); + Field f2_nullable = new Field.Builder().Name("f2").DataType(UInt8Type.Default).Build(); + + List<Field> fields = new List<Field>() { f0_nullable, f1_nullable, f2_nullable }; + StructType struct_type = new StructType(fields); + + var structFields = struct_type.Fields; + FieldComparer.Compare(structFields.ElementAt(0), f0_nullable); + FieldComparer.Compare(structFields.ElementAt(1), f1_nullable); + FieldComparer.Compare(structFields.ElementAt(2), f2_nullable); + } + + [Fact] + public void TestStructGetFieldByName() + { + + Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f1_nullable = new Field.Builder().Name("f1").DataType(StringType.Default).Build(); + Field f2_nullable = new Field.Builder().Name("f2").DataType(UInt8Type.Default).Build(); + + List<Field> fields = new List<Field>() { f0_nullable, f1_nullable, f2_nullable }; + StructType struct_type = new StructType(fields); + + FieldComparer.Compare(struct_type.GetFieldByName("f0"), f0_nullable); + FieldComparer.Compare(struct_type.GetFieldByName("f1"), f1_nullable); + FieldComparer.Compare(struct_type.GetFieldByName("f2"), f2_nullable); + Assert.True(struct_type.GetFieldByName("not_found") == null); + } + + [Fact] + public void TestStructGetFieldIndex() + { + Field f0_nullable = new Field.Builder().Name("f0").DataType(Int32Type.Default).Build(); + Field f1_nullable = new Field.Builder().Name("f1").DataType(StringType.Default).Build(); + Field f2_nullable = new Field.Builder().Name("f2").DataType(UInt8Type.Default).Build(); + + StructType struct_type = new StructType(new[] { f0_nullable, f1_nullable, f2_nullable }); + + Assert.Equal(0, struct_type.GetFieldIndex("f0")); + Assert.Equal(1, struct_type.GetFieldIndex("f1")); + Assert.Equal(2, struct_type.GetFieldIndex("F2", StringComparer.OrdinalIgnoreCase)); + Assert.Equal(-1, struct_type.GetFieldIndex("F2")); + Assert.Equal(-1, struct_type.GetFieldIndex("F2", StringComparer.Ordinal)); + Assert.Equal(-1, struct_type.GetFieldIndex("not_found")); + } + + [Fact] + public void TestListTypeConstructor() + { + var stringField = new Field.Builder().Name("item").DataType(StringType.Default).Build(); + var stringType1 = new ListType(stringField); + var stringType2 = new ListType(StringType.Default); + + FieldComparer.Compare(stringType1.ValueField, stringType2.ValueField); + Assert.Equal(stringType1.ValueDataType.TypeId, stringType2.ValueDataType.TypeId); + } + + // Todo: StructType::GetFieldIndexDuplicate test + + + } +} diff --git a/src/arrow/csharp/test/Directory.Build.props b/src/arrow/csharp/test/Directory.Build.props new file mode 100644 index 000000000..4f17847df --- /dev/null +++ b/src/arrow/csharp/test/Directory.Build.props @@ -0,0 +1,26 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<Project> + + <Import Project="..\Directory.Build.props" /> + + <PropertyGroup> + <IsPackable>false</IsPackable> + </PropertyGroup> + +</Project> |