diff options
Diffstat (limited to 'src/arrow/csharp/test/Apache.Arrow.Benchmarks')
4 files changed, 265 insertions, 0 deletions
diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj new file mode 100644 index 000000000..e38d538af --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>netcoreapp3.1</TargetFramework> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="BenchmarkDotNet" Version="0.12.1" /> + <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\src\Apache.Arrow\Apache.Arrow.csproj" /> + <ProjectReference Include="..\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj" /> + </ItemGroup> + +</Project> diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs new file mode 100644 index 000000000..4e491a2a6 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowReaderBenchmark.cs @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Memory; +using Apache.Arrow.Tests; +using Apache.Arrow.Types; +using BenchmarkDotNet.Attributes; +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Apache.Arrow.Benchmarks +{ + //[EtwProfiler] - needs elevated privileges + [MemoryDiagnoser] + public class ArrowReaderBenchmark + { + [Params(10_000, 1_000_000)] + public int Count { get; set; } + + private MemoryStream _memoryStream; + private static readonly MemoryAllocator s_allocator = new TestMemoryAllocator(); + + [GlobalSetup] + public async Task GlobalSetup() + { + RecordBatch batch = TestData.CreateSampleRecordBatch(length: Count); + _memoryStream = new MemoryStream(); + + ArrowStreamWriter writer = new ArrowStreamWriter(_memoryStream, batch.Schema); + await writer.WriteRecordBatchAsync(batch); + } + + [IterationSetup] + public void Setup() + { + _memoryStream.Position = 0; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemoryStream() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemoryStream_ManagedMemory() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream, s_allocator); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + [Benchmark] + public async Task<double> ArrowReaderWithMemory() + { + double sum = 0; + var reader = new ArrowStreamReader(_memoryStream.GetBuffer()); + RecordBatch recordBatch; + while ((recordBatch = await reader.ReadNextRecordBatchAsync()) != null) + { + using (recordBatch) + { + sum += SumAllNumbers(recordBatch); + } + } + return sum; + } + + private static double SumAllNumbers(RecordBatch recordBatch) + { + double sum = 0; + + for (int k = 0; k < recordBatch.ColumnCount; k++) + { + var array = recordBatch.Arrays.ElementAt(k); + switch (recordBatch.Schema.GetFieldByIndex(k).DataType.TypeId) + { + case ArrowTypeId.Int64: + Int64Array int64Array = (Int64Array)array; + sum += Sum(int64Array); + break; + case ArrowTypeId.Double: + DoubleArray doubleArray = (DoubleArray)array; + sum += Sum(doubleArray); + break; + case ArrowTypeId.Decimal128: + Decimal128Array decimalArray = (Decimal128Array)array; + sum += Sum(decimalArray); + break; + } + } + return sum; + } + + private static double Sum(DoubleArray doubleArray) + { + double sum = 0; + ReadOnlySpan<double> values = doubleArray.Values; + for (int valueIndex = 0; valueIndex < values.Length; valueIndex++) + { + sum += values[valueIndex]; + } + return sum; + } + + private static long Sum(Int64Array int64Array) + { + long sum = 0; + ReadOnlySpan<long> values = int64Array.Values; + for (int valueIndex = 0; valueIndex < values.Length; valueIndex++) + { + sum += values[valueIndex]; + } + return sum; + } + + private static double Sum(Decimal128Array decimal128Array) + { + double sum = 0; + for (int valueIndex = 0; valueIndex < decimal128Array.Length; valueIndex++) + { + sum += (double)decimal128Array.GetValue(valueIndex); + } + return sum; + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs new file mode 100644 index 000000000..c791c9969 --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Ipc; +using Apache.Arrow.Tests; +using BenchmarkDotNet.Attributes; +using System.IO; +using System.Threading.Tasks; + +namespace Apache.Arrow.Benchmarks +{ + //[EtwProfiler] - needs elevated privileges + [MemoryDiagnoser] + public class ArrowWriterBenchmark + { + [Params(10_000, 1_000_000)] + public int BatchLength{ get; set; } + + //Max column set count is 15 before reaching 2gb limit of memory stream + [Params(10, 14)] + public int ColumnSetCount { get; set; } + + private MemoryStream _memoryStream; + private RecordBatch _batch; + + [GlobalSetup] + public void GlobalSetup() + { + _batch = TestData.CreateSampleRecordBatch(BatchLength, ColumnSetCount, false); + _memoryStream = new MemoryStream(); + } + + [IterationSetup] + public void Setup() + { + _memoryStream.Position = 0; + } + + [Benchmark] + public async Task WriteBatch() + { + ArrowStreamWriter writer = new ArrowStreamWriter(_memoryStream, _batch.Schema); + await writer.WriteRecordBatchAsync(_batch); + } + } +} diff --git a/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs new file mode 100644 index 000000000..0f1410fcb --- /dev/null +++ b/src/arrow/csharp/test/Apache.Arrow.Benchmarks/Program.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using BenchmarkDotNet.Running; + +namespace Apache.Arrow.Benchmarks +{ + public static class Program + { + public static void Main(string[] args) + { + BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .Run(args); + } + } +}
\ No newline at end of file |