// Licensed to the Apache Software Foundation (ASF) under one or more // contributor license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright ownership. // The ASF licenses this file to You under the Apache License, Version 2.0 // (the "License"); you may not use this file except in compliance with // the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using System; using Xunit; namespace Apache.Arrow.Tests { public class ArrowArrayTests { [Fact] public void ThrowsWhenGetValueIndexOutOfBounds() { var array = new Int64Array.Builder().Append(1).Append(2).Build(); Assert.Throws(() => array.GetValue(-1)); Assert.Equal(1, array.GetValue(0)); Assert.Equal(2, array.GetValue(1)); Assert.Throws(() => array.GetValue(2)); } [Fact] public void ThrowsWhenGetValueAndOffsetIndexOutOfBounds() { var array = new BinaryArray.Builder().Append(1).Append(2).Build(); Assert.Throws(() => array.GetValueLength(-1)); Assert.Equal(1, array.GetValueLength(0)); Assert.Equal(1, array.GetValueLength(1)); Assert.Throws(() => array.GetValueLength(2)); #pragma warning disable 618 Assert.Throws(() => array.GetValueOffset(-1)); Assert.Equal(0, array.GetValueOffset(0)); Assert.Equal(1, array.GetValueOffset(1)); Assert.Equal(2, array.GetValueOffset(2)); Assert.Throws(() => array.GetValueOffset(3)); #pragma warning restore 618 Assert.Throws(() => array.ValueOffsets[-1]); Assert.Equal(0, array.ValueOffsets[0]); Assert.Equal(1, array.ValueOffsets[1]); Assert.Equal(2, array.ValueOffsets[2]); Assert.Throws(() => array.ValueOffsets[3]); } [Fact] public void IsValidValue() { const int totalValueCount = 8; const byte nullBitmap = 0b_11110011; var nullBitmapBuffer = new ArrowBuffer.Builder().Append(nullBitmap).Build(); var valueBuffer = new ArrowBuffer.Builder().Append(0).Append(1).Append(4).Append(5).Append(6).Append(7).Append(8).Build(); //Check all offset and length for (var offset = 0; offset < totalValueCount; offset++) { var nullCount = totalValueCount - offset - BitUtility.CountBits(nullBitmapBuffer.Span, offset); for (var length = 1; length + offset < totalValueCount; length++) { TestIsValid(valueBuffer, nullBitmapBuffer, length, nullCount, offset); } } void TestIsValid(ArrowBuffer valueBuf, ArrowBuffer nullBitmapBuf, int length, int nullCount, int offset) { var array = new Int64Array(valueBuf, nullBitmapBuf, length, nullCount, offset); for (var i = 0; i < length; i++) { if (BitUtility.GetBit(nullBitmap, i + offset)) { Assert.True(array.IsValid(i)); } else { Assert.False(array.IsValid(i)); } } } } [Fact] public void SliceArray() { TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30)); TestSlice(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).Append(new DateTime(2019, 1, 3))); TestSlice(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).Append(new DateTime(2019, 1, 3))); TestSlice(x => x.Append("10").Append("20").Append("30")); } [Fact] public void SlicePrimitiveArrayWithNulls() { TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).AppendNull().Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).Append(30).AppendNull()); TestSlice(x => x.Append(10).Append(20).AppendNull().AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.AppendNull().Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(10).Append(20).AppendNull().Append(30)); TestSlice(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).AppendNull().Append(new DateTime(2019, 1, 3))); TestSlice(x => x.Append(new DateTime(2019, 1, 1)).Append(new DateTime(2019, 1, 2)).AppendNull().Append(new DateTime(2019, 1, 3))); } [Fact] public void SliceBooleanArray() { TestSlice(x => x.Append(true).Append(false).Append(true)); TestSlice(x => x.Append(true).Append(false).AppendNull().Append(true)); } [Fact] public void SliceStringArrayWithNullsAndEmptyStrings() { TestSlice(x => x.Append("10").AppendNull().Append("30")); TestSlice(x => x.Append("10").Append(string.Empty).Append("30")); TestSlice(x => x.Append("10").Append(string.Empty).AppendNull().Append("30")); TestSlice(x => x.Append("10").AppendNull().Append(string.Empty).Append("30")); TestSlice(x => x.Append("10").AppendNull().Append(string.Empty).AppendNull().Append("30")); } private static void TestSlice(Action action) where TArray : IArrowArray where TArrayBuilder : IArrowArrayBuilder, new() { var builder = new TArrayBuilder(); action(builder); var baseArray = builder.Build(default) as Array; Assert.NotNull(baseArray); var totalLength = baseArray.Length; var validator = new ArraySliceValidator(baseArray); //Check all offset and length for (var offset = 0; offset < totalLength; offset++) { for (var length = 1; length + offset <= totalLength; length++) { var targetArray = baseArray.Slice(offset, length); targetArray.Accept(validator); } } } private class ArraySliceValidator : IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor, IArrowArrayVisitor { private readonly IArrowArray _baseArray; public ArraySliceValidator(IArrowArray baseArray) { _baseArray = baseArray; } public void Visit(Int8Array array) => ValidateArrays(array); public void Visit(Int16Array array) => ValidateArrays(array); public void Visit(Int32Array array) => ValidateArrays(array); public void Visit(Int64Array array) => ValidateArrays(array); public void Visit(UInt8Array array) => ValidateArrays(array); public void Visit(UInt16Array array) => ValidateArrays(array); public void Visit(UInt32Array array) => ValidateArrays(array); public void Visit(UInt64Array array) => ValidateArrays(array); public void Visit(Date32Array array) { ValidateArrays(array); Assert.IsAssignableFrom(_baseArray); var baseArray = (Date32Array)_baseArray; Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0)); } public void Visit(Date64Array array) { ValidateArrays(array); Assert.IsAssignableFrom(_baseArray); var baseArray = (Date64Array)_baseArray; Assert.Equal(baseArray.GetDateTimeOffset(array.Offset), array.GetDateTimeOffset(0)); } public void Visit(FloatArray array) => ValidateArrays(array); public void Visit(DoubleArray array) => ValidateArrays(array); public void Visit(StringArray array) => ValidateArrays(array); public void Visit(BooleanArray array) => ValidateArrays(array); public void Visit(IArrowArray array) => throw new NotImplementedException(); private void ValidateArrays(PrimitiveArray slicedArray) where T : struct, IEquatable { Assert.IsAssignableFrom>(_baseArray); var baseArray = (PrimitiveArray)_baseArray; Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); Assert.True( baseArray.ValueBuffer.Span.CastTo().Slice(slicedArray.Offset, slicedArray.Length) .SequenceEqual(slicedArray.Values)); Assert.Equal(baseArray.GetValue(slicedArray.Offset), slicedArray.GetValue(0)); } private void ValidateArrays(BooleanArray slicedArray) { Assert.IsAssignableFrom(_baseArray); var baseArray = (BooleanArray)_baseArray; Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); Assert.True(baseArray.Values.SequenceEqual(slicedArray.Values)); Assert.True( baseArray.ValueBuffer.Span.Slice(0, (int) Math.Ceiling(slicedArray.Length / 8.0)) .SequenceEqual(slicedArray.Values)); Assert.Equal(baseArray.GetValue(slicedArray.Offset), slicedArray.GetValue(0)); #pragma warning disable CS0618 Assert.Equal(baseArray.GetBoolean(slicedArray.Offset), slicedArray.GetBoolean(0)); #pragma warning restore CS0618 } private void ValidateArrays(BinaryArray slicedArray) { Assert.IsAssignableFrom(_baseArray); var baseArray = (BinaryArray)_baseArray; Assert.True(baseArray.Values.SequenceEqual(slicedArray.Values)); Assert.True(baseArray.NullBitmapBuffer.Span.SequenceEqual(slicedArray.NullBitmapBuffer.Span)); Assert.True( baseArray.ValueOffsetsBuffer.Span.CastTo().Slice(slicedArray.Offset, slicedArray.Length + 1) .SequenceEqual(slicedArray.ValueOffsets)); Assert.True(baseArray.GetBytes(slicedArray.Offset).SequenceEqual(slicedArray.GetBytes(0))); } } } }