diff options
Diffstat (limited to 'src/arrow/java/adapter/avro/src/test')
69 files changed, 3638 insertions, 0 deletions
diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java new file mode 100644 index 000000000..050a50dda --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static junit.framework.TestCase.assertNull; +import static junit.framework.TestCase.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.util.DateUtility; +import org.apache.avro.Conversions; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericFixed; +import org.junit.Test; + +public class AvroLogicalTypesTest extends AvroTestBase { + + @Test + public void testTimestampMicros() throws Exception { + Schema schema = getSchema("logical/test_timestamp_micros.avsc"); + + List<Long> data = Arrays.asList(10000L, 20000L, 30000L, 40000L, 50000L); + List<LocalDateTime> expected = Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMicro(10000), + DateUtility.getLocalDateTimeFromEpochMicro(20000), + DateUtility.getLocalDateTimeFromEpochMicro(30000), + DateUtility.getLocalDateTimeFromEpochMicro(40000), + DateUtility.getLocalDateTimeFromEpochMicro(50000) + ); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + + @Test + public void testTimestampMillis() throws Exception { + Schema schema = getSchema("logical/test_timestamp_millis.avsc"); + + List<Long> data = Arrays.asList(10000L, 20000L, 30000L, 40000L, 50000L); + List<LocalDateTime> expected = Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMilli(10000), + DateUtility.getLocalDateTimeFromEpochMilli(20000), + DateUtility.getLocalDateTimeFromEpochMilli(30000), + DateUtility.getLocalDateTimeFromEpochMilli(40000), + DateUtility.getLocalDateTimeFromEpochMilli(50000) + ); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + + @Test + public void testTimeMicros() throws Exception { + Schema schema = getSchema("logical/test_time_micros.avsc"); + + List<Long> data = Arrays.asList(10000L, 20000L, 30000L, 40000L, 50000L); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testTimeMillis() throws Exception { + Schema schema = getSchema("logical/test_time_millis.avsc"); + + List<Integer> data = Arrays.asList(100, 200, 300, 400, 500); + List<LocalDateTime> expected = Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMilli(100), + DateUtility.getLocalDateTimeFromEpochMilli(200), + DateUtility.getLocalDateTimeFromEpochMilli(300), + DateUtility.getLocalDateTimeFromEpochMilli(400), + DateUtility.getLocalDateTimeFromEpochMilli(500) + ); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + + @Test + public void testDate() throws Exception { + Schema schema = getSchema("logical/test_date.avsc"); + + List<Integer> data = Arrays.asList(100, 200, 300, 400, 500); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testDecimalWithOriginalBytes() throws Exception { + Schema schema = getSchema("logical/test_decimal_with_original_bytes.avsc"); + List<ByteBuffer> data = new ArrayList<>(); + List<BigDecimal> expected = new ArrayList<>(); + + Conversions.DecimalConversion conversion = new Conversions.DecimalConversion(); + + for (int i = 0; i < 5; i++) { + BigDecimal value = new BigDecimal(i * i).setScale(2); + ByteBuffer buffer = conversion.toBytes(value, schema, schema.getLogicalType()); + data.add(buffer); + expected.add(value); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + checkPrimitiveResult(expected, vector); + + } + + @Test + public void testDecimalWithOriginalFixed() throws Exception { + Schema schema = getSchema("logical/test_decimal_with_original_fixed.avsc"); + + List<GenericFixed> data = new ArrayList<>(); + List<BigDecimal> expected = new ArrayList<>(); + + Conversions.DecimalConversion conversion = new Conversions.DecimalConversion(); + + for (int i = 0; i < 5; i++) { + BigDecimal value = new BigDecimal(i * i).setScale(2); + GenericFixed fixed = conversion.toFixed(value, schema, schema.getLogicalType()); + data.add(fixed); + expected.add(value); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + checkPrimitiveResult(expected, vector); + } + + @Test + public void testInvalidDecimalPrecision() throws Exception { + Schema schema = getSchema("logical/test_decimal_invalid1.avsc"); + List<ByteBuffer> data = new ArrayList<>(); + + Conversions.DecimalConversion conversion = new Conversions.DecimalConversion(); + + for (int i = 0; i < 5; i++) { + BigDecimal value = new BigDecimal(i * i).setScale(2); + ByteBuffer buffer = conversion.toBytes(value, schema, schema.getLogicalType()); + data.add(buffer); + } + + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> writeAndRead(schema, data)); + assertTrue(e.getMessage().contains("Precision must be in range of 1 to 38")); + + } + + @Test + public void testFailedToCreateDecimalLogicalType() throws Exception { + // For decimal logical type, if avro validate schema failed, it will not create logical type, + // and the schema will be treated as its original type. + + // java.lang.IllegalArgumentException: Invalid decimal scale: -1 (must be positive) + Schema schema1 = getSchema("logical/test_decimal_invalid2.avsc"); + assertNull(schema1.getLogicalType()); + + // java.lang.IllegalArgumentException: Invalid decimal scale: 40 (greater than precision: 20) + Schema schema2 = getSchema("logical/test_decimal_invalid3.avsc"); + assertNull(schema2.getLogicalType()); + + // java.lang.IllegalArgumentException: fixed(1) cannot store 30 digits (max 2) + Schema schema3 = getSchema("logical/test_decimal_invalid4.avsc"); + assertNull(schema3.getLogicalType()); + } + +} diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java new file mode 100644 index 000000000..b946dbd86 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java @@ -0,0 +1,626 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; + +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.types.Types; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.junit.Test; + +public class AvroSkipFieldTest extends AvroTestBase { + + @Test + public void testSkipUnionWithOneField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_union_before.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_union_one_field_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i % 2 == 0 ? "test" + i : null); + record.put(2, i % 2 == 0 ? "test" + i : i); + record.put(3, i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(1)); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedData.add(expectedRecord); + } + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipUnionWithNullableOneField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_union_before.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_union_nullable_field_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i % 2 == 0 ? "test" + i : null); + record.put(2, i % 2 == 0 ? "test" + i : i); + record.put(3, i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedData.add(expectedRecord); + } + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipUnionWithMultiFields() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f2"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_union_before.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_union_multi_fields_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i % 2 == 0 ? "test" + i : null); + record.put(2, i % 2 == 0 ? "test" + i : i); + record.put(3, i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(1)); + expectedRecord.put(2, record.get(3)); + expectedData.add(expectedRecord); + } + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipMapField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_map_before.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_map_expected.avsc"); + + HashMap map = new HashMap(); + map.put("key1", "value1"); + map.put("key2", "value3"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, map); + record.put(2, i % 2 == 0); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(2)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipArrayField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_array_before.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_array_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, Arrays.asList("test" + i, "test" + i)); + record.put(2, i % 2 == 0); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(2)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipMultiFields() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + skipFieldNames.add("f2"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("test_record.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_multi_fields_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i); + record.put(2, i % 2 == 0); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipStringField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f2"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base1.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_string_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + final byte[] testBytes = ("test" + i).getBytes(); + GenericRecord record = new GenericData.Record(schema); + GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); + fixed.bytes(testBytes); + record.put(0, fixed); + GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + record.put(1, symbol); + record.put(2, "testtest" + i); + record.put(3, ByteBuffer.wrap(testBytes)); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, testBytes); + expectedRecord.put(1, (byte) i % 2); + expectedRecord.put(2, testBytes); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipBytesField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f3"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base1.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_bytes_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + final byte[] testBytes = ("test" + i).getBytes(); + GenericRecord record = new GenericData.Record(schema); + GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); + fixed.bytes(testBytes); + record.put(0, fixed); + GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + record.put(1, symbol); + record.put(2, "testtest" + i); + record.put(3, ByteBuffer.wrap(testBytes)); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, testBytes); + expectedRecord.put(1, (byte) i % 2); + expectedRecord.put(2, record.get(2)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipFixedField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base1.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_fixed_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + final byte[] testBytes = ("test" + i).getBytes(); + GenericRecord record = new GenericData.Record(schema); + GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); + fixed.bytes(testBytes); + record.put(0, fixed); + GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + record.put(1, symbol); + record.put(2, "testtest" + i); + record.put(3, ByteBuffer.wrap(testBytes)); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, (byte) i % 2); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipEnumField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base1.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_fixed_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + final byte[] testBytes = ("test" + i).getBytes(); + GenericRecord record = new GenericData.Record(schema); + GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); + fixed.bytes(testBytes); + record.put(0, fixed); + GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + record.put(1, symbol); + record.put(2, "testtest" + i); + record.put(3, ByteBuffer.wrap(testBytes)); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, testBytes); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipBooleanField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base2.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_boolean_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0); + record.put(1, i); + record.put(2, (long) i); + record.put(3, (float) i); + record.put(4, (double) i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(1)); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedRecord.put(3, record.get(4)); + + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipIntField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f1"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base2.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_int_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0); + record.put(1, i); + record.put(2, (long) i); + record.put(3, (float) i); + record.put(4, (double) i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(2)); + expectedRecord.put(2, record.get(3)); + expectedRecord.put(3, record.get(4)); + + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipLongField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f2"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base2.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_long_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0); + record.put(1, i); + record.put(2, (long) i); + record.put(3, (float) i); + record.put(4, (double) i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(1)); + expectedRecord.put(2, record.get(3)); + expectedRecord.put(3, record.get(4)); + + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipFloatField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f3"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base2.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_float_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0); + record.put(1, i); + record.put(2, (long) i); + record.put(3, (float) i); + record.put(4, (double) i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(1)); + expectedRecord.put(2, record.get(2)); + expectedRecord.put(3, record.get(4)); + + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipDoubleField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f4"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_base2.avsc"); + Schema expectedSchema = getSchema("skip/test_skip_double_expected.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0); + record.put(1, i); + record.put(2, (long) i); + record.put(3, (float) i); + record.put(4, (double) i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, record.get(0)); + expectedRecord.put(1, record.get(1)); + expectedRecord.put(2, record.get(2)); + expectedRecord.put(3, record.get(3)); + + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipRecordField() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("skip/test_skip_record_before.avsc"); + Schema nestedSchema = schema.getFields().get(0).schema(); + ArrayList<GenericRecord> data = new ArrayList<>(); + + Schema expectedSchema = getSchema("skip/test_skip_record_expected.avsc"); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + GenericRecord nestedRecord = new GenericData.Record(nestedSchema); + nestedRecord.put(0, "test" + i); + nestedRecord.put(1, i); + record.put(0, nestedRecord); + record.put(1, i); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + expectedRecord.put(0, i); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipNestedFields() throws Exception { + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0.f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + Schema schema = getSchema("test_nested_record.avsc"); + Schema nestedSchema = schema.getFields().get(0).schema(); + ArrayList<GenericRecord> data = new ArrayList<>(); + + Schema expectedSchema = getSchema("skip/test_skip_second_level_expected.avsc"); + Schema expectedNestedSchema = expectedSchema.getFields().get(0).schema(); + ArrayList<GenericRecord> expectedData = new ArrayList<>(); + + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + GenericRecord nestedRecord = new GenericData.Record(nestedSchema); + nestedRecord.put(0, "test" + i); + nestedRecord.put(1, i); + record.put(0, nestedRecord); + data.add(record); + + GenericRecord expectedRecord = new GenericData.Record(expectedSchema); + GenericRecord expectedNestedRecord = new GenericData.Record(expectedNestedSchema); + expectedNestedRecord.put(0, nestedRecord.get(1)); + expectedRecord.put(0, expectedNestedRecord); + expectedData.add(expectedRecord); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkNestedRecordResult(expectedSchema, expectedData, root); + } + + @Test + public void testSkipThirdLevelField() throws Exception { + Schema firstLevelSchema = getSchema("skip/test_skip_third_level_expected.avsc"); + Schema secondLevelSchema = firstLevelSchema.getFields().get(0).schema(); + Schema thirdLevelSchema = secondLevelSchema.getFields().get(0).schema(); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord firstLevelRecord = new GenericData.Record(firstLevelSchema); + GenericRecord secondLevelRecord = new GenericData.Record(secondLevelSchema); + GenericRecord thirdLevelRecord = new GenericData.Record(thirdLevelSchema); + + thirdLevelRecord.put(0, i); + thirdLevelRecord.put(1, "test" + i); + thirdLevelRecord.put(2, i % 2 == 0); + + secondLevelRecord.put(0, thirdLevelRecord); + firstLevelRecord.put(0, secondLevelRecord); + data.add(firstLevelRecord); + } + + // do not skip any fields first + VectorSchemaRoot root1 = writeAndRead(firstLevelSchema, data); + + assertEquals(1, root1.getFieldVectors().size()); + assertEquals(Types.MinorType.STRUCT, root1.getFieldVectors().get(0).getMinorType()); + StructVector secondLevelVector = (StructVector) root1.getFieldVectors().get(0); + assertEquals(1, secondLevelVector.getChildrenFromFields().size()); + assertEquals(Types.MinorType.STRUCT, secondLevelVector.getChildrenFromFields().get(0).getMinorType()); + StructVector thirdLevelVector = (StructVector) secondLevelVector.getChildrenFromFields().get(0); + assertEquals(3, thirdLevelVector.getChildrenFromFields().size()); + + // skip third level field and validate + Set<String> skipFieldNames = new HashSet<>(); + skipFieldNames.add("f0.f0.f0"); + config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + VectorSchemaRoot root2 = writeAndRead(firstLevelSchema, data); + + assertEquals(1, root2.getFieldVectors().size()); + assertEquals(Types.MinorType.STRUCT, root2.getFieldVectors().get(0).getMinorType()); + StructVector secondStruct = (StructVector) root2.getFieldVectors().get(0); + assertEquals(1, secondStruct.getChildrenFromFields().size()); + assertEquals(Types.MinorType.STRUCT, secondStruct.getChildrenFromFields().get(0).getMinorType()); + StructVector thirdStruct = (StructVector) secondStruct.getChildrenFromFields().get(0); + assertEquals(2, thirdStruct.getChildrenFromFields().size()); + + assertEquals(Types.MinorType.INT, thirdStruct.getChildrenFromFields().get(0).getMinorType()); + assertEquals(Types.MinorType.BIT, thirdStruct.getChildrenFromFields().get(1).getMinorType()); + } +} diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java new file mode 100644 index 000000000..a00cd7704 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.util.Text; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.TemporaryFolder; + +public class AvroTestBase { + + @ClassRule + public static final TemporaryFolder TMP = new TemporaryFolder(); + + protected AvroToArrowConfig config; + + @Before + public void init() { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + config = new AvroToArrowConfigBuilder(allocator).build(); + } + + protected Schema getSchema(String schemaName) throws Exception { + Path schemaPath = Paths.get(TestWriteReadAvroRecord.class.getResource("/").getPath(), + "schema", schemaName); + return new Schema.Parser().parse(schemaPath.toFile()); + } + + protected VectorSchemaRoot writeAndRead(Schema schema, List data) throws Exception { + File dataFile = TMP.newFile(); + + BinaryEncoder + encoder = new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); + DatumWriter writer = new GenericDatumWriter(schema); + BinaryDecoder + decoder = new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); + + for (Object value : data) { + writer.write(value, encoder); + } + + return AvroToArrow.avroToArrow(schema, decoder, config); + } + + protected void checkArrayResult(List<List<?>> expected, ListVector vector) { + assertEquals(expected.size(), vector.getValueCount()); + for (int i = 0; i < expected.size(); i++) { + checkArrayElement(expected.get(i), vector.getObject(i)); + } + } + + protected void checkArrayElement(List expected, List actual) { + assertEquals(expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + Object value1 = expected.get(i); + Object value2 = actual.get(i); + if (value1 == null) { + assertTrue(value2 == null); + continue; + } + if (value2 instanceof byte[]) { + value2 = ByteBuffer.wrap((byte[]) value2); + } else if (value2 instanceof Text) { + value2 = value2.toString(); + } + assertEquals(value1, value2); + } + } + + protected void checkPrimitiveResult(List data, FieldVector vector) { + assertEquals(data.size(), vector.getValueCount()); + for (int i = 0; i < data.size(); i++) { + Object value1 = data.get(i); + Object value2 = vector.getObject(i); + if (value1 == null) { + assertTrue(value2 == null); + continue; + } + if (value2 instanceof byte[]) { + value2 = ByteBuffer.wrap((byte[]) value2); + if (value1 instanceof byte[]) { + value1 = ByteBuffer.wrap((byte[]) value1); + } + } else if (value2 instanceof Text) { + value2 = value2.toString(); + } else if (value2 instanceof Byte) { + value2 = ((Byte) value2).intValue(); + } + assertEquals(value1, value2); + } + } + + protected void checkRecordResult(Schema schema, ArrayList<GenericRecord> data, VectorSchemaRoot root) { + assertEquals(data.size(), root.getRowCount()); + assertEquals(schema.getFields().size(), root.getFieldVectors().size()); + + for (int i = 0; i < schema.getFields().size(); i++) { + ArrayList fieldData = new ArrayList(); + for (GenericRecord record : data) { + fieldData.add(record.get(i)); + } + + checkPrimitiveResult(fieldData, root.getFieldVectors().get(i)); + } + + } + + protected void checkNestedRecordResult(Schema schema, List<GenericRecord> data, VectorSchemaRoot root) { + assertEquals(data.size(), root.getRowCount()); + assertTrue(schema.getFields().size() == 1); + + final Schema nestedSchema = schema.getFields().get(0).schema(); + final StructVector structVector = (StructVector) root.getFieldVectors().get(0); + + for (int i = 0; i < nestedSchema.getFields().size(); i++) { + ArrayList fieldData = new ArrayList(); + for (GenericRecord record : data) { + GenericRecord nestedRecord = (GenericRecord) record.get(0); + fieldData.add(nestedRecord.get(i)); + } + + checkPrimitiveResult(fieldData, structVector.getChildrenFromFields().get(i)); + } + + } + + + // belows are for iterator api + + protected void checkArrayResult(List<List<?>> expected, List<ListVector> vectors) { + int valueCount = vectors.stream().mapToInt(v -> v.getValueCount()).sum(); + assertEquals(expected.size(), valueCount); + + int index = 0; + for (ListVector vector : vectors) { + for (int i = 0; i < vector.getValueCount(); i++) { + checkArrayElement(expected.get(index++), vector.getObject(i)); + } + } + } + + protected void checkRecordResult(Schema schema, ArrayList<GenericRecord> data, List<VectorSchemaRoot> roots) { + roots.forEach(root -> { + assertEquals(schema.getFields().size(), root.getFieldVectors().size()); + }); + + for (int i = 0; i < schema.getFields().size(); i++) { + List fieldData = new ArrayList(); + List<FieldVector> vectors = new ArrayList<>(); + for (GenericRecord record : data) { + fieldData.add(record.get(i)); + } + final int columnIndex = i; + roots.forEach(root -> vectors.add(root.getFieldVectors().get(columnIndex))); + + checkPrimitiveResult(fieldData, vectors); + } + + } + + protected void checkPrimitiveResult(List data, List<FieldVector> vectors) { + int valueCount = vectors.stream().mapToInt(v -> v.getValueCount()).sum(); + assertEquals(data.size(), valueCount); + + int index = 0; + for (FieldVector vector : vectors) { + for (int i = 0; i < vector.getValueCount(); i++) { + Object value1 = data.get(index++); + Object value2 = vector.getObject(i); + if (value1 == null) { + assertNull(value2); + continue; + } + if (value2 instanceof byte[]) { + value2 = ByteBuffer.wrap((byte[]) value2); + if (value1 instanceof byte[]) { + value1 = ByteBuffer.wrap((byte[]) value1); + } + } else if (value2 instanceof Text) { + value2 = value2.toString(); + } + assertEquals(value1, value2); + } + } + } +} diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java new file mode 100644 index 000000000..2b05a19f3 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static org.junit.Assert.assertEquals; + +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.util.Utf8; +import org.junit.Test; + +public class AvroToArrowIteratorTest extends AvroTestBase { + + @Override + public void init() { + final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + this.config = new AvroToArrowConfigBuilder(allocator).setTargetBatchSize(3).build(); + } + + private AvroToArrowVectorIterator convert(Schema schema, List data) throws Exception { + File dataFile = TMP.newFile(); + + BinaryEncoder + encoder = new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); + DatumWriter writer = new GenericDatumWriter(schema); + BinaryDecoder + decoder = new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); + + for (Object value : data) { + writer.write(value, encoder); + } + + return AvroToArrow.avroToArrowIterator(schema, decoder, config); + } + + @Test + public void testStringType() throws Exception { + Schema schema = getSchema("test_primitive_string.avsc"); + List<String> data = Arrays.asList("v1", "v2", "v3", "v4", "v5"); + + List<VectorSchemaRoot> roots = new ArrayList<>(); + List<FieldVector> vectors = new ArrayList<>(); + try (AvroToArrowVectorIterator iterator = convert(schema, data)) { + while (iterator.hasNext()) { + VectorSchemaRoot root = iterator.next(); + FieldVector vector = root.getFieldVectors().get(0); + roots.add(root); + vectors.add(vector); + } + } + checkPrimitiveResult(data, vectors); + AutoCloseables.close(roots); + } + + @Test + public void testNullableStringType() throws Exception { + Schema schema = getSchema("test_nullable_string.avsc"); + + List<GenericRecord> data = new ArrayList<>(); + List<String> expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + String value = i % 2 == 0 ? "test" + i : null; + record.put(0, value); + expected.add(value); + data.add(record); + } + + List<VectorSchemaRoot> roots = new ArrayList<>(); + List<FieldVector> vectors = new ArrayList<>(); + try (AvroToArrowVectorIterator iterator = convert(schema, data);) { + while (iterator.hasNext()) { + VectorSchemaRoot root = iterator.next(); + FieldVector vector = root.getFieldVectors().get(0); + roots.add(root); + vectors.add(vector); + } + } + checkPrimitiveResult(expected, vectors); + AutoCloseables.close(roots); + + } + + @Test + public void testRecordType() throws Exception { + Schema schema = getSchema("test_record.avsc"); + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i); + record.put(2, i % 2 == 0); + data.add(record); + } + + List<VectorSchemaRoot> roots = new ArrayList<>(); + try (AvroToArrowVectorIterator iterator = convert(schema, data)) { + while (iterator.hasNext()) { + roots.add(iterator.next()); + } + } + checkRecordResult(schema, data, roots); + AutoCloseables.close(roots); + + } + + @Test + public void testArrayType() throws Exception { + Schema schema = getSchema("test_array.avsc"); + List<List<?>> data = Arrays.asList( + Arrays.asList("11", "222", "999"), + Arrays.asList("12222", "2333", "1000"), + Arrays.asList("1rrr", "2ggg"), + Arrays.asList("1vvv", "2bbb"), + Arrays.asList("1fff", "2")); + + List<VectorSchemaRoot> roots = new ArrayList<>(); + List<ListVector> vectors = new ArrayList<>(); + try (AvroToArrowVectorIterator iterator = convert(schema, data)) { + while (iterator.hasNext()) { + VectorSchemaRoot root = iterator.next(); + roots.add(root); + vectors.add((ListVector) root.getFieldVectors().get(0)); + } + } + checkArrayResult(data, vectors); + AutoCloseables.close(roots); + } + + @Test + public void runLargeNumberOfRows() throws Exception { + Schema schema = getSchema("test_large_data.avsc"); + int x = 0; + final int targetRows = 600000; + Decoder fakeDecoder = new FakeDecoder(targetRows); + try (AvroToArrowVectorIterator iter = AvroToArrow.avroToArrowIterator(schema, fakeDecoder, + new AvroToArrowConfigBuilder(config.getAllocator()).build())) { + while (iter.hasNext()) { + VectorSchemaRoot root = iter.next(); + x += root.getRowCount(); + root.close(); + } + } + + assertEquals(x, targetRows); + } + + /** + * Fake avro decoder to test large data. + */ + private class FakeDecoder extends Decoder { + + private int numRows; + + FakeDecoder(int numRows) { + this.numRows = numRows; + } + + // note that Decoder has no hasNext() API, assume enum is the first type in schema + // and fixed is the last type in schema and they are unique. + private void validate() throws EOFException { + if (numRows <= 0) { + throw new EOFException(); + } + } + + @Override + public void readNull() throws IOException { + } + + @Override + public boolean readBoolean() throws IOException { + return false; + } + + @Override + public int readInt() throws IOException { + return 0; + } + + @Override + public long readLong() throws IOException { + return 0; + } + + @Override + public float readFloat() throws IOException { + return 0; + } + + @Override + public double readDouble() throws IOException { + return 0; + } + + @Override + public Utf8 readString(Utf8 old) throws IOException { + return new Utf8("test123test123" + numRows); + } + + @Override + public String readString() throws IOException { + return "test123test123" + numRows; + } + + @Override + public void skipString() throws IOException { + + } + + @Override + public ByteBuffer readBytes(ByteBuffer old) throws IOException { + return ByteBuffer.allocate(0); + } + + @Override + public void skipBytes() throws IOException { + + } + + @Override + public void readFixed(byte[] bytes, int start, int length) throws IOException { + // fixed type is last column, after read value, decrease numRows + numRows--; + } + + @Override + public void skipFixed(int length) throws IOException { + + } + + @Override + public int readEnum() throws IOException { + // enum type is first column, validate numRows first. + validate(); + return 0; + } + + @Override + public long readArrayStart() throws IOException { + return 5; + } + + @Override + public long arrayNext() throws IOException { + return 0; + } + + @Override + public long skipArray() throws IOException { + return 0; + } + + @Override + public long readMapStart() throws IOException { + return 5; + } + + @Override + public long mapNext() throws IOException { + return 0; + } + + @Override + public long skipMap() throws IOException { + return 0; + } + + @Override + public int readIndex() throws IOException { + return 0; + } + } +} diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java new file mode 100644 index 000000000..c007e1ac7 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java @@ -0,0 +1,477 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.junit.Test; + +public class AvroToArrowTest extends AvroTestBase { + + @Test + public void testStringType() throws Exception { + Schema schema = getSchema("test_primitive_string.avsc"); + List<String> data = Arrays.asList("v1", "v2", "v3", "v4", "v5"); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableStringType() throws Exception { + Schema schema = getSchema("test_nullable_string.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? "test" + i : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testRecordType() throws Exception { + Schema schema = getSchema("test_record.avsc"); + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, "test" + i); + record.put(1, i); + record.put(2, i % 2 == 0); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testFixedAttributes() throws Exception { + Schema schema = getSchema("attrs/test_fixed_attr.avsc"); + + List<GenericData.Fixed> data = new ArrayList<>(); + List<byte[]> expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + byte[] value = ("value" + i).getBytes(StandardCharsets.UTF_8); + expected.add(value); + GenericData.Fixed fixed = new GenericData.Fixed(schema); + fixed.bytes(value); + data.add(fixed); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + Map<String, String> metadata = vector.getField().getMetadata(); + assertEquals("fixed doc", metadata.get("doc")); + assertEquals("[\"alias1\",\"alias2\"]", metadata.get("aliases")); + } + + @Test + public void testEnumAttributes() throws Exception { + Schema schema = getSchema("attrs/test_enum_attrs.avsc"); + List<GenericData.EnumSymbol> data = Arrays.asList( + new GenericData.EnumSymbol(schema, "SPADES"), + new GenericData.EnumSymbol(schema, "HEARTS"), + new GenericData.EnumSymbol(schema, "DIAMONDS"), + new GenericData.EnumSymbol(schema, "CLUBS"), + new GenericData.EnumSymbol(schema, "SPADES")); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + Map<String, String> metadata = vector.getField().getMetadata(); + assertEquals("enum doc", metadata.get("doc")); + assertEquals("[\"alias1\",\"alias2\"]", metadata.get("aliases")); + } + + @Test + public void testRecordAttributes() throws Exception { + Schema schema = getSchema("attrs/test_record_attrs.avsc"); + Schema nestedSchema = schema.getFields().get(0).schema(); + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + GenericRecord nestedRecord = new GenericData.Record(nestedSchema); + nestedRecord.put(0, "test" + i); + nestedRecord.put(1, i); + record.put(0, nestedRecord); + + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + + StructVector structVector = (StructVector) root.getFieldVectors().get(0); + Map<String, String> structMeta = structVector.getField().getMetadata(); + Map<String, String> childMeta1 = structVector.getChildByOrdinal(0).getField().getMetadata(); + Map<String, String> childMeta2 = structVector.getChildByOrdinal(1).getField().getMetadata(); + + assertEquals("f0 doc", structMeta.get("doc")); + assertEquals("[\"f0.a1\"]", structMeta.get("aliases")); + assertEquals("f1 doc", childMeta1.get("doc")); + assertEquals("[\"f1.a1\",\"f1.a2\"]", childMeta1.get("aliases")); + assertEquals("f2 doc", childMeta2.get("doc")); + assertEquals("[\"f2.a1\",\"f2.a2\"]", childMeta2.get("aliases")); + } + + @Test + public void testNestedRecordType() throws Exception { + Schema schema = getSchema("test_nested_record.avsc"); + Schema nestedSchema = schema.getFields().get(0).schema(); + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + GenericRecord nestedRecord = new GenericData.Record(nestedSchema); + nestedRecord.put(0, "test" + i); + nestedRecord.put(1, i); + record.put(0, nestedRecord); + + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkNestedRecordResult(schema, data, root); + } + + @Test + public void testEnumType() throws Exception { + Schema schema = getSchema("test_primitive_enum.avsc"); + List<GenericData.EnumSymbol> data = Arrays.asList( + new GenericData.EnumSymbol(schema, "SPADES"), + new GenericData.EnumSymbol(schema, "HEARTS"), + new GenericData.EnumSymbol(schema, "DIAMONDS"), + new GenericData.EnumSymbol(schema, "CLUBS"), + new GenericData.EnumSymbol(schema, "SPADES")); + + List<Integer> expectedIndices = Arrays.asList(0, 1, 2, 3, 0); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expectedIndices, vector); + + VarCharVector dictVector = (VarCharVector) config.getProvider().lookup(0).getVector(); + assertEquals(4, dictVector.getValueCount()); + + assertEquals("SPADES", dictVector.getObject(0).toString()); + assertEquals("HEARTS", dictVector.getObject(1).toString()); + assertEquals("DIAMONDS", dictVector.getObject(2).toString()); + assertEquals("CLUBS", dictVector.getObject(3).toString()); + } + + @Test + public void testIntType() throws Exception { + Schema schema = getSchema("test_primitive_int.avsc"); + List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableIntType() throws Exception { + Schema schema = getSchema("test_nullable_int.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? i : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testLongType() throws Exception { + Schema schema = getSchema("test_primitive_long.avsc"); + List<Long> data = Arrays.asList(1L, 2L, 3L, 4L, 5L); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableLongType() throws Exception { + Schema schema = getSchema("test_nullable_long.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? (long) i : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testFloatType() throws Exception { + Schema schema = getSchema("test_primitive_float.avsc"); + List<Float> data = Arrays.asList(1.1f, 2.2f, 3.3f, 4.4f, 5.5f); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableFloatType() throws Exception { + Schema schema = getSchema("test_nullable_float.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? i + 0.1f : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testDoubleType() throws Exception { + Schema schema = getSchema("test_primitive_double.avsc"); + List<Double> data = Arrays.asList(1.1, 2.2, 3.3, 4.4, 5.5); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableDoubleType() throws Exception { + Schema schema = getSchema("test_nullable_double.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? i + 0.1 : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testBytesType() throws Exception { + Schema schema = getSchema("test_primitive_bytes.avsc"); + List<ByteBuffer> data = Arrays.asList( + ByteBuffer.wrap("value1".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value2".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value3".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value4".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value5".getBytes(StandardCharsets.UTF_8))); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableBytesType() throws Exception { + Schema schema = getSchema("test_nullable_bytes.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? ByteBuffer.wrap(("test" + i).getBytes(StandardCharsets.UTF_8)) : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testBooleanType() throws Exception { + Schema schema = getSchema("test_primitive_boolean.avsc"); + List<Boolean> data = Arrays.asList(true, false, true, false, true); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(data, vector); + } + + @Test + public void testNullableBooleanType() throws Exception { + Schema schema = getSchema("test_nullable_boolean.avsc"); + + ArrayList<GenericRecord> data = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? true : null); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + checkRecordResult(schema, data, root); + } + + @Test + public void testArrayType() throws Exception { + Schema schema = getSchema("test_array.avsc"); + List<List<?>> data = Arrays.asList( + Arrays.asList("11", "222", "999"), + Arrays.asList("12222", "2333", "1000"), + Arrays.asList("1rrr", "2ggg"), + Arrays.asList("1vvv", "2bbb"), + Arrays.asList("1fff", "2")); + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkArrayResult(data, (ListVector) vector); + } + + @Test + public void testMapType() throws Exception { + Schema schema = getSchema("test_map.avsc"); + + List keys = Arrays.asList("key1", "key2", "key3", "key4", "key5", "key6"); + List vals = Arrays.asList("val1", "val2", "val3", "val4", "val5", "val6"); + + List<LinkedHashMap> data = new ArrayList<>(); + LinkedHashMap map1 = new LinkedHashMap(); + map1.put(keys.get(0), vals.get(0)); + map1.put(keys.get(1), vals.get(1)); + data.add(map1); + + LinkedHashMap map2 = new LinkedHashMap(); + map2.put(keys.get(2), vals.get(2)); + map2.put(keys.get(3), vals.get(3)); + data.add(map2); + + LinkedHashMap map3 = new LinkedHashMap(); + map3.put(keys.get(4), vals.get(4)); + map3.put(keys.get(5), vals.get(5)); + data.add(map3); + + VectorSchemaRoot root = writeAndRead(schema, data); + MapVector vector = (MapVector) root.getFieldVectors().get(0); + + checkPrimitiveResult(keys, vector.getDataVector().getChildrenFromFields().get(0)); + checkPrimitiveResult(vals, vector.getDataVector().getChildrenFromFields().get(1)); + assertEquals(0, vector.getOffsetBuffer().getInt(0)); + assertEquals(2, vector.getOffsetBuffer().getInt(1 * 4)); + assertEquals(4, vector.getOffsetBuffer().getInt(2 * 4)); + assertEquals(6, vector.getOffsetBuffer().getInt(3 * 4)); + } + + @Test + public void testFixedType() throws Exception { + Schema schema = getSchema("test_fixed.avsc"); + + List<GenericData.Fixed> data = new ArrayList<>(); + List<byte[]> expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + byte[] value = ("value" + i).getBytes(StandardCharsets.UTF_8); + expected.add(value); + GenericData.Fixed fixed = new GenericData.Fixed(schema); + fixed.bytes(value); + data.add(fixed); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + + @Test + public void testUnionType() throws Exception { + Schema schema = getSchema("test_union.avsc"); + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<Object> expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + record.put(0, i % 2 == 0 ? "test" + i : i); + expected.add(i % 2 == 0 ? "test" + i : i); + data.add(record); + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + + @Test + public void testNullableUnionType() throws Exception { + Schema schema = getSchema("test_nullable_union.avsc"); + ArrayList<GenericRecord> data = new ArrayList<>(); + ArrayList<Object> expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + GenericRecord record = new GenericData.Record(schema); + if (i % 3 == 0) { + record.put(0, "test" + i); + expected.add("test" + i); + data.add(record); + } else if (i % 3 == 1) { + record.put(0, i); + expected.add(i); + data.add(record); + } else { + record.put(0, null); + expected.add(null); + data.add(record); + } + } + + VectorSchemaRoot root = writeAndRead(schema, data); + FieldVector vector = root.getFieldVectors().get(0); + + checkPrimitiveResult(expected, vector); + } + +} diff --git a/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java new file mode 100644 index 000000000..bf695d193 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + + +public class TestWriteReadAvroRecord { + + @ClassRule + public static final TemporaryFolder TMP = new TemporaryFolder(); + + @Test + public void testWriteAndRead() throws Exception { + + File dataFile = TMP.newFile(); + Path schemaPath = Paths.get(TestWriteReadAvroRecord.class.getResource("/").getPath(), "schema", "test.avsc"); + Schema schema = new Schema.Parser().parse(schemaPath.toFile()); + + //write data to disk + GenericRecord user1 = new GenericData.Record(schema); + user1.put("name", "Alyssa"); + user1.put("favorite_number", 256); + + GenericRecord user2 = new GenericData.Record(schema); + user2.put("name", "Ben"); + user2.put("favorite_number", 7); + user2.put("favorite_color", "red"); + + DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + dataFileWriter.create(schema, dataFile); + dataFileWriter.append(user1); + dataFileWriter.append(user2); + dataFileWriter.close(); + + //read data from disk + DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema); + DataFileReader<GenericRecord> + dataFileReader = new DataFileReader<GenericRecord>(dataFile, datumReader); + List<GenericRecord> result = new ArrayList<>(); + while (dataFileReader.hasNext()) { + GenericRecord user = dataFileReader.next(); + result.add(user); + } + + assertEquals(2, result.size()); + GenericRecord deUser1 = result.get(0); + assertEquals("Alyssa", deUser1.get("name").toString()); + assertEquals(256, deUser1.get("favorite_number")); + assertEquals(null, deUser1.get("favorite_color")); + + GenericRecord deUser2 = result.get(1); + assertEquals("Ben", deUser2.get("name").toString()); + assertEquals(7, deUser2.get("favorite_number")); + assertEquals("red", deUser2.get("favorite_color").toString()); + } + +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_enum_attrs.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_enum_attrs.avsc new file mode 100644 index 000000000..afd00b8d9 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_enum_attrs.avsc @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "type": "enum", + "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"], + "name": "testEnum", + "doc" : "enum doc", + "aliases" : ["alias1", "alias2"] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_fixed_attr.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_fixed_attr.avsc new file mode 100644 index 000000000..55e504def --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_fixed_attr.avsc @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "type": "fixed", + "size": 6, + "name": "testFixed", + "doc" : "fixed doc", + "aliases" : ["alias1", "alias2"] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_record_attrs.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_record_attrs.avsc new file mode 100644 index 000000000..2e2e311a9 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/attrs/test_record_attrs.avsc @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testAttrs", + "fields": [ + { + "name" : "f0", + "type" : { + "type" : "record", + "name" : "nestedInRecord", + "doc" : "f0 doc", + "aliases" : ["f0.a1"], + "fields": [ + {"name": "f1", "type": "string", "doc": "f1 doc", "aliases" : ["f1.a1", "f1.a2"]}, + {"name": "f2", "type": "int", "doc": "f2 doc", "aliases" : ["f2.a1", "f2.a2"]} + ] + } + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_date.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_date.avsc new file mode 100644 index 000000000..f661e6506 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_date.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "int", + "logicalType" : "date" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc new file mode 100644 index 000000000..18d7d63fc --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "bytes", + "logicalType" : "decimal", + "precision": 39, + "scale": 2 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid2.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid2.avsc new file mode 100644 index 000000000..eed7bd781 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid2.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "bytes", + "logicalType" : "decimal", + "precision": 20, + "scale": -1 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid3.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid3.avsc new file mode 100644 index 000000000..1667b8aff --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid3.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "bytes", + "logicalType" : "decimal", + "precision": 20, + "scale": 40 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid4.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid4.avsc new file mode 100644 index 000000000..e1f710416 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid4.avsc @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "fixed", + "size" : 1, + "logicalType" : "decimal", + "precision": 30, + "scale": 2 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_bytes.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_bytes.avsc new file mode 100644 index 000000000..944b5d85d --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_bytes.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "bytes", + "logicalType" : "decimal", + "precision": 10, + "scale": 2 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_fixed.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_fixed.avsc new file mode 100644 index 000000000..1901f90a9 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_decimal_with_original_fixed.avsc @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "fixed", + "size" : 10, + "logicalType" : "decimal", + "precision": 10, + "scale": 2 +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_micros.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_micros.avsc new file mode 100644 index 000000000..ee7d4e937 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_micros.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "time-micros" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_millis.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_millis.avsc new file mode 100644 index 000000000..54877babc --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_time_millis.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "int", + "logicalType" : "time-millis" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_micros.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_micros.avsc new file mode 100644 index 000000000..15c0bf53d --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_micros.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "timestamp-micros" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_millis.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_millis.avsc new file mode 100644 index 000000000..822a2c360 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/logical/test_timestamp_millis.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "timestamp-millis" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_before.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_before.avsc new file mode 100644 index 000000000..e836aa768 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_before.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f1", "type": {"type" : "array", "items": "string"}}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_expected.avsc new file mode 100644 index 000000000..36e7fdfb0 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_array_expected.avsc @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base1.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base1.avsc new file mode 100644 index 000000000..5338253f4 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base1.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": {"type" : "fixed", "size":5, "name" : "fix"}}, + {"name": "f1", "type": {"type" : "enum", "name" : "enum", "symbols": ["TEST0", "TEST1"]}}, + {"name": "f2", "type": "string"}, + {"name": "f3", "type": "bytes"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base2.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base2.avsc new file mode 100644 index 000000000..50655a70e --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_base2.avsc @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "boolean"}, + {"name": "f1", "type": "int"}, + {"name": "f2", "type": "long"}, + {"name": "f3", "type": "float"}, + {"name": "f4", "type": "double"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_boolean_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_boolean_expected.avsc new file mode 100644 index 000000000..9b62e3149 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_boolean_expected.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f1", "type": "int"}, + {"name": "f2", "type": "long"}, + {"name": "f3", "type": "float"}, + {"name": "f4", "type": "double"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_bytes_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_bytes_expected.avsc new file mode 100644 index 000000000..8a1903b34 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_bytes_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": {"type" : "fixed", "size":5, "name" : "fix"}}, + {"name": "f1", "type": {"type" : "enum", "name" : "enum", "symbols": ["TEST0", "TEST1"]}}, + {"name": "f2", "type": "string"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_double_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_double_expected.avsc new file mode 100644 index 000000000..6021c4454 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_double_expected.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "boolean"}, + {"name": "f1", "type": "int"}, + {"name": "f2", "type": "long"}, + {"name": "f3", "type": "float"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_enum_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_enum_expected.avsc new file mode 100644 index 000000000..f5ed86a28 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_enum_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": {"type" : "fixed", "size":5, "name" : "fix"}}, + {"name": "f2", "type": "string"}, + {"name": "f3", "type": "bytes"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_fixed_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_fixed_expected.avsc new file mode 100644 index 000000000..5423a7977 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_fixed_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f1", "type": {"type" : "enum", "name" : "enum", "symbols": ["TEST0", "TEST1"]}}, + {"name": "f2", "type": "string"}, + {"name": "f3", "type": "bytes"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_float_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_float_expected.avsc new file mode 100644 index 000000000..dea106331 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_float_expected.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "boolean"}, + {"name": "f1", "type": "int"}, + {"name": "f2", "type": "long"}, + {"name": "f4", "type": "double"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_int_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_int_expected.avsc new file mode 100644 index 000000000..53d4f1025 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_int_expected.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "boolean"}, + {"name": "f2", "type": "long"}, + {"name": "f3", "type": "float"}, + {"name": "f4", "type": "double"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_long_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_long_expected.avsc new file mode 100644 index 000000000..bf16601dd --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_long_expected.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "boolean"}, + {"name": "f1", "type": "int"}, + {"name": "f3", "type": "float"}, + {"name": "f4", "type": "double"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_before.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_before.avsc new file mode 100644 index 000000000..8cbb1a1d7 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_before.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f1", "type": {"type" : "map", "values": "string"}}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_expected.avsc new file mode 100644 index 000000000..36e7fdfb0 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_map_expected.avsc @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_multi_fields_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_multi_fields_expected.avsc new file mode 100644 index 000000000..b5d637b1d --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_multi_fields_expected.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testSkip", + "fields": [ + {"name": "f0", "type": "string"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_before.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_before.avsc new file mode 100644 index 000000000..7aee92b92 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_before.avsc @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + { + "name" : "f0", + "type" : { + "type" : "record", + "name" : "nestedInRecord", + "fields": [ + {"name": "f00", "type": "string"}, + {"name": "f01", "type": "int"} + ] + } + }, + { + "name" : "f1", "type" : "int" + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_expected.avsc new file mode 100644 index 000000000..3e2495203 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_record_expected.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + { "name" : "f1", "type" : "int"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_second_level_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_second_level_expected.avsc new file mode 100644 index 000000000..f3b7f8c09 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_second_level_expected.avsc @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testSkipNested", + "fields": [ + { + "name" : "nested", + "type" : { + "type" : "record", + "name" : "nestedInRecord", + "fields": [ + {"name": "f1", "type": "int"} + ] + } + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_single_field_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_single_field_expected.avsc new file mode 100644 index 000000000..553525847 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_single_field_expected.avsc @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testSkip", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_string_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_string_expected.avsc new file mode 100644 index 000000000..2d2c08174 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_string_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": {"type" : "fixed", "size":5, "name" : "fix"}}, + {"name": "f1", "type": {"type" : "enum", "name" : "enum", "symbols": ["TEST0", "TEST1"]}}, + {"name": "f3", "type": "bytes"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_third_level_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_third_level_expected.avsc new file mode 100644 index 000000000..6f42da893 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_third_level_expected.avsc @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "firstLevel", + "fields": [ + { + "name" : "f0", + "type" : { + "type" : "record", + "name" : "secondLevel", + "fields": [ + { + "name" : "f0", + "type" : { + "type" : "record", + "name" : "thirdLevel", + "fields" : [ + {"name": "f1", "type": "int"}, + {"name": "f0", "type": "string"}, + {"name": "f2", "type": "boolean"} + ] + } + } + ] + } + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_before.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_before.avsc new file mode 100644 index 000000000..fc1105911 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_before.avsc @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": ["string"]}, + {"name": "f1", "type": ["string", "null"]}, + {"name": "f2", "type": ["string", "int"]}, + {"name": "f3", "type": "int"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_multi_fields_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_multi_fields_expected.avsc new file mode 100644 index 000000000..308e027a2 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_multi_fields_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": ["string"]}, + {"name": "f1", "type": ["string", "null"]}, + {"name": "f3", "type": "int"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_nullable_field_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_nullable_field_expected.avsc new file mode 100644 index 000000000..cbc83e566 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_nullable_field_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f0", "type": ["string"]}, + {"name": "f2", "type": ["string", "int"]}, + {"name": "f3", "type": "int"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_one_field_expected.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_one_field_expected.avsc new file mode 100644 index 000000000..0f72fb432 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/skip/test_skip_union_one_field_expected.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "test", + "fields": [ + {"name": "f1", "type": ["string", "null"]}, + {"name": "f2", "type": ["string", "int"]}, + {"name": "f3", "type": ["string", "int"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test.avsc new file mode 100644 index 000000000..92c0873de --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_array.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_array.avsc new file mode 100644 index 000000000..5b75a4031 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_array.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "array", + "items": "string", + "name": "testArray" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_fixed.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_fixed.avsc new file mode 100644 index 000000000..a4d96e9ab --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_fixed.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "fixed", + "size": 6, + "name": "testFixed" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_large_data.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_large_data.avsc new file mode 100644 index 000000000..f784ae623 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_large_data.avsc @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testLargeData", + "fields": [ + { + "name": "f0", + "type": { + "name" : "f0", + "type" : "enum", + "symbols" : ["value1", "value2", "value3", "value4", "value5"] + } + }, + { + "name" : "f1", + "type" : { + "type" : "record", + "name" : "nestedRecord", + "fields": [ + {"name": "f1_0", "type": "string"}, + {"name": "f1_1", "type": "int"} + ] + } + }, + + {"name": "f2", "type": "string"}, + {"name": "f3", "type": "int"}, + {"name": "f4", "type": "boolean"}, + {"name": "f5", "type": "float"}, + {"name": "f6", "type": "double"}, + {"name": "f7", "type": "bytes"}, + {"name": "f8", "type": ["string", "int"]}, + { + "name": "f9", + "type": { + "name" : "f9", + "type" : "array", + "items" : "string" + } + }, + { + "name": "f10", + "type": { + "name" : "f10", + "type" : "map", + "values" : "string" + } + }, + { + "name": "f11", + "type": { + "type" : "fixed", + "name" : "f11", + "size" : 5 + } + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_map.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_map.avsc new file mode 100644 index 000000000..0dfa3a595 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_map.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "map", + "values": "string", + "name": "testMap" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nested_record.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nested_record.avsc new file mode 100644 index 000000000..29dddfd1a --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nested_record.avsc @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testNestedRecord", + "fields": [ + { + "name" : "f0", + "type" : { + "type" : "record", + "name" : "nestedInRecord", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f1", "type": "int"} + ] + } + } + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_boolean.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_boolean.avsc new file mode 100644 index 000000000..62af1a85d --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_boolean.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableBoolean", + "fields": [ + {"name": "f0", "type": ["null", "boolean"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_bytes.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_bytes.avsc new file mode 100644 index 000000000..002bc7ce2 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_bytes.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableBytes", + "fields": [ + {"name": "f0", "type": ["null", "bytes"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_double.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_double.avsc new file mode 100644 index 000000000..642b7aa16 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_double.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableDouble", + "fields": [ + {"name": "f0", "type": ["null", "double"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_float.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_float.avsc new file mode 100644 index 000000000..dff285909 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_float.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableFloat", + "fields": [ + {"name": "f0", "type": ["null", "float"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_int.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_int.avsc new file mode 100644 index 000000000..abb2fc48a --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_int.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableInt", + "fields": [ + {"name": "f0", "type": ["null", "int"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_long.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_long.avsc new file mode 100644 index 000000000..0624d2737 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_long.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableLong", + "fields": [ + {"name": "f0", "type": ["null", "long"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_string.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_string.avsc new file mode 100644 index 000000000..347808ce6 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_string.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "nullableString", + "fields": [ + {"name": "f0", "type": ["null", "string"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_union.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_union.avsc new file mode 100644 index 000000000..af94812d7 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_nullable_union.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testNullableUnions", + "fields": [ + {"name": "f0", "type": ["string", "int", "null"]} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_boolean.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_boolean.avsc new file mode 100644 index 000000000..7652ce723 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_boolean.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "boolean", + "name": "TestBoolean" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_bytes.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_bytes.avsc new file mode 100644 index 000000000..5102430b6 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_bytes.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "bytes", + "name": "TestBytes" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_double.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_double.avsc new file mode 100644 index 000000000..d1ae0b605 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_double.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "double", + "name": "TestDouble" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_enum.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_enum.avsc new file mode 100644 index 000000000..bd8df6102 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_enum.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "enum", + "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"], + "name": "testEnum" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_float.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_float.avsc new file mode 100644 index 000000000..675d1090d --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_float.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "float", + "name": "TestFloat" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_int.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_int.avsc new file mode 100644 index 000000000..8fc848828 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_int.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "int", + "name": "TestInt" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_long.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_long.avsc new file mode 100644 index 000000000..b9706107c --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_long.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "long", + "name": "TestLong" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_string.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_string.avsc new file mode 100644 index 000000000..b4a89a7f6 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_primitive_string.avsc @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "string", + "name": "TestString" +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_record.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_record.avsc new file mode 100644 index 000000000..e83cf1180 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_record.avsc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testRecord", + "fields": [ + {"name": "f0", "type": "string"}, + {"name": "f1", "type": "int"}, + {"name": "f2", "type": "boolean"} + ] +} diff --git a/src/arrow/java/adapter/avro/src/test/resources/schema/test_union.avsc b/src/arrow/java/adapter/avro/src/test/resources/schema/test_union.avsc new file mode 100644 index 000000000..f181e36e3 --- /dev/null +++ b/src/arrow/java/adapter/avro/src/test/resources/schema/test_union.avsc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "type": "record", + "name": "testUnions", + "fields": [ + {"name": "f0", "type": ["string", "int"]} + ] +} |