diff options
Diffstat (limited to 'src/arrow/ruby/red-arrow/test')
93 files changed, 13228 insertions, 0 deletions
diff --git a/src/arrow/ruby/red-arrow/test/fixture/TestOrcFile.test1.orc b/src/arrow/ruby/red-arrow/test/fixture/TestOrcFile.test1.orc Binary files differnew file mode 100644 index 000000000..4fb0beff8 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/TestOrcFile.test1.orc diff --git a/src/arrow/ruby/red-arrow/test/fixture/float-integer.csv b/src/arrow/ruby/red-arrow/test/fixture/float-integer.csv new file mode 100644 index 000000000..5eae562bc --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/float-integer.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +score +2.9 +10 +-1.1 diff --git a/src/arrow/ruby/red-arrow/test/fixture/integer-float.csv b/src/arrow/ruby/red-arrow/test/fixture/integer-float.csv new file mode 100644 index 000000000..da7614199 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/integer-float.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +score +10 +2.9 +-1.1 diff --git a/src/arrow/ruby/red-arrow/test/fixture/null-with-double-quote.csv b/src/arrow/ruby/red-arrow/test/fixture/null-with-double-quote.csv new file mode 100644 index 000000000..d84545928 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/null-with-double-quote.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name,score +alice,10 +bob,"" +chris,-1 diff --git a/src/arrow/ruby/red-arrow/test/fixture/null-without-double-quote.csv b/src/arrow/ruby/red-arrow/test/fixture/null-without-double-quote.csv new file mode 100644 index 000000000..c91c8880a --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/null-without-double-quote.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name,score +alice,10 +bob, +chris,-1 diff --git a/src/arrow/ruby/red-arrow/test/fixture/with-header-float.csv b/src/arrow/ruby/red-arrow/test/fixture/with-header-float.csv new file mode 100644 index 000000000..f62fc00b6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/with-header-float.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name,score +alice,10.1 +bob,29.2 +chris,-1.3 diff --git a/src/arrow/ruby/red-arrow/test/fixture/with-header.csv b/src/arrow/ruby/red-arrow/test/fixture/with-header.csv new file mode 100644 index 000000000..a93fc5aec --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/with-header.csv @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name,score +alice,10 +bob,29 +chris,-1 diff --git a/src/arrow/ruby/red-arrow/test/fixture/without-header-float.csv b/src/arrow/ruby/red-arrow/test/fixture/without-header-float.csv new file mode 100644 index 000000000..584a20996 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/without-header-float.csv @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +alice,10.1 +bob,29.2 +chris,-1.3 diff --git a/src/arrow/ruby/red-arrow/test/fixture/without-header.csv b/src/arrow/ruby/red-arrow/test/fixture/without-header.csv new file mode 100644 index 000000000..1f775eae4 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/fixture/without-header.csv @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +alice,10 +bob,29 +chris,-1 diff --git a/src/arrow/ruby/red-arrow/test/helper.rb b/src/arrow/ruby/red-arrow/test/helper.rb new file mode 100644 index 000000000..29e5f9cbc --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/helper.rb @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "arrow" + +require "fiddle" +require "pathname" +require "tempfile" +require "zlib" + +require "test-unit" + +require_relative "helper/fixture" +require_relative "helper/omittable" diff --git a/src/arrow/ruby/red-arrow/test/helper/fixture.rb b/src/arrow/ruby/red-arrow/test/helper/fixture.rb new file mode 100644 index 000000000..24445a7e4 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/helper/fixture.rb @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Helper + module Fixture + def fixture_dir + Pathname.new(__dir__).join("..", "fixture").expand_path + end + + def fixture_path(*components) + fixture_dir.join(*components) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/helper/omittable.rb b/src/arrow/ruby/red-arrow/test/helper/omittable.rb new file mode 100644 index 000000000..a1c0334b6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/helper/omittable.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Helper + module Omittable + def require_gi_bindings(major, minor, micro) + return if GLib.check_binding_version?(major, minor, micro) + message = + "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " + + GLib::BINDING_VERSION.join(".") + omit(message) + end + + def require_gi(major, minor, micro) + return if GObjectIntrospection::Version.or_later?(major, minor, micro) + message = + "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " + + GObjectIntrospection::Version::STRING + omit(message) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-basic-arrays.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-basic-arrays.rb new file mode 100644 index 000000000..c80020666 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-basic-arrays.rb @@ -0,0 +1,365 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsBasicArraysTests + def test_null + records = [ + [nil], + [nil], + [nil], + [nil], + ] + target = build({column: :null}, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [true], + [nil], + [false], + ] + target = build({column: :boolean}, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [-(2 ** 7)], + [nil], + [(2 ** 7) - 1], + ] + target = build({column: :int8}, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [0], + [nil], + [(2 ** 8) - 1], + ] + target = build({column: :uint8}, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [-(2 ** 15)], + [nil], + [(2 ** 15) - 1], + ] + target = build({column: :int16}, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [0], + [nil], + [(2 ** 16) - 1], + ] + target = build({column: :uint16}, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [-(2 ** 31)], + [nil], + [(2 ** 31) - 1], + ] + target = build({column: :int32}, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [0], + [nil], + [(2 ** 32) - 1], + ] + target = build({column: :uint32}, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [-(2 ** 63)], + [nil], + [(2 ** 63) - 1], + ] + target = build({column: :int64}, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [0], + [nil], + [(2 ** 64) - 1], + ] + target = build({column: :uint64}, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [-1.0], + [nil], + [1.0], + ] + target = build({column: :float}, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [-1.0], + [nil], + [1.0], + ] + target = build({column: :double}, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + ["\x00".b], + [nil], + ["\xff".b], + ] + target = build({column: :binary}, records) + assert_equal(records, target.raw_records) + end + + def test_tring + records = [ + ["Ruby"], + [nil], + ["\u3042"], # U+3042 HIRAGANA LETTER A + ] + target = build({column: :string}, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [Date.new(1960, 1, 1)], + [nil], + [Date.new(2017, 8, 23)], + ] + target = build({column: :date32}, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [DateTime.new(1960, 1, 1, 2, 9, 30)], + [nil], + [DateTime.new(2017, 8, 23, 14, 57, 2)], + ] + target = build({column: :date64}, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [Time.parse("1960-01-01T02:09:30Z")], + [nil], + [Time.parse("2017-08-23T14:57:02Z")], + ] + target = build({ + column: { + type: :timestamp, + unit: :second, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [Time.parse("1960-01-01T02:09:30.123Z")], + [nil], + [Time.parse("2017-08-23T14:57:02.987Z")], + ] + target = build({ + column: { + type: :timestamp, + unit: :milli, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [Time.parse("1960-01-01T02:09:30.123456Z")], + [nil], + [Time.parse("2017-08-23T14:57:02.987654Z")], + ] + target = build({ + column: { + type: :timestamp, + unit: :micro, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [Time.parse("1960-01-01T02:09:30.123456789Z")], + [nil], + [Time.parse("2017-08-23T14:57:02.987654321Z")], + ] + target = build({ + column: { + type: :timestamp, + unit: :nano, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + [Arrow::Time.new(unit, 60 * 10)], # 00:10:00 + [nil], + [Arrow::Time.new(unit, 60 * 60 * 2 + 9)], # 02:00:09 + ] + target = build({ + column: { + type: :time32, + unit: :second, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + [Arrow::Time.new(unit, (60 * 10) * 1000 + 123)], # 00:10:00.123 + [nil], + [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987)], # 02:00:09.987 + ] + target = build({ + column: { + type: :time32, + unit: :milli, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + # 00:10:00.123456 + [Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)], + [nil], + # 02:00:09.987654 + [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654)], + ] + target = build({ + column: { + type: :time64, + unit: :micro, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + # 00:10:00.123456789 + [Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)], + [nil], + # 02:00:09.987654321 + [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321)], + ] + target = build({ + column: { + type: :time64, + unit: :nano, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [BigDecimal("92.92")], + [nil], + [BigDecimal("29.29")], + ] + target = build({ + column: { + type: :decimal128, + precision: 8, + scale: 2, + } + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [BigDecimal("92.92")], + [nil], + [BigDecimal("29.29")], + ] + target = build({ + column: { + type: :decimal256, + precision: 38, + scale: 2, + } + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase + include RawRecordsBasicArraysTests + + def build(schema, records) + Arrow::RecordBatch.new(schema, records) + end +end + +class RawRecordsTableBasicArraysTest < Test::Unit::TestCase + include RawRecordsBasicArraysTests + + def build(schema, records) + Arrow::Table.new(schema, records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-dense-union-array.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-dense-union-array.rb new file mode 100644 index 000000000..8d94a77fe --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-dense-union-array.rb @@ -0,0 +1,494 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsDenseUnionArrayTests + def build_schema(type, type_codes) + field_description = {} + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :dense_union, + fields: [ + field_description.merge(name: "0"), + field_description.merge(name: "1"), + ], + type_codes: type_codes, + }, + } + end + + # TODO: Use Arrow::RecordBatch.new(build_schema(type, type_codes), records) + def build_record_batch(type, records) + type_codes = [0, 1] + schema = Arrow::Schema.new(build_schema(type, type_codes)) + type_ids = [] + offsets = [] + arrays = schema.fields[0].data_type.fields.collect do |field| + sub_schema = Arrow::Schema.new([field]) + sub_records = [] + records.each do |record| + column = record[0] + next if column.nil? + next unless column.key?(field.name) + sub_records << [column[field.name]] + end + sub_record_batch = Arrow::RecordBatch.new(sub_schema, + sub_records) + sub_record_batch.columns[0].data + end + records.each do |record| + column = record[0] + if column.key?("0") + type_id = type_codes[0] + type_ids << type_id + offsets << (type_ids.count(type_id) - 1) + elsif column.key?("1") + type_id = type_codes[1] + type_ids << type_id + offsets << (type_ids.count(type_id) - 1) + end + end + union_array = Arrow::DenseUnionArray.new(schema.fields[0].data_type, + Arrow::Int8Array.new(type_ids), + Arrow::Int32Array.new(offsets), + arrays) + schema = Arrow::Schema.new(column: union_array.value_data_type) + Arrow::RecordBatch.new(schema, + records.size, + [union_array]) + end + + def test_null + records = [ + [{"0" => nil}], + ] + target = build(:null, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [{"0" => true}], + [{"1" => nil}], + ] + target = build(:boolean, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [{"0" => -(2 ** 7)}], + [{"1" => nil}], + ] + target = build(:int8, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [{"0" => (2 ** 8) - 1}], + [{"1" => nil}], + ] + target = build(:uint8, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [{"0" => -(2 ** 15)}], + [{"1" => nil}], + ] + target = build(:int16, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [{"0" => (2 ** 16) - 1}], + [{"1" => nil}], + ] + target = build(:uint16, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [{"0" => -(2 ** 31)}], + [{"1" => nil}], + ] + target = build(:int32, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [{"0" => (2 ** 32) - 1}], + [{"1" => nil}], + ] + target = build(:uint32, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [{"0" => -(2 ** 63)}], + [{"1" => nil}], + ] + target = build(:int64, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [{"0" => (2 ** 64) - 1}], + [{"1" => nil}], + ] + target = build(:uint64, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [{"0" => -1.0}], + [{"1" => nil}], + ] + target = build(:float, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [{"0" => -1.0}], + [{"1" => nil}], + ] + target = build(:double, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + [{"0" => "\xff".b}], + [{"1" => nil}], + ] + target = build(:binary, records) + assert_equal(records, target.raw_records) + end + + def test_string + records = [ + [{"0" => "Ruby"}], + [{"1" => nil}], + ] + target = build(:string, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [{"0" => Date.new(1960, 1, 1)}], + [{"1" => nil}], + ] + target = build(:date32, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}], + [{"1" => nil}], + ] + target = build(:date64, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + # 00:10:00 + [{"0" => Arrow::Time.new(unit, 60 * 10)}], + [{"1" => nil}], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + # 00:10:00.123 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}], + [{"1" => nil}], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + # 00:10:00.123456 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}], + [{"1" => nil}], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + # 00:10:00.123456789 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}], + [{"1" => nil}], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [{"0" => BigDecimal("92.92")}], + [{"1" => nil}], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [{"0" => BigDecimal("92.92")}], + [{"1" => nil}], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_list + records = [ + [{"0" => [true, nil, false]}], + [{"1" => nil}], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_struct + records = [ + [{"0" => {"sub_field" => true}}], + [{"1" => nil}], + [{"0" => {"sub_field" => nil}}], + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_map + records = [ + [{"0" => {"key1" => true, "key2" => nil}}], + [{"1" => nil}], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + records = [ + [{"0" => {"field1" => true}}], + [{"1" => nil}], + [{"0" => {"field2" => nil}}], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + records = [ + [{"0" => {"field1" => true}}], + [{"1" => nil}], + [{"0" => {"field2" => nil}}], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + records = [ + [{"0" => "Ruby"}], + [{"1" => nil}], + [{"0" => "GLib"}], + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchDenseUnionArrayTest < Test::Unit::TestCase + include RawRecordsDenseUnionArrayTests + + def build(type, records) + build_record_batch(type, records) + end +end + +class RawRecordsTableDenseUnionArrayTest < Test::Unit::TestCase + include RawRecordsDenseUnionArrayTests + + def build(type, records) + build_record_batch(type, records).to_table + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-list-array.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-list-array.rb new file mode 100644 index 000000000..6d7d4c079 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-list-array.rb @@ -0,0 +1,571 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsListArrayTests + def build_schema(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :list, + field: field_description, + }, + } + end + + def test_null + records = [ + [[nil, nil, nil]], + [nil], + ] + target = build(:null, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [[true, nil, false]], + [nil], + ] + target = build(:boolean, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [[-(2 ** 7), nil, (2 ** 7) - 1]], + [nil], + ] + target = build(:int8, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [[0, nil, (2 ** 8) - 1]], + [nil], + ] + target = build(:uint8, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [[-(2 ** 15), nil, (2 ** 15) - 1]], + [nil], + ] + target = build(:int16, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [[0, nil, (2 ** 16) - 1]], + [nil], + ] + target = build(:uint16, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [[-(2 ** 31), nil, (2 ** 31) - 1]], + [nil], + ] + target = build(:int32, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [[0, nil, (2 ** 32) - 1]], + [nil], + ] + target = build(:uint32, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [[-(2 ** 63), nil, (2 ** 63) - 1]], + [nil], + ] + target = build(:int64, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [[0, nil, (2 ** 64) - 1]], + [nil], + ] + target = build(:uint64, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [[-1.0, nil, 1.0]], + [nil], + ] + target = build(:float, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [[-1.0, nil, 1.0]], + [nil], + ] + target = build(:double, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + [["\x00".b, nil, "\xff".b]], + [nil], + ] + target = build(:binary, records) + assert_equal(records, target.raw_records) + end + + def test_string + records = [ + [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + ], + ], + [nil], + ] + target = build(:string, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + ], + ], + [nil], + ] + target = build(:date32, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + ], + ], + [nil], + ] + target = build(:date64, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + ], + [nil], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + ], + [nil], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_struct + records = [ + [ + [ + {"field" => true}, + nil, + {"field" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_map + records = [ + [ + [ + {"key1" => true, "key2" => nil}, + nil, + ], + ], + [nil], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_sparse + omit("Need to add support for SparseUnionArrayBuilder") + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dense + omit("Need to add support for DenseUnionArrayBuilder") + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + records = [ + [ + [ + "Ruby", + nil, + "GLib", + ], + ], + [nil], + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase + include RawRecordsListArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end +end + +class RawRecordsTableListArrayTest < Test::Unit::TestCase + include RawRecordsListArrayTests + + def build(type, records) + Arrow::Table.new(build_schema(type), records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-map-array.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-map-array.rb new file mode 100644 index 000000000..c5abb7d77 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-map-array.rb @@ -0,0 +1,441 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsMapArrayTests + def build_schema(type) + { + column: { + type: :map, + key: :string, + item: type + }, + } + end + + def test_null + records = [ + [{"key1" => nil}], + [nil], + ] + target = build(:null, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [{"key1" => true, "key2" => nil}], + [nil], + ] + target = build(:boolean, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [{"key1" => -(2 ** 7), "key2" => nil}], + [nil], + ] + target = build(:int8, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [{"key1" => (2 ** 8) - 1, "key2" => nil}], + [nil], + ] + target = build(:uint8, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [{"key1" => -(2 ** 15), "key2" => nil}], + [nil], + ] + target = build(:int16, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [{"key1" => (2 ** 16) - 1, "key2" => nil}], + [nil], + ] + target = build(:uint16, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [{"key1" => -(2 ** 31), "key2" => nil}], + [nil], + ] + target = build(:int32, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [{"key1" => (2 ** 32) - 1, "key2" => nil}], + [nil], + ] + target = build(:uint32, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [{"key1" => -(2 ** 63), "key2" => nil}], + [nil], + ] + target = build(:int64, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [{"key1" => (2 ** 64) - 1, "key2" => nil}], + [nil], + ] + target = build(:uint64, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [{"key1" => -1.0, "key2" => nil}], + [nil], + ] + target = build(:float, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [{"key1" => -1.0, "key2" => nil}], + [nil], + ] + target = build(:double, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + [{"key1" => "\xff".b, "key2" => nil}], + [nil], + ] + target = build(:binary, records) + assert_equal(records, target.raw_records) + end + + def test_string + records = [ + [{"key1" => "Ruby", "key2" => nil}], + [nil], + ] + target = build(:string, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [{"key1" => Date.new(1960, 1, 1), "key2" => nil}], + [nil], + ] + target = build(:date32, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [{"key1" => DateTime.new(1960, 1, 1, 2, 9, 30), "key2" => nil}], + [nil], + ] + target = build(:date64, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [{"key1" => Time.parse("1960-01-01T02:09:30Z"), "key2" => nil}], + [nil], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [{"key1" => Time.parse("1960-01-01T02:09:30.123Z"), "key2" => nil}], + [nil], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [{"key1" => Time.parse("1960-01-01T02:09:30.123456Z"), "key2" => nil}], + [nil], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [{"key1" => Time.parse("1960-01-01T02:09:30.123456789Z"), "key2" => nil}], + [nil], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + # 00:10:00 + [{"key1" => Arrow::Time.new(unit, 60 * 10), "key2" => nil}], + [nil], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + # 00:10:00.123 + [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123), "key2" => nil}], + [nil], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + # 00:10:00.123456 + [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), "key2" => nil}], + [nil], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + # 00:10:00.123456789 + [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), "key2" => nil}], + [nil], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [{"key1" => BigDecimal("92.92"), "key2" => nil}], + [nil], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [{"key1" => BigDecimal("92.92"), "key2" => nil}], + [nil], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_list + records = [ + [{"key1" => [true, nil, false], "key2" => nil}], + [nil], + ] + target = build({ + type: :list, + field: { + name: :element, + type: :boolean, + }, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_struct + records = [ + [{"key1" => {"field" => true}, "key2" => nil, "key3" => {"field" => nil}}], + [nil], + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_map + records = [ + [{"key1" => {"sub_key1" => true, "sub_key2" => nil}, "key2" => nil}], + [nil], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + records = [ + [{"key1" => {"field" => true, "key2" => nil, "key3" => {"field" => nil}}}], + [nil], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + records = [ + [{"key1" => {"field1" => true}, "key2" => nil, "key3" => {"field2" => nil}}], + [nil], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + records = [ + [{"key1" => "Ruby", "key2" => nil, "key3" => "GLib"}], + [nil], + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchMapArrayTest < Test::Unit::TestCase + include RawRecordsMapArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end +end + +class RawRecordsTableMapArrayTest < Test::Unit::TestCase + include RawRecordsMapArrayTests + + def build(type, records) + Arrow::Table.new(build_schema(type), records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-multiple-columns.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-multiple-columns.rb new file mode 100644 index 000000000..50dff67ce --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-multiple-columns.rb @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsMultipleColumnsTests + def test_3_elements + records = [ + [true, nil, "Ruby"], + [nil, 0, "GLib"], + [false, 2 ** 8 - 1, nil], + ] + target = build([ + {name: :column0, type: :boolean}, + {name: :column1, type: :uint8}, + {name: :column2, type: :string}, + ], + records) + assert_equal(records, target.raw_records) + end + + def test_4_elements + records = [ + [true, nil, "Ruby", -(2 ** 63)], + [nil, 0, "GLib", nil], + [false, 2 ** 8 - 1, nil, (2 ** 63) - 1], + ] + target = build([ + {name: :column0, type: :boolean}, + {name: :column1, type: :uint8}, + {name: :column2, type: :string}, + {name: :column3, type: :int64}, + ], + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchMultipleColumnsTest < Test::Unit::TestCase + include RawRecordsMultipleColumnsTests + + def build(schema, records) + Arrow::RecordBatch.new(schema, records) + end +end + +class RawRecordsTableMultipleColumnsTest < Test::Unit::TestCase + include RawRecordsMultipleColumnsTests + + def build(schema, records) + Arrow::Table.new(schema, records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-sparse-union-array.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-sparse-union-array.rb new file mode 100644 index 000000000..415401216 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-sparse-union-array.rb @@ -0,0 +1,484 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsSparseUnionArrayTests + def build_schema(type, type_codes) + field_description = {} + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :sparse_union, + fields: [ + field_description.merge(name: "0"), + field_description.merge(name: "1"), + ], + type_codes: type_codes, + }, + } + end + + # TODO: Use Arrow::RecordBatch.new(build_schema(type, type_codes), records) + def build_record_batch(type, records) + type_codes = [0, 1] + schema = Arrow::Schema.new(build_schema(type, type_codes)) + type_ids = [] + arrays = schema.fields[0].data_type.fields.collect do |field| + sub_schema = Arrow::Schema.new([field]) + sub_records = records.collect do |record| + [record[0].nil? ? nil : record[0][field.name]] + end + sub_record_batch = Arrow::RecordBatch.new(sub_schema, + sub_records) + sub_record_batch.columns[0].data + end + records.each do |record| + column = record[0] + if column.key?("0") + type_ids << type_codes[0] + elsif column.key?("1") + type_ids << type_codes[1] + end + end + union_array = Arrow::SparseUnionArray.new(schema.fields[0].data_type, + Arrow::Int8Array.new(type_ids), + arrays) + schema = Arrow::Schema.new(column: union_array.value_data_type) + Arrow::RecordBatch.new(schema, + records.size, + [union_array]) + end + + def test_null + records = [ + [{"0" => nil}], + ] + target = build(:null, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [{"0" => true}], + [{"1" => nil}], + ] + target = build(:boolean, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [{"0" => -(2 ** 7)}], + [{"1" => nil}], + ] + target = build(:int8, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [{"0" => (2 ** 8) - 1}], + [{"1" => nil}], + ] + target = build(:uint8, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [{"0" => -(2 ** 15)}], + [{"1" => nil}], + ] + target = build(:int16, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [{"0" => (2 ** 16) - 1}], + [{"1" => nil}], + ] + target = build(:uint16, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [{"0" => -(2 ** 31)}], + [{"1" => nil}], + ] + target = build(:int32, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [{"0" => (2 ** 32) - 1}], + [{"1" => nil}], + ] + target = build(:uint32, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [{"0" => -(2 ** 63)}], + [{"1" => nil}], + ] + target = build(:int64, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [{"0" => (2 ** 64) - 1}], + [{"1" => nil}], + ] + target = build(:uint64, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [{"0" => -1.0}], + [{"1" => nil}], + ] + target = build(:float, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [{"0" => -1.0}], + [{"1" => nil}], + ] + target = build(:double, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + [{"0" => "\xff".b}], + [{"1" => nil}], + ] + target = build(:binary, records) + assert_equal(records, target.raw_records) + end + + def test_string + records = [ + [{"0" => "Ruby"}], + [{"1" => nil}], + ] + target = build(:string, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [{"0" => Date.new(1960, 1, 1)}], + [{"1" => nil}], + ] + target = build(:date32, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}], + [{"1" => nil}], + ] + target = build(:date64, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}], + [{"1" => nil}], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + # 00:10:00 + [{"0" => Arrow::Time.new(unit, 60 * 10)}], + [{"1" => nil}], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + # 00:10:00.123 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}], + [{"1" => nil}], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + # 00:10:00.123456 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}], + [{"1" => nil}], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + # 00:10:00.123456789 + [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}], + [{"1" => nil}], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [{"0" => BigDecimal("92.92")}], + [{"1" => nil}], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [{"0" => BigDecimal("92.92")}], + [{"1" => nil}], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_list + records = [ + [{"0" => [true, nil, false]}], + [{"1" => nil}], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_struct + records = [ + [{"0" => {"sub_field" => true}}], + [{"1" => nil}], + [{"0" => {"sub_field" => nil}}], + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_map + records = [ + [{"0" => {"key1" => true, "key2" => nil}}], + [{"1" => nil}], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + records = [ + [{"0" => {"field1" => true}}], + [{"1" => nil}], + [{"0" => {"field2" => nil}}], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + records = [ + [{"0" => {"field1" => true}}], + [{"1" => nil}], + [{"0" => {"field2" => nil}}], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + records = [ + [{"0" => "Ruby"}], + [{"1" => nil}], + [{"0" => "GLib"}], + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchSparseUnionArrayTest < Test::Unit::TestCase + include RawRecordsSparseUnionArrayTests + + def build(type, records) + build_record_batch(type, records) + end +end + +class RawRecordsTableSparseUnionArrayTest < Test::Unit::TestCase + include RawRecordsSparseUnionArrayTests + + def build(type, records) + build_record_batch(type, records).to_table + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-struct-array.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-struct-array.rb new file mode 100644 index 000000000..6c01facf8 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-struct-array.rb @@ -0,0 +1,485 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsStructArrayTests + def build_schema(type) + field_description = { + name: :field, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :struct, + fields: [ + field_description, + ], + }, + } + end + + def test_null + records = [ + [{"field" => nil}], + [nil], + ] + target = build(:null, records) + assert_equal(records, target.raw_records) + end + + def test_boolean + records = [ + [{"field" => true}], + [nil], + [{"field" => nil}], + ] + target = build(:boolean, records) + assert_equal(records, target.raw_records) + end + + def test_int8 + records = [ + [{"field" => -(2 ** 7)}], + [nil], + [{"field" => nil}], + ] + target = build(:int8, records) + assert_equal(records, target.raw_records) + end + + def test_uint8 + records = [ + [{"field" => (2 ** 8) - 1}], + [nil], + [{"field" => nil}], + ] + target = build(:uint8, records) + assert_equal(records, target.raw_records) + end + + def test_int16 + records = [ + [{"field" => -(2 ** 15)}], + [nil], + [{"field" => nil}], + ] + target = build(:int16, records) + assert_equal(records, target.raw_records) + end + + def test_uint16 + records = [ + [{"field" => (2 ** 16) - 1}], + [nil], + [{"field" => nil}], + ] + target = build(:uint16, records) + assert_equal(records, target.raw_records) + end + + def test_int32 + records = [ + [{"field" => -(2 ** 31)}], + [nil], + [{"field" => nil}], + ] + target = build(:int32, records) + assert_equal(records, target.raw_records) + end + + def test_uint32 + records = [ + [{"field" => (2 ** 32) - 1}], + [nil], + [{"field" => nil}], + ] + target = build(:uint32, records) + assert_equal(records, target.raw_records) + end + + def test_int64 + records = [ + [{"field" => -(2 ** 63)}], + [nil], + [{"field" => nil}], + ] + target = build(:int64, records) + assert_equal(records, target.raw_records) + end + + def test_uint64 + records = [ + [{"field" => (2 ** 64) - 1}], + [nil], + [{"field" => nil}], + ] + target = build(:uint64, records) + assert_equal(records, target.raw_records) + end + + def test_float + records = [ + [{"field" => -1.0}], + [nil], + [{"field" => nil}], + ] + target = build(:float, records) + assert_equal(records, target.raw_records) + end + + def test_double + records = [ + [{"field" => -1.0}], + [nil], + [{"field" => nil}], + ] + target = build(:double, records) + assert_equal(records, target.raw_records) + end + + def test_binary + records = [ + [{"field" => "\xff".b}], + [nil], + [{"field" => nil}], + ] + target = build(:binary, records) + assert_equal(records, target.raw_records) + end + + def test_string + records = [ + [{"field" => "Ruby"}], + [nil], + [{"field" => nil}], + ] + target = build(:string, records) + assert_equal(records, target.raw_records) + end + + def test_date32 + records = [ + [{"field" => Date.new(1960, 1, 1)}], + [nil], + [{"field" => nil}], + ] + target = build(:date32, records) + assert_equal(records, target.raw_records) + end + + def test_date64 + records = [ + [{"field" => DateTime.new(1960, 1, 1, 2, 9, 30)}], + [nil], + [{"field" => nil}], + ] + target = build(:date64, records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_second + records = [ + [{"field" => Time.parse("1960-01-01T02:09:30Z")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_milli + records = [ + [{"field" => Time.parse("1960-01-01T02:09:30.123Z")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_micro + records = [ + [{"field" => Time.parse("1960-01-01T02:09:30.123456Z")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_timestamp_nano + records = [ + [{"field" => Time.parse("1960-01-01T02:09:30.123456789Z")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + # 00:10:00 + [{"field" => Arrow::Time.new(unit, 60 * 10)}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + # 00:10:00.123 + [{"field" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + # 00:10:00.123456 + [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + # 00:10:00.123456789 + [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal128 + records = [ + [{"field" => BigDecimal("92.92")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_decimal256 + records = [ + [{"field" => BigDecimal("92.92")}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_list + records = [ + [{"field" => [true, nil, false]}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_struct + records = [ + [{"field" => {"sub_field" => true}}], + [nil], + [{"field" => nil}], + [{"field" => {"sub_field" => nil}}], + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_map + records = [ + [{"field" => {"key1" => true, "key2" => nil}}], + [nil], + [{"field" => nil}], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, target.raw_records) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + records = [ + [{"field" => {"field1" => true}}], + [nil], + [{"field" => nil}], + [{"field" => {"field2" => nil}}], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + records = [ + [{"field" => {"field1" => true}}], + [nil], + [{"field" => nil}], + [{"field" => {"field2" => nil}}], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(records, target.raw_records) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + records = [ + [{"field" => "Ruby"}], + [nil], + [{"field" => nil}], + [{"field" => "GLib"}], + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + records) + assert_equal(records, target.raw_records) + end +end + +class RawRecordsRecordBatchStructArrayTest < Test::Unit::TestCase + include RawRecordsStructArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end +end + +class RawRecordsTableStructArrayTest < Test::Unit::TestCase + include RawRecordsStructArrayTests + + def build(type, records) + Arrow::Table.new(build_schema(type), records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/raw-records/test-table.rb b/src/arrow/ruby/red-arrow/test/raw-records/test-table.rb new file mode 100644 index 000000000..ae90217c2 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/raw-records/test-table.rb @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RawRecordsTableTest < Test::Unit::TestCase + test("2 arrays") do + raw_record_batches = [ + [ + [true, nil, "Ruby"], + [nil, 0, "GLib"], + [false, 2 ** 8 - 1, nil], + ], + [ + [nil, 10, "A"], + [true, 20, "B"], + [false, nil, "C"], + [nil, 40, nil], + ] + ] + raw_records = raw_record_batches.inject do |all_records, record_batch| + all_records + record_batch + end + schema = [ + {name: :column0, type: :boolean}, + {name: :column1, type: :uint8}, + {name: :column2, type: :string}, + ] + record_batches = raw_record_batches.collect do |record_batch| + Arrow::RecordBatch.new(schema, record_batch) + end + table = Arrow::Table.new(schema, record_batches) + assert_equal(raw_records, table.raw_records) + end +end diff --git a/src/arrow/ruby/red-arrow/test/run-test.rb b/src/arrow/ruby/red-arrow/test/run-test.rb new file mode 100755 index 000000000..41ab73cb6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/run-test.rb @@ -0,0 +1,71 @@ +#!/usr/bin/env ruby +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +$VERBOSE = true + +require "fileutils" +require "pathname" + +(ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path| + RubyInstaller::Runtime.add_dll_directory(path) +end + +base_dir = Pathname.new(__dir__).parent.expand_path + +lib_dir = base_dir + "lib" +ext_dir = base_dir + "ext" + "arrow" +test_dir = base_dir + "test" + +build_dir = ENV["BUILD_DIR"] +if build_dir + build_dir = File.join(build_dir, "red-arrow") + FileUtils.mkdir_p(build_dir) +else + build_dir = ext_dir +end + +make = nil +if ENV["NO_MAKE"] != "yes" + if ENV["MAKE"] + make = ENV["MAKE"] + elsif system("which gmake > #{File::NULL} 2>&1") + make = "gmake" + elsif system("which make > #{File::NULL} 2>&1") + make = "make" + end +end +if make + Dir.chdir(build_dir.to_s) do + unless File.exist?("Makefile") + system(RbConfig.ruby, + (ext_dir + "extconf.rb").to_s, + "--enable-debug-build") or exit(false) + end + system("#{make} > #{File::NULL}") or exit(false) + end +end + +$LOAD_PATH.unshift(build_dir.to_s) +$LOAD_PATH.unshift(lib_dir.to_s) + +require_relative "helper" + +ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "10000" + +exit(Test::Unit::AutoRunner.run(true, test_dir.to_s)) diff --git a/src/arrow/ruby/red-arrow/test/test-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-array-builder.rb new file mode 100644 index 000000000..318167d51 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-array-builder.rb @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ArrayBuilderTest < Test::Unit::TestCase + sub_test_case(".build") do + def assert_build(builder_class, raw_array) + array = builder_class.build(raw_array) + assert_equal(raw_array, array.to_a) + end + + sub_test_case("generic builder") do + test("strings") do + assert_build(Arrow::ArrayBuilder, + ["Hello", nil, "World"]) + end + + test("symbols") do + array = Arrow::ArrayBuilder.build([:hello, nil, :world]) + expected_builder = Arrow::StringDictionaryArrayBuilder.new + assert_equal(expected_builder.build(["hello", nil, "world"]), + array) + end + + test("boolean") do + assert_build(Arrow::ArrayBuilder, + [true, nil, false]) + end + + test("positive integers") do + assert_build(Arrow::ArrayBuilder, + [1, nil, 2, nil, 3]) + end + + test("negative integers") do + assert_build(Arrow::ArrayBuilder, + [nil, -1, nil, -2, nil, -3]) + end + + test("times") do + assert_build(Arrow::ArrayBuilder, + [Time.at(0), Time.at(1), Time.at(2)]) + end + + test("dates") do + assert_build(Arrow::ArrayBuilder, + [Date.new(2018, 1, 4), Date.new(2018, 1, 5)]) + end + + test("datetimes") do + assert_build(Arrow::ArrayBuilder, + [ + DateTime.new(2018, 1, 4, 23, 18, 23), + DateTime.new(2018, 1, 5, 0, 23, 21), + ]) + end + + test("list<boolean>s") do + assert_build(Arrow::ArrayBuilder, + [ + [nil, true, false], + nil, + [false], + ]) + end + + test("list<string>s") do + assert_build(Arrow::ArrayBuilder, + [ + ["Hello", "World"], + ["Apache Arrow"], + ]) + end + end + + sub_test_case("specific builder") do + test("empty") do + assert_build(Arrow::Int32ArrayBuilder, + []) + end + + test("values") do + assert_build(Arrow::Int32ArrayBuilder, + [1, -2]) + end + + test("values, nils") do + assert_build(Arrow::Int32ArrayBuilder, + [1, -2, nil, nil]) + end + + test("values, nils, values") do + assert_build(Arrow::Int32ArrayBuilder, + [1, -2, nil, nil, 3, -4]) + end + + test("values, nils, values, nils") do + assert_build(Arrow::Int32ArrayBuilder, + [1, -2, nil, nil, 3, -4, nil, nil]) + end + + test("nils") do + assert_build(Arrow::Int32ArrayBuilder, + [nil, nil]) + end + + test("nils, values") do + assert_build(Arrow::Int32ArrayBuilder, + [nil, nil, 3, -4]) + end + + test("nils, values, nil") do + assert_build(Arrow::Int32ArrayBuilder, + [nil, nil, 3, -4, nil, nil]) + end + + test("nils, values, nil, values") do + assert_build(Arrow::Int32ArrayBuilder, + [nil, nil, 3, -4, nil, nil, 5, -6]) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-array.rb b/src/arrow/ruby/red-arrow/test/test-array.rb new file mode 100644 index 000000000..2b7112da6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-array.rb @@ -0,0 +1,325 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Boolean") do + array = Arrow::BooleanArray.new([true, false, true]) + assert_equal([true, false, true], + array.to_a) + end + end + + sub_test_case("instance methods") do + def setup + @values = [true, false, nil, true] + @array = Arrow::BooleanArray.new(@values) + end + + test("#each") do + assert_equal(@values, @array.to_a) + end + + sub_test_case("#[]") do + test("valid range") do + assert_equal(@values, + @array.length.times.collect {|i| @array[i]}) + end + + test("out of range") do + assert_nil(@array[@array.length]) + end + + test("negative index") do + assert_equal(@values.last, + @array[-1]) + end + end + + sub_test_case("#==") do + test("Arrow::Array") do + assert do + @array == @array + end + end + + test("not Arrow::Array") do + assert do + not (@array == 29) + end + end + end + + sub_test_case("#equal_array?") do + test("no options") do + array1 = Arrow::FloatArray.new([1.1, Float::NAN]) + array2 = Arrow::FloatArray.new([1.1, Float::NAN]) + assert do + not array1.equal_array?(array2) + end + end + + test("approx") do + array1 = Arrow::FloatArray.new([1.1]) + array2 = Arrow::FloatArray.new([1.100001]) + assert do + array1.equal_array?(array2, approx: true) + end + end + + test("nans-equal") do + array1 = Arrow::FloatArray.new([1.1, Float::NAN]) + array2 = Arrow::FloatArray.new([1.1, Float::NAN]) + assert do + array1.equal_array?(array2, nans_equal: true) + end + end + + test("absolute-tolerance") do + array1 = Arrow::FloatArray.new([1.1]) + array2 = Arrow::FloatArray.new([1.101]) + assert do + array1.equal_array?(array2, approx: true, absolute_tolerance: 0.01) + end + end + end + + sub_test_case("#cast") do + test("Symbol") do + assert_equal(Arrow::Int32Array.new([1, 2, 3]), + Arrow::StringArray.new(["1", "2", "3"]).cast(:int32)) + end + end + end + + sub_test_case("#filter") do + def setup + values = [true, false, false, true] + @array = Arrow::BooleanArray.new(values) + @options = Arrow::FilterOptions.new + @options.null_selection_behavior = :emit_null + end + + test("Array: boolean") do + filter = [nil, true, true, false] + filtered_array = Arrow::BooleanArray.new([nil, false, false]) + assert_equal(filtered_array, + @array.filter(filter, @options)) + end + + test("Arrow::BooleanArray") do + filter = Arrow::BooleanArray.new([nil, true, true, false]) + filtered_array = Arrow::BooleanArray.new([nil, false, false]) + assert_equal(filtered_array, + @array.filter(filter, @options)) + end + + test("Arrow::ChunkedArray") do + chunks = [ + Arrow::BooleanArray.new([nil, true]), + Arrow::BooleanArray.new([true, false]), + ] + filter = Arrow::ChunkedArray.new(chunks) + filtered_array = Arrow::BooleanArray.new([nil, false, false]) + assert_equal(filtered_array, + @array.filter(filter, @options)) + end + end + + sub_test_case("#take") do + def setup + values = [1, 0 ,2] + @array = Arrow::Int16Array.new(values) + end + + test("Arrow: boolean") do + indices = [1, 0, 2] + assert_equal(Arrow::Int16Array.new([0, 1, 2]), + @array.take(indices)) + end + + test("Arrow::Array") do + indices = Arrow::Int16Array.new([1, 0, 2]) + assert_equal(Arrow::Int16Array.new([0, 1, 2]), + @array.take(indices)) + end + + test("Arrow::ChunkedArray") do + taken_chunks = [ + Arrow::Int16Array.new([0, 1]), + Arrow::Int16Array.new([2]) + ] + taken_chunked_array = Arrow::ChunkedArray.new(taken_chunks) + indices_chunks = [ + Arrow::Int16Array.new([1, 0]), + Arrow::Int16Array.new([2]) + ] + indices = Arrow::ChunkedArray.new(indices_chunks) + assert_equal(taken_chunked_array, + @array.take(indices)) + end + end + + sub_test_case("#is_in") do + def setup + values = [1, 0, 1, 2] + @array = Arrow::Int16Array.new(values) + end + + test("Arrow: Array") do + right = [2, 0] + assert_equal(Arrow::BooleanArray.new([false, true, false, true]), + @array.is_in(right)) + end + + test("Arrow::Array") do + right = Arrow::Int16Array.new([2, 0]) + assert_equal(Arrow::BooleanArray.new([false, true, false, true]), + @array.is_in(right)) + end + + test("Arrow::ChunkedArray") do + chunks = [ + Arrow::Int16Array.new([1, 4]), + Arrow::Int16Array.new([0, 3]) + ] + right = Arrow::ChunkedArray.new(chunks) + assert_equal(Arrow::BooleanArray.new([true, true, true, false]), + @array.is_in(right)) + end + end + + sub_test_case("#concatenate") do + test("Arrow::Array: same") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]), + Arrow::Int32Array.new([1, 2, nil]). + concatenate(Arrow::Int32Array.new([4, 5]), + Arrow::Int32Array.new([6]))) + end + + test("Arrow::Array: castable") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]), + Arrow::Int32Array.new([1, 2, nil]). + concatenate(Arrow::Int8Array.new([4, 5]), + Arrow::UInt32Array.new([6]))) + end + + test("Arrow::Array: non-castable") do + assert_raise(Arrow::Error::Invalid) do + Arrow::Int32Array.new([1, 2, nil]). + concatenate(Arrow::StringArray.new(["X"])) + end + end + + test("Array") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,nil, 6]), + Arrow::Int32Array.new([1, 2, nil]). + concatenate([4, nil], + [6])) + end + + test("invalid") do + message = "[array][resolve] can't build int32 array: 4" + assert_raise(ArgumentError.new(message)) do + Arrow::Int32Array.new([1, 2, nil]). + concatenate(4) + end + end + end + + sub_test_case("#+") do + test("Arrow::Array: same") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]), + Arrow::Int32Array.new([1, 2, nil]) + + Arrow::Int32Array.new([4, 5, 6])) + end + + test("Arrow::Array: castable") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]), + Arrow::Int32Array.new([1, 2, nil]) + + Arrow::Int8Array.new([4, 5, 6])) + end + + test("Arrow::Array: non-castable") do + assert_raise(Arrow::Error::Invalid) do + Arrow::Int32Array.new([1, 2, nil]) + + Arrow::StringArray.new(["X"]) + end + end + + test("Array") do + assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,nil, 6]), + Arrow::Int32Array.new([1, 2, nil]) + + [4, nil, 6]) + end + + test("invalid") do + message = "[array][resolve] can't build int32 array: 4" + assert_raise(ArgumentError.new(message)) do + Arrow::Int32Array.new([1, 2, nil]) + 4 + end + end + end + + sub_test_case("#resolve") do + test("Arrow::Array: same") do + assert_equal(Arrow::Int32Array.new([1, 2, nil]), + Arrow::Int32Array.new([]). + resolve(Arrow::Int32Array.new([1, 2, nil]))) + end + + test("Arrow::Array: castable") do + assert_equal(Arrow::Int32Array.new([1, 2, nil]), + Arrow::Int32Array.new([]). + resolve(Arrow::Int8Array.new([1, 2, nil]))) + end + + test("Arrow::Array: non-castable") do + assert_raise(Arrow::Error::Invalid) do + Arrow::Int32Array.new([]) + + Arrow::StringArray.new(["X"]) + end + end + + test("Array: non-parametric") do + assert_equal(Arrow::Int32Array.new([1, 2, nil]), + Arrow::Int32Array.new([]). + resolve([1, 2, nil])) + end + + test("Array: parametric") do + list_data_type = Arrow::ListDataType.new(name: "visible", type: :boolean) + list_array = Arrow::ListArray.new(list_data_type, []) + assert_equal(Arrow::ListArray.new(list_data_type, + [ + [true, false], + nil, + ]), + list_array.resolve([ + [true, false], + nil, + ])) + end + + test("invalid") do + message = "[array][resolve] can't build int32 array: 4" + assert_raise(ArgumentError.new(message)) do + Arrow::Int32Array.new([]).resolve(4) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-bigdecimal.rb b/src/arrow/ruby/red-arrow/test/test-bigdecimal.rb new file mode 100644 index 000000000..424f12d39 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-bigdecimal.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class BigDecimalTest < Test::Unit::TestCase + sub_test_case("#to_arrow") do + def test_128_positive + assert_equal(Arrow::Decimal128.new("0.1e38"), + BigDecimal("0.1e38").to_arrow) + end + + def test_128_negative + assert_equal(Arrow::Decimal128.new("-0.1e38"), + BigDecimal("-0.1e38").to_arrow) + end + + def test_256_positive + assert_equal(Arrow::Decimal256.new("0.1e39"), + BigDecimal("0.1e39").to_arrow) + end + + def test_256_negative + assert_equal(Arrow::Decimal256.new("-0.1e39"), + BigDecimal("-0.1e39").to_arrow) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-binary-dictionary-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-binary-dictionary-array-builder.rb new file mode 100644 index 000000000..743dbae5e --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-binary-dictionary-array-builder.rb @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class BinaryDictionaryArrayBuilderTest < Test::Unit::TestCase + def setup + @builder = Arrow::BinaryDictionaryArrayBuilder.new + end + + sub_test_case("#append_values") do + test("[nil]") do + @builder.append_values([nil]) + array = @builder.finish + assert_equal([ + [], + [nil], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[String]") do + @builder.append_values(["he\xffllo"]) + array = @builder.finish + assert_equal([ + ["he\xffllo".b], + [0], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[Symbol]") do + @builder.append_values([:hello]) + array = @builder.finish + assert_equal([ + ["hello"], + [0], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[nil, String, Symbol]") do + @builder.append_values([ + nil, + "He\xffllo", + :world, + "world", + ]) + array = @builder.finish + assert_equal([ + ["He\xffllo".b, "world"], + [nil, 0, 1, 1], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("is_valids") do + @builder.append_values([ + "He\xffllo", + :world, + :goodbye, + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + ["He\xffllo".b, "goodbye"], + [0, nil, 1], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-boolean-scalar.rb b/src/arrow/ruby/red-arrow/test/test-boolean-scalar.rb new file mode 100644 index 000000000..1053d1716 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-boolean-scalar.rb @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class BooleanScalarTest < Test::Unit::TestCase + def setup + @scalar = Arrow::BooleanScalar.new(true) + end + + test("#value") do + assert_equal(true, @scalar.value) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-buffer.rb b/src/arrow/ruby/red-arrow/test/test-buffer.rb new file mode 100644 index 000000000..b47a1abba --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-buffer.rb @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class BufferTest < Test::Unit::TestCase + sub_test_case(".new") do + test("GC") do + data = "Hello" + data_id = data.object_id + _buffer = Arrow::Buffer.new(data) + data = nil + GC.start + assert_equal("Hello", ObjectSpace._id2ref(data_id)) + end + end + + sub_test_case("instance methods") do + def setup + @buffer = Arrow::Buffer.new("Hello") + end + + sub_test_case("#==") do + test("Arrow::Buffer") do + assert do + @buffer == @buffer + end + end + + test("not Arrow::Buffer") do + assert do + not (@buffer == 29) + end + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-chunked-array.rb b/src/arrow/ruby/red-arrow/test/test-chunked-array.rb new file mode 100644 index 000000000..3785e9868 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-chunked-array.rb @@ -0,0 +1,183 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ChunkedArrayTest < Test::Unit::TestCase + test("#each") do + arrays = [ + Arrow::BooleanArray.new([true, false]), + Arrow::BooleanArray.new([nil, true]), + ] + chunked_array = Arrow::ChunkedArray.new(arrays) + assert_equal([true, false, nil, true], + chunked_array.to_a) + end + + sub_test_case("#pack") do + test("basic array") do + arrays = [ + Arrow::BooleanArray.new([true, false]), + Arrow::BooleanArray.new([nil, true]), + ] + chunked_array = Arrow::ChunkedArray.new(arrays) + packed_chunked_array = chunked_array.pack + assert_equal([ + Arrow::BooleanArray, + [true, false, nil, true], + ], + [ + packed_chunked_array.class, + packed_chunked_array.to_a, + ]) + end + + test("TimestampArray") do + type = Arrow::TimestampDataType.new(:nano) + arrays = [ + Arrow::TimestampArrayBuilder.new(type).build([Time.at(0)]), + Arrow::TimestampArrayBuilder.new(type).build([Time.at(1)]), + ] + chunked_array = Arrow::ChunkedArray.new(arrays) + packed_chunked_array = chunked_array.pack + assert_equal([ + Arrow::TimestampArray, + [Time.at(0), Time.at(1)], + ], + [ + packed_chunked_array.class, + packed_chunked_array.to_a, + ]) + end + end + + sub_test_case("#==") do + def setup + arrays = [ + Arrow::BooleanArray.new([true]), + Arrow::BooleanArray.new([false, true]), + ] + @chunked_array = Arrow::ChunkedArray.new(arrays) + end + + test("Arrow::ChunkedArray") do + assert do + @chunked_array == @chunked_array + end + end + + test("not Arrow::ChunkedArray") do + assert do + not (@chunked_array == 29) + end + end + end + + sub_test_case("#filter") do + def setup + arrays = [ + Arrow::BooleanArray.new([false, true]), + Arrow::BooleanArray.new([false, true, false]), + ] + @chunked_array = Arrow::ChunkedArray.new(arrays) + @options = Arrow::FilterOptions.new + @options.null_selection_behavior = :emit_null + end + + test("Array: boolean") do + filter = [nil, true, true, false, true] + chunks = [ + Arrow::BooleanArray.new([nil, true]), + Arrow::BooleanArray.new([false, false]), + ] + filtered_chunked_array = Arrow::ChunkedArray.new(chunks) + assert_equal(filtered_chunked_array, + @chunked_array.filter(filter, @options)) + end + + test("Arrow::BooleanArray") do + filter = Arrow::BooleanArray.new([nil, true, true, false, true]) + chunks = [ + Arrow::BooleanArray.new([nil, true]), + Arrow::BooleanArray.new([false, false]), + ] + filtered_chunked_array = Arrow::ChunkedArray.new(chunks) + assert_equal(filtered_chunked_array, + @chunked_array.filter(filter, @options)) + end + + test("Arrow::ChunkedArray") do + chunks = [ + Arrow::BooleanArray.new([nil, true]), + Arrow::BooleanArray.new([true, false, true]), + ] + filter = Arrow::ChunkedArray.new(chunks) + filtered_chunks = [ + Arrow::BooleanArray.new([nil, true]), + Arrow::BooleanArray.new([false, false]), + ] + filtered_chunked_array = Arrow::ChunkedArray.new(filtered_chunks) + assert_equal(filtered_chunked_array, + @chunked_array.filter(filter, @options)) + end + end + + sub_test_case("#take") do + def setup + chunks = [ + Arrow::Int16Array.new([1, 0]), + Arrow::Int16Array.new([2]), + ] + @chunked_array = Arrow::ChunkedArray.new(chunks) + end + + test("Arrow: boolean") do + chunks = [ + Arrow::Int16Array.new([0, 1]), + Arrow::Int16Array.new([2]) + ] + taken_chunked_array = Arrow::ChunkedArray.new(chunks) + indices = [1, 0, 2] + assert_equal(taken_chunked_array, + @chunked_array.take(indices)) + end + + test("Arrow::Array") do + chunks = [ + Arrow::Int16Array.new([0, 1]), + Arrow::Int16Array.new([2]) + ] + taken_chunked_array = Arrow::ChunkedArray.new(chunks) + indices = Arrow::Int16Array.new([1, 0, 2]) + assert_equal(taken_chunked_array, + @chunked_array.take(indices)) + end + + test("Arrow::ChunkedArray") do + taken_chunks = [ + Arrow::Int16Array.new([0, 1]), + Arrow::Int16Array.new([2]) + ] + taken_chunked_array = Arrow::ChunkedArray.new(taken_chunks) + indices_chunks = [ + Arrow::Int16Array.new([1, 0]), + Arrow::Int16Array.new([2]) + ] + indices = Arrow::ChunkedArray.new(indices_chunks) + assert_equal(taken_chunked_array, + @chunked_array.take(indices)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-column.rb b/src/arrow/ruby/red-arrow/test/test-column.rb new file mode 100644 index 000000000..613b01ccc --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-column.rb @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ColumnTest < Test::Unit::TestCase + def setup + table = Arrow::Table.new("visible" => [true, nil, false]) + @column = table.visible + end + + test("#name") do + assert_equal("visible", @column.name) + end + + test("#data_type") do + assert_equal(Arrow::BooleanDataType.new, @column.data_type) + end + + test("#null?") do + assert do + @column.null?(1) + end + end + + test("#valid?") do + assert do + @column.valid?(0) + end + end + + test("#each") do + assert_equal([true, nil, false], @column.each.to_a) + end + + test("#reverse_each") do + assert_equal([false, nil, true], @column.reverse_each.to_a) + end + + test("#n_rows") do + assert_equal(3, @column.n_rows) + end + + test("#n_nulls") do + assert_equal(1, @column.n_nulls) + end + + sub_test_case("#==") do + test("same value") do + table1 = Arrow::Table.new("visible" => [true, false]) + table2 = Arrow::Table.new("visible" => [true, false]) + assert do + table1.visible == table2.visible + end + end + + test("different name") do + table1 = Arrow::Table.new("visible" => [true, false]) + table2 = Arrow::Table.new("invisible" => [true, false]) + assert do + not table1.visible == table2.invisible + end + end + + test("different value") do + table1 = Arrow::Table.new("visible" => [true, false]) + table2 = Arrow::Table.new("visible" => [true, true]) + assert do + not table1.visible == table2.visible + end + end + + test("not Arrow::Column") do + table = Arrow::Table.new("visible" => [true, false]) + assert do + not table.visible == 29 + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-csv-loader.rb b/src/arrow/ruby/red-arrow/test/test-csv-loader.rb new file mode 100644 index 000000000..7f7f23498 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-csv-loader.rb @@ -0,0 +1,250 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class CSVLoaderTest < Test::Unit::TestCase + include Helper::Fixture + + def load_csv(input) + Arrow::CSVLoader.load(input, skip_lines: /^#/) + end + + sub_test_case(".load") do + test("String: data: with header") do + data = fixture_path("with-header-float.csv").read + assert_equal(<<-TABLE, load_csv(data).to_s) + name score +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("String: data: without header") do + data = fixture_path("without-header-float.csv").read + assert_equal(<<-TABLE, load_csv(data).to_s) + 0 1 +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("String: path: with header") do + path = fixture_path("with-header-float.csv").to_s + assert_equal(<<-TABLE, load_csv(path).to_s) + name score +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("String: path: without header") do + path = fixture_path("without-header-float.csv").to_s + assert_equal(<<-TABLE, load_csv(path).to_s) + 0 1 +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("Pathname: with header") do + path = fixture_path("with-header-float.csv") + assert_equal(<<-TABLE, load_csv(path).to_s) + name score +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("Pathname: without header") do + path = fixture_path("without-header-float.csv") + assert_equal(<<-TABLE, load_csv(path).to_s) + 0 1 +0 alice 10.100000 +1 bob 29.200000 +2 chris -1.300000 + TABLE + end + + test("null: with double quote") do + path = fixture_path("null-with-double-quote.csv").to_s + assert_equal(<<-TABLE, load_csv(path).to_s) + name score +0 alice 10 +1 bob (null) +2 chris -1 + TABLE + end + + test("null: without double quote") do + path = fixture_path("null-without-double-quote.csv").to_s + assert_equal(<<-TABLE, load_csv(path).to_s) + name score +0 alice 10 +1 bob (null) +2 chris -1 + TABLE + end + + test("number: float, integer") do + path = fixture_path("float-integer.csv").to_s + assert_equal([2.9, 10, -1.1], + load_csv(path)[:score].to_a) + end + + test("number: integer, float") do + path = fixture_path("integer-float.csv").to_s + assert_equal([10.0, 2.9, -1.1], + load_csv(path)[:score].to_a) + end + end + + sub_test_case("CSVReader") do + def load_csv(data, **options) + Arrow::CSVLoader.load(data, **options) + end + + sub_test_case(":headers") do + test("true") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(value: values), + load_csv(<<-CSV, headers: true)) +value +a +b +c + CSV + end + + test(":first_line") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(value: values), + load_csv(<<-CSV, headers: :first_line)) +value +a +b +c + CSV + end + + test("truthy") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(value: values), + load_csv(<<-CSV, headers: 0)) +value +a +b +c + CSV + end + + test("Array of column names") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(column: values), + load_csv(<<-CSV, headers: ["column"])) +a +b +c + CSV + end + + test("false") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(f0: values), + load_csv(<<-CSV, headers: false)) +a +b +c + CSV + end + + test("nil") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(f0: values), + load_csv(<<-CSV, headers: nil)) +a +b +c + CSV + end + + test("string") do + values = Arrow::StringArray.new(["a", "b", "c"]) + assert_equal(Arrow::Table.new(column: values), + load_csv(<<-CSV, headers: "column")) +a +b +c + CSV + end + end + + test(":column_types") do + assert_equal(Arrow::Table.new(:count => Arrow::UInt16Array.new([1, 2, 4])), + load_csv(<<-CSV, column_types: {count: :uint16})) +count +1 +2 +4 + CSV + end + + test(":schema") do + table = Arrow::Table.new(:count => Arrow::UInt16Array.new([1, 2, 4])) + assert_equal(table, + load_csv(<<-CSV, schema: table.schema)) +count +1 +2 +4 + CSV + end + + test(":encoding") do + messages = [ + "\u3042", # U+3042 HIRAGANA LETTER A + "\u3044", # U+3044 HIRAGANA LETTER I + "\u3046", # U+3046 HIRAGANA LETTER U + ] + table = Arrow::Table.new(:message => Arrow::StringArray.new(messages)) + encoding = "cp932" + assert_equal(table, + load_csv((["message"] + messages).join("\n").encode(encoding), + schema: table.schema, + encoding: encoding)) + end + + test(":encoding and :compression") do + messages = [ + "\u3042", # U+3042 HIRAGANA LETTER A + "\u3044", # U+3044 HIRAGANA LETTER I + "\u3046", # U+3046 HIRAGANA LETTER U + ] + table = Arrow::Table.new(:message => Arrow::StringArray.new(messages)) + encoding = "cp932" + csv = (["message"] + messages).join("\n").encode(encoding) + assert_equal(table, + load_csv(Zlib::Deflate.deflate(csv), + schema: table.schema, + encoding: encoding, + compression: :gzip)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-data-type.rb b/src/arrow/ruby/red-arrow/test/test-data-type.rb new file mode 100644 index 000000000..f54831780 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-data-type.rb @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class DataTypeTest < Test::Unit::TestCase + sub_test_case(".resolve") do + test("DataType") do + assert_equal(Arrow::BooleanDataType.new, + Arrow::DataType.resolve(Arrow::BooleanDataType.new)) + end + + test("String") do + assert_equal(Arrow::BooleanDataType.new, + Arrow::DataType.resolve("boolean")) + end + + test("Symbol") do + assert_equal(Arrow::BooleanDataType.new, + Arrow::DataType.resolve(:boolean)) + end + + test("Array") do + field = Arrow::Field.new(:visible, :boolean) + assert_equal(Arrow::ListDataType.new(field), + Arrow::DataType.resolve([:list, field])) + end + + test("Hash") do + field = Arrow::Field.new(:visible, :boolean) + assert_equal(Arrow::ListDataType.new(field), + Arrow::DataType.resolve(type: :list, field: field)) + end + + test("_") do + assert_equal(Arrow::FixedSizeBinaryDataType.new(10), + Arrow::DataType.resolve([:fixed_size_binary, 10])) + end + + test("abstract") do + message = + "abstract type: <:floating_point>: " + + "use one of not abstract type: [" + + "Arrow::DoubleDataType, " + + "Arrow::FloatDataType]" + assert_raise(ArgumentError.new(message)) do + Arrow::DataType.resolve(:floating_point) + end + end + end + + sub_test_case("instance methods") do + def setup + @data_type = Arrow::StringDataType.new + end + + sub_test_case("#==") do + test("Arrow::DataType") do + assert do + @data_type == @data_type + end + end + + test("not Arrow::DataType") do + assert do + not (@data_type == 29) + end + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-date32-array.rb b/src/arrow/ruby/red-arrow/test/test-date32-array.rb new file mode 100644 index 000000000..6918b48db --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-date32-array.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Date32ArrayTest < Test::Unit::TestCase + test("#[]") do + n_days_since_epoch = 17406 # 2017-08-28 + array = Arrow::Date32Array.new([n_days_since_epoch]) + assert_equal(Date.new(2017, 8, 28), array[0]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-date64-array.rb b/src/arrow/ruby/red-arrow/test/test-date64-array.rb new file mode 100644 index 000000000..ec1c6db7c --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-date64-array.rb @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Date64ArrayTest < Test::Unit::TestCase + test("#[]") do + n_msecs_since_epoch = 1503878400000 # 2017-08-28T00:00:00Z + array = Arrow::Date64Array.new([n_msecs_since_epoch]) + assert_equal(DateTime.new(2017, 8, 28, 0, 0, 0), + array[0]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal128-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-decimal128-array-builder.rb new file mode 100644 index 000000000..31d58bd58 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal128-array-builder.rb @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal128ArrayBuilderTest < Test::Unit::TestCase + def setup + @data_type = Arrow::Decimal128DataType.new(3, 1) + @builder = Arrow::Decimal128ArrayBuilder.new(@data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal(nil, array[0]) + end + + test("Arrow::Decimal128") do + @builder.append_value(Arrow::Decimal128.new("10.1")) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("String") do + @builder.append_value("10.1") + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("Float") do + @builder.append_value(10.1) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("BigDecimal") do + @builder.append_value(BigDecimal("10.1")) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + end + + sub_test_case("#append_values") do + test("mixed") do + @builder.append_values([ + Arrow::Decimal128.new("10.1"), + nil, + "10.1", + 10.1, + BigDecimal("10.1"), + ]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + BigDecimal("10.1"), + BigDecimal("10.1"), + ], + array.to_a) + end + + test("is_valids") do + @builder.append_values([ + Arrow::Decimal128.new("10.1"), + Arrow::Decimal128.new("10.1"), + Arrow::Decimal128.new("10.1"), + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + ], + array.to_a) + end + + test("packed") do + @builder.append_values(Arrow::Decimal128.new("10.1").to_bytes.to_s * 3, + [true, false, true]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal128-array.rb b/src/arrow/ruby/red-arrow/test/test-decimal128-array.rb new file mode 100644 index 000000000..88ab1c26c --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal128-array.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal128ArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + data_type = Arrow::Decimal128DataType.new(3, 1) + values = [ + 10.1, + nil, + "10.1", + BigDecimal("10.1"), + ] + array = Arrow::Decimal128Array.new(data_type, values) + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + BigDecimal("10.1"), + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal128-data-type.rb b/src/arrow/ruby/red-arrow/test/test-decimal128-data-type.rb new file mode 100644 index 000000000..5390a7a44 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal128-data-type.rb @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal128DataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("ordered arguments") do + assert_equal("decimal128(8, 2)", + Arrow::Decimal128DataType.new(8, 2).to_s) + end + + test("description") do + assert_equal("decimal128(8, 2)", + Arrow::Decimal128DataType.new(precision: 8, + scale: 2).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal128.rb b/src/arrow/ruby/red-arrow/test/test-decimal128.rb new file mode 100644 index 000000000..9e7f8792c --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal128.rb @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal128Test < Test::Unit::TestCase + sub_test_case("instance methods") do + def setup + @decimal128 = Arrow::Decimal128.new("10.1") + end + + sub_test_case("#==") do + test("Arrow::Decimal128") do + assert do + @decimal128 == @decimal128 + end + end + + test("not Arrow::Decimal128") do + assert do + not (@decimal128 == 10.1) + end + end + end + + sub_test_case("#!=") do + test("Arrow::Decimal128") do + assert do + not (@decimal128 != @decimal128) + end + end + + test("not Arrow::Decimal128") do + assert do + @decimal128 != 10.1 + end + end + end + + sub_test_case("#to_s") do + test("default") do + assert_equal("101", + @decimal128.to_s) + end + + test("scale") do + assert_equal("10.1", + @decimal128.to_s(1)) + end + end + + test("#abs") do + decimal128 = Arrow::Decimal128.new("-10.1") + assert_equal([ + Arrow::Decimal128.new("-10.1"), + Arrow::Decimal128.new("10.1"), + ], + [ + decimal128, + decimal128.abs, + ]) + end + + test("#abs!") do + decimal128 = Arrow::Decimal128.new("-10.1") + decimal128.abs! + assert_equal(Arrow::Decimal128.new("10.1"), + decimal128) + end + + test("#negate") do + decimal128 = Arrow::Decimal128.new("-10.1") + assert_equal([ + Arrow::Decimal128.new("-10.1"), + Arrow::Decimal128.new("10.1"), + ], + [ + decimal128, + decimal128.negate, + ]) + end + + test("#negate!") do + decimal128 = Arrow::Decimal128.new("-10.1") + decimal128.negate! + assert_equal(Arrow::Decimal128.new("10.1"), + decimal128) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal256-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-decimal256-array-builder.rb new file mode 100644 index 000000000..f0769b662 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal256-array-builder.rb @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal256ArrayBuilderTest < Test::Unit::TestCase + def setup + @data_type = Arrow::Decimal256DataType.new(3, 1) + @builder = Arrow::Decimal256ArrayBuilder.new(@data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal(nil, array[0]) + end + + test("Arrow::Decimal256") do + @builder.append_value(Arrow::Decimal256.new("10.1")) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("String") do + @builder.append_value("10.1") + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("Float") do + @builder.append_value(10.1) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + + test("BigDecimal") do + @builder.append_value(BigDecimal("10.1")) + array = @builder.finish + assert_equal(BigDecimal("10.1"), + array[0]) + end + end + + sub_test_case("#append_values") do + test("mixed") do + @builder.append_values([ + Arrow::Decimal256.new("10.1"), + nil, + "10.1", + 10.1, + BigDecimal("10.1"), + ]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + BigDecimal("10.1"), + BigDecimal("10.1"), + ], + array.to_a) + end + + test("is_valids") do + @builder.append_values([ + Arrow::Decimal256.new("10.1"), + Arrow::Decimal256.new("10.1"), + Arrow::Decimal256.new("10.1"), + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + ], + array.to_a) + end + + test("packed") do + @builder.append_values(Arrow::Decimal256.new("10.1").to_bytes.to_s * 3, + [true, false, true]) + array = @builder.finish + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal256-array.rb b/src/arrow/ruby/red-arrow/test/test-decimal256-array.rb new file mode 100644 index 000000000..7049a4509 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal256-array.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal256ArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + data_type = Arrow::Decimal256DataType.new(3, 1) + values = [ + 10.1, + nil, + "10.1", + BigDecimal("10.1"), + ] + array = Arrow::Decimal256Array.new(data_type, values) + assert_equal([ + BigDecimal("10.1"), + nil, + BigDecimal("10.1"), + BigDecimal("10.1"), + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal256-data-type.rb b/src/arrow/ruby/red-arrow/test/test-decimal256-data-type.rb new file mode 100644 index 000000000..96b2a505b --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal256-data-type.rb @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal256DataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("ordered arguments") do + assert_equal("decimal256(8, 2)", + Arrow::Decimal256DataType.new(8, 2).to_s) + end + + test("description") do + assert_equal("decimal256(8, 2)", + Arrow::Decimal256DataType.new(precision: 8, + scale: 2).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-decimal256.rb b/src/arrow/ruby/red-arrow/test/test-decimal256.rb new file mode 100644 index 000000000..422167f99 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-decimal256.rb @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Decimal256Test < Test::Unit::TestCase + sub_test_case("instance methods") do + def setup + @decimal256 = Arrow::Decimal256.new("10.1") + end + + sub_test_case("#==") do + test("Arrow::Decimal256") do + assert do + @decimal256 == @decimal256 + end + end + + test("not Arrow::Decimal256") do + assert do + not (@decimal256 == 10.1) + end + end + end + + sub_test_case("#!=") do + test("Arrow::Decimal256") do + assert do + not (@decimal256 != @decimal256) + end + end + + test("not Arrow::Decimal256") do + assert do + @decimal256 != 10.1 + end + end + end + + sub_test_case("#to_s") do + test("default") do + assert_equal("101", + @decimal256.to_s) + end + + test("scale") do + assert_equal("10.1", + @decimal256.to_s(1)) + end + end + + test("#abs") do + decimal256 = Arrow::Decimal256.new("-10.1") + assert_equal([ + Arrow::Decimal256.new("-10.1"), + Arrow::Decimal256.new("10.1"), + ], + [ + decimal256, + decimal256.abs, + ]) + end + + test("#abs!") do + decimal256 = Arrow::Decimal256.new("-10.1") + decimal256.abs! + assert_equal(Arrow::Decimal256.new("10.1"), + decimal256) + end + + test("#negate") do + decimal256 = Arrow::Decimal256.new("-10.1") + assert_equal([ + Arrow::Decimal256.new("-10.1"), + Arrow::Decimal256.new("10.1"), + ], + [ + decimal256, + decimal256.negate, + ]) + end + + test("#negate!") do + decimal256 = Arrow::Decimal256.new("-10.1") + decimal256.negate! + assert_equal(Arrow::Decimal256.new("10.1"), + decimal256) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-dense-union-data-type.rb b/src/arrow/ruby/red-arrow/test/test-dense-union-data-type.rb new file mode 100644 index 000000000..d8da6f772 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-dense-union-data-type.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class DenseUnionDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + def setup + @fields = [ + Arrow::Field.new("visible", :boolean), + { + name: "count", + type: :int32, + }, + ] + end + + test("ordered arguments") do + assert_equal("dense_union<visible: bool=2, count: int32=9>", + Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s) + end + + test("description") do + assert_equal("dense_union<visible: bool=2, count: int32=9>", + Arrow::DenseUnionDataType.new(fields: @fields, + type_codes: [2, 9]).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-dictionary-array.rb b/src/arrow/ruby/red-arrow/test/test-dictionary-array.rb new file mode 100644 index 000000000..83368e9ec --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-dictionary-array.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class DictionaryArrayTest < Test::Unit::TestCase + sub_test_case("instance methods") do + def setup + @values = ["a", "b", "c", "b", "a"] + @string_array = Arrow::StringArray.new(@values) + @array = @string_array.dictionary_encode + end + + test("#[]") do + assert_equal(@values, @array.to_a) + end + + test("#get_value") do + assert_equal([ + @values[0], + @values[3], + ], + [ + @array.get_value(0), + @array.get_value(3), + ]) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-dictionary-data-type.rb b/src/arrow/ruby/red-arrow/test/test-dictionary-data-type.rb new file mode 100644 index 000000000..c5b6dd1bf --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-dictionary-data-type.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class DictionaryDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + def setup + @index_data_type = :int8 + @value_data_type = :string + @ordered = true + end + + test("ordered arguments") do + assert_equal("dictionary<values=string, indices=int8, ordered=1>", + Arrow::DictionaryDataType.new(@index_data_type, + @value_data_type, + @ordered).to_s) + end + + test("description") do + assert_equal("dictionary<values=string, indices=int8, ordered=1>", + Arrow::DictionaryDataType.new(index_data_type: @index_data_type, + value_data_type: @value_data_type, + ordered: @ordered).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-expression.rb b/src/arrow/ruby/red-arrow/test/test-expression.rb new file mode 100644 index 000000000..e172e78be --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-expression.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestExpression < Test::Unit::TestCase + sub_test_case(".try_convert") do + test("Symbol") do + assert_equal(Arrow::FieldExpression.new("visible"), + Arrow::Expression.try_convert(:visible)) + end + + test("[String]") do + assert_equal(Arrow::CallExpression.new("func", []), + Arrow::Expression.try_convert(["func"])) + end + + test("[Symbol]") do + assert_equal(Arrow::CallExpression.new("func", []), + Arrow::Expression.try_convert([:func])) + end + + test("[String, String]") do + assert_equal(Arrow::CallExpression.new("func", ["argument1"]), + Arrow::Expression.try_convert(["func", "argument1"])) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-feather.rb b/src/arrow/ruby/red-arrow/test/test-feather.rb new file mode 100644 index 000000000..21d8a2c31 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-feather.rb @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FeatherTest < Test::Unit::TestCase + include Helper::Fixture + + def setup + columns = { + "message" => Arrow::StringArray.new(["Start", "Crash", "Shutdown"]), + "is_critical" => Arrow::BooleanArray.new([false, true, false]), + } + @table = Arrow::Table.new(columns) + + @output = Tempfile.new(["red-arrow", ".feather"]) + begin + yield(@output) + ensure + @output.close! + end + end + + def test_default + @table.save(@output.path) + @output.close + + assert_equal(@table, Arrow::Table.load(@output.path)) + end + + def test_compression + @table.save(@output.path, compression: :zstd) + @output.close + + assert_equal(@table, Arrow::Table.load(@output.path)) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-field.rb b/src/arrow/ruby/red-arrow/test/test-field.rb new file mode 100644 index 000000000..1b6bc4b17 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-field.rb @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FieldTest < Test::Unit::TestCase + sub_test_case(".new") do + test("String, Arrow::DataType") do + assert_equal("visible: bool", + Arrow::Field.new("visible", Arrow::BooleanDataType.new).to_s) + end + + test("Symbol, Arrow::DataType") do + assert_equal("visible: bool", + Arrow::Field.new(:visible, Arrow::BooleanDataType.new).to_s) + end + + test("String, Symbol") do + assert_equal("visible: bool", + Arrow::Field.new(:visible, :boolean).to_s) + end + + test("String, Hash") do + assert_equal("visible: bool", + Arrow::Field.new(:visible, type: :boolean).to_s) + end + + test("description: String") do + assert_equal("visible: bool", + Arrow::Field.new(name: "visible", + data_type: :boolean).to_s) + end + + test("description: Symbol") do + assert_equal("visible: bool", + Arrow::Field.new(name: :visible, + data_type: :boolean).to_s) + end + + test("description: shortcut") do + assert_equal("visible: bool", + Arrow::Field.new(name: :visible, + type: :boolean).to_s) + end + + test("Hash: shortcut: additional") do + description = { + name: :tags, + type: :list, + field: { + name: "tag", + type: :string, + }, + } + assert_equal("tags: list<tag: string>", + Arrow::Field.new(description).to_s) + end + end + + sub_test_case("instance methods") do + def setup + @field = Arrow::Field.new("count", :uint32) + end + + sub_test_case("#==") do + test("Arrow::Field") do + assert do + @field == @field + end + end + + test("not Arrow::Field") do + assert do + not (@field == 29) + end + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-file-output-stream.rb b/src/arrow/ruby/red-arrow/test/test-file-output-stream.rb new file mode 100644 index 000000000..559406a4e --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-file-output-stream.rb @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestFileOutputStream < Test::Unit::TestCase + sub_test_case(".open") do + def setup + @file = Tempfile.open("arrow-file-output-stream") + @file.write("Hello") + @file.close + end + + def test_default + Arrow::FileOutputStream.open(@file.path) do |file| + file.write(" World") + end + assert_equal(" World", File.read(@file.path)) + end + + def test_options_append + Arrow::FileOutputStream.open(@file.path, append: true) do |file| + file.write(" World") + end + assert_equal("Hello World", File.read(@file.path)) + end + + def test_append_true + Arrow::FileOutputStream.open(@file.path, true) do |file| + file.write(" World") + end + assert_equal("Hello World", File.read(@file.path)) + end + + def test_append_false + Arrow::FileOutputStream.open(@file.path, false) do |file| + file.write(" World") + end + assert_equal(" World", File.read(@file.path)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array-builder.rb new file mode 100644 index 000000000..fae79f285 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array-builder.rb @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FixedSizeBinaryArrayBuilderTest < Test::Unit::TestCase + def setup + @data_type = Arrow::FixedSizeBinaryDataType.new(4) + @builder = Arrow::FixedSizeBinaryArrayBuilder.new(@data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal(nil, array[0]) + end + + test("String") do + @builder.append_value("0123") + array = @builder.finish + assert_equal("0123", array[0]) + end + + test("GLib::Bytes") do + @builder.append_value(GLib::Bytes.new("0123")) + array = @builder.finish + assert_equal("0123", array[0]) + end + end + + sub_test_case("#append_values") do + test("mixed") do + @builder.append_values([ + "0123", + nil, + GLib::Bytes.new("abcd"), + ]) + array = @builder.finish + assert_equal([ + "0123", + nil, + "abcd", + ], + array.to_a) + end + + test("is_valids") do + @builder.append_values([ + "0123", + "0123", + "0123", + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + "0123", + nil, + "0123", + ], + array.to_a) + end + + test("packed") do + @builder.append_values("0123" * 3, + [true, false, true]) + array = @builder.finish + assert_equal([ + "0123", + nil, + "0123", + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array.rb b/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array.rb new file mode 100644 index 000000000..3cb46b964 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-fixed-size-binary-array.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FixedSizeBinaryArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + data_type = Arrow::FixedSizeBinaryDataType.new(4) + values = [ + "0123", + nil, + GLib::Bytes.new("abcd"), + ] + array = Arrow::FixedSizeBinaryArray.new(data_type, values) + assert_equal([ + "0123", + nil, + "abcd", + ], + array.to_a) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-float-scalar.rb b/src/arrow/ruby/red-arrow/test/test-float-scalar.rb new file mode 100644 index 000000000..1117d7728 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-float-scalar.rb @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FloatScalarTest < Test::Unit::TestCase + sub_test_case("#equal_scalar?") do + test("no options") do + scalar1 = Arrow::FloatScalar.new(1.1) + scalar2 = Arrow::FloatScalar.new(1.1000001) + assert do + not scalar1.equal_scalar?(scalar2) + end + end + + test(":approx") do + scalar1 = Arrow::FloatScalar.new(1.1) + scalar2 = Arrow::FloatScalar.new(1.1000001) + assert do + scalar1.equal_scalar?(scalar2, approx: true) + end + end + + test(":absolute_tolerance") do + scalar1 = Arrow::FloatScalar.new(1.1) + scalar2 = Arrow::FloatScalar.new(1.1001) + assert do + scalar1.equal_scalar?(scalar2, + approx: true, + absolute_tolerance: 0.001) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-function.rb b/src/arrow/ruby/red-arrow/test/test-function.rb new file mode 100644 index 000000000..95667e66c --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-function.rb @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class FunctionTest < Test::Unit::TestCase + sub_test_case("#execute") do + test("Arrow::Array") do + or_function = Arrow::Function.find("or") + args = [ + Arrow::BooleanArray.new([true, false, false]), + Arrow::BooleanArray.new([true, false, true]), + ] + assert_equal([true, false, true], + or_function.execute(args).value.to_a) + end + + test("Array") do + or_function = Arrow::Function.find("or") + args = [ + [true, false, false], + [true, false, true], + ] + assert_equal([true, false, true], + or_function.execute(args).value.to_a) + end + + test("Arrow::ChunkedArray") do + or_function = Arrow::Function.find("or") + args = [ + Arrow::ChunkedArray.new([ + Arrow::BooleanArray.new([true]), + Arrow::BooleanArray.new([false, false]), + ]), + Arrow::ChunkedArray.new([ + Arrow::BooleanArray.new([true, false]), + Arrow::BooleanArray.new([true]), + ]), + ] + assert_equal([true, false, true], + or_function.execute(args).value.to_a) + end + + test("Arrow::Scalar") do + add_function = Arrow::Function.find("add") + args = [ + Arrow::Int8Array.new([1, 2, 3]), + Arrow::Int8Scalar.new(5), + ] + assert_equal([6, 7, 8], + add_function.execute(args).value.to_a) + end + + test("Integer") do + add_function = Arrow::Function.find("add") + args = [ + [1, 2, 3], + 5, + ] + assert_equal([6, 7, 8], + add_function.execute(args).value.to_a) + end + + test("Float") do + add_function = Arrow::Function.find("add") + args = [ + [1, 2, 3], + 5.1, + ] + assert_equal([6.1, 7.1, 8.1], + add_function.execute(args).value.to_a) + end + + test("true") do + and_function = Arrow::Function.find("and") + args = [ + Arrow::BooleanArray.new([true, false, false]), + true, + ] + assert_equal([true, false, false], + and_function.execute(args).value.to_a) + end + + test("false") do + or_function = Arrow::Function.find("or") + args = [ + Arrow::BooleanArray.new([true, false, false]), + false, + ] + assert_equal([true, false, false], + or_function.execute(args).value.to_a) + end + + test("String") do + ascii_upper_function = Arrow::Function.find("ascii_upper") + args = [ + "Hello", + ] + assert_equal("HELLO", + ascii_upper_function.execute(args).value.to_s) + end + + test("Date") do + cast_function = Arrow::Function.find("cast") + date = Date.new(2021, 6, 12) + args = [date] + options = Arrow::CastOptions.new + options.to_data_type = Arrow::TimestampDataType.new(:second) + time = Time.utc(date.year, + date.month, + date.day) + assert_equal(Arrow::TimestampScalar.new(options.to_data_type, + time.to_i), + cast_function.execute(args, options).value) + end + + test("Arrow::Time: second") do + cast_function = Arrow::Function.find("cast") + arrow_time = Arrow::Time.new(Arrow::TimeUnit::SECOND, + # 00:10:00 + 60 * 10) + args = [arrow_time] + options = Arrow::CastOptions.new + options.to_data_type = Arrow::Time64DataType.new(:micro) + assert_equal(Arrow::Time64Scalar.new(options.to_data_type, + # 00:10:00.000000 + 60 * 10 * 1000 * 1000), + cast_function.execute(args, options).value) + end + + test("Arrow::Time: micro") do + cast_function = Arrow::Function.find("cast") + arrow_time = Arrow::Time.new(Arrow::TimeUnit::MICRO, + # 00:10:00.000000 + 60 * 10 * 1000 * 1000) + args = [arrow_time] + options = Arrow::CastOptions.new + options.to_data_type = Arrow::Time32DataType.new(:second) + options.allow_time_truncate = true + assert_equal(Arrow::Time32Scalar.new(options.to_data_type, + # 00:10:00 + 60 * 10), + cast_function.execute(args, options).value) + end + + test("Time") do + cast_function = Arrow::Function.find("cast") + time = Time.utc(2021, 6, 12, 1, 2, 3, 1) + args = [time] + options = Arrow::CastOptions.new + options.to_data_type = Arrow::TimestampDataType.new(:second) + options.allow_time_truncate = true + time = Time.utc(time.year, + time.month, + time.day, + time.hour, + time.min, + time.sec) + assert_equal(Arrow::TimestampScalar.new(options.to_data_type, + time.to_i), + cast_function.execute(args, options).value) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-group.rb b/src/arrow/ruby/red-arrow/test/test-group.rb new file mode 100644 index 000000000..2823977d5 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-group.rb @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class GroupTest < Test::Unit::TestCase + include Helper::Fixture + + def setup + raw_table = { + :group_key1 => Arrow::UInt8Array.new([1, 1, 2, 3, 3, 3]), + :group_key2 => Arrow::UInt8Array.new([1, 1, 1, 1, 2, 2]), + :int => Arrow::Int32Array.new([-1, -2, nil, -4, -5, -6]), + :uint => Arrow::UInt32Array.new([1, nil, 3, 4, 5, 6]), + :float => Arrow::FloatArray.new([nil, 2.2, 3.3, 4.4, 5.5, 6.6]), + :string => Arrow::StringArray.new(["a", "b", "c", nil, "e", "f"]), + } + @table = Arrow::Table.new(raw_table) + end + + sub_test_case("key") do + test("Time") do + time_values = [ + Time.parse("2018-01-29"), + Time.parse("2018-01-30"), + ] + raw_table = { + :time => Arrow::ArrayBuilder.build(time_values), + :int => Arrow::Int32Array.new([-1, -2]), + } + table = Arrow::Table.new(raw_table) + assert_equal(<<-TABLE, table.group(:time).count.to_s) + count(int) time +0 1 #{time_values[0].iso8601} +1 1 #{time_values[1].iso8601} + TABLE + end + end + + sub_test_case("#count") do + test("single") do + assert_equal(<<-TABLE, @table.group(:group_key1).count.to_s) + count(group_key2) count(int) count(uint) count(float) count(string) group_key1 +0 2 2 1 1 2 1 +1 1 0 1 1 1 2 +2 3 3 3 3 2 3 + TABLE + end + + test("multiple") do + assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).count.to_s) + count(int) count(uint) count(float) count(string) group_key1 group_key2 +0 2 1 1 2 1 1 +1 0 1 1 1 2 1 +2 1 1 1 0 3 1 +3 2 2 2 2 3 2 + TABLE + end + + test("column") do + group = @table.group(:group_key1, :group_key2) + assert_equal(<<-TABLE, group.count(:int, :uint).to_s) + count(int) count(uint) group_key1 group_key2 +0 2 1 1 1 +1 0 1 2 1 +2 1 1 3 1 +3 2 2 3 2 + TABLE + end + end + + sub_test_case("#sum") do + test("single") do + assert_equal(<<-TABLE, @table.group(:group_key1).sum.to_s) + sum(group_key2) sum(int) sum(uint) sum(float) group_key1 +0 2 -3 1 2.200000 1 +1 1 (null) 3 3.300000 2 +2 5 -15 15 16.500000 3 + TABLE + end + + test("multiple") do + assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).sum.to_s) + sum(int) sum(uint) sum(float) group_key1 group_key2 +0 -3 1 2.200000 1 1 +1 (null) 3 3.300000 2 1 +2 -4 4 4.400000 3 1 +3 -11 11 12.100000 3 2 + TABLE + end + end + + sub_test_case("#mean") do + test("single") do + assert_equal(<<-TABLE, @table.group(:group_key1).mean.to_s) + mean(group_key2) mean(int) mean(uint) mean(float) group_key1 +0 1.000000 -1.500000 1.000000 2.200000 1 +1 1.000000 (null) 3.000000 3.300000 2 +2 1.666667 -5.000000 5.000000 5.500000 3 + TABLE + end + + test("multiple") do + assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).mean.to_s) + mean(int) mean(uint) mean(float) group_key1 group_key2 +0 -1.500000 1.000000 2.200000 1 1 +1 (null) 3.000000 3.300000 2 1 +2 -4.000000 4.000000 4.400000 3 1 +3 -5.500000 5.500000 6.050000 3 2 + TABLE + end + end + + sub_test_case("#min") do + test("single") do + assert_equal(<<-TABLE, @table.group(:group_key1).min.to_s) + min(group_key2) min(int) min(uint) min(float) group_key1 +0 1 -2 1 2.200000 1 +1 1 (null) 3 3.300000 2 +2 1 -6 4 4.400000 3 + TABLE + end + + test("multiple") do + assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).min.to_s) + min(int) min(uint) min(float) group_key1 group_key2 +0 -2 1 2.200000 1 1 +1 (null) 3 3.300000 2 1 +2 -4 4 4.400000 3 1 +3 -6 5 5.500000 3 2 + TABLE + end + end + + sub_test_case("#max") do + test("single") do + assert_equal(<<-TABLE, @table.group(:group_key1).max.to_s) + max(group_key2) max(int) max(uint) max(float) group_key1 +0 1 -1 1 2.200000 1 +1 1 (null) 3 3.300000 2 +2 2 -4 6 6.600000 3 + TABLE + end + + test("multiple") do + assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).max.to_s) + max(int) max(uint) max(float) group_key1 group_key2 +0 -1 1 2.200000 1 1 +1 (null) 3 3.300000 2 1 +2 -4 4 4.400000 3 1 +3 -5 6 6.600000 3 2 + TABLE + end + end + + sub_test_case("#aggregate") do + test("function()") do + group = @table.group(:group_key1, :group_key2) + assert_equal(<<-TABLE, group.aggregate("count(int)", "sum(uint)").to_s) + count(int) sum(uint) group_key1 group_key2 +0 2 1 1 1 +1 0 3 2 1 +2 1 4 3 1 +3 2 11 3 2 + TABLE + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-list-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-list-array-builder.rb new file mode 100644 index 000000000..aee31e73b --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-list-array-builder.rb @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ListArrayBuilderTest < Test::Unit::TestCase + def setup + @data_type = Arrow::ListDataType.new(name: "visible", type: :boolean) + @builder = Arrow::ListArrayBuilder.new(@data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal(nil, array[0]) + end + + test("Array") do + @builder.append_value([true, false, true]) + array = @builder.finish + assert_equal([true, false, true], array[0].to_a) + end + end + + sub_test_case("#append_values") do + test("[nil, Array]") do + @builder.append_values([[false], nil, [true, false, true]]) + array = @builder.finish + assert_equal([ + [false], + nil, + [true, false, true], + ], + array.collect {|list| list ? list.to_a : nil}) + end + + test("is_valids") do + @builder.append_values([[false], [true, true], [true, false, true]], + [true, false, true]) + array = @builder.finish + assert_equal([ + [false], + nil, + [true, false, true], + ], + array.collect {|list| list ? list.to_a : nil}) + end + end + + sub_test_case("#append") do + test("backward compatibility") do + @builder.append + @builder.value_builder.append(true) + @builder.value_builder.append(false) + @builder.append + @builder.value_builder.append(true) + array = @builder.finish + + assert_equal([ + [true, false], + [true], + ], + array.collect(&:to_a)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-list-array.rb b/src/arrow/ruby/red-arrow/test/test-list-array.rb new file mode 100644 index 000000000..c1f762492 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-list-array.rb @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ListArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + data_type = Arrow::ListDataType.new(name: "visible", type: :boolean) + values = [ + [true, false], + nil, + [false, true, false], + ] + array = Arrow::ListArray.new(data_type, values) + assert_equal(values, + array.collect {|value| value ? value.to_a : nil}) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-list-data-type.rb b/src/arrow/ruby/red-arrow/test/test-list-data-type.rb new file mode 100644 index 000000000..ada46394d --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-list-data-type.rb @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ListDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Arrow::Field") do + field = Arrow::Field.new(:tag, :string) + assert_equal("list<tag: string>", + Arrow::ListDataType.new(field).to_s) + end + + test("name: String") do + assert_equal("list<tag: string>", + Arrow::ListDataType.new(name: "tag", type: :string).to_s) + end + + test("field: Arrow::Field") do + field = Arrow::Field.new(:tag, :string) + assert_equal("list<tag: string>", + Arrow::ListDataType.new(field: field).to_s) + end + + test("field: Hash") do + field_description = {name: "tag", type: :string} + assert_equal("list<tag: string>", + Arrow::ListDataType.new(field: field_description).to_s) + end + + test("Arrow::DataType") do + data_type = Arrow::BooleanDataType.new + assert_equal("list<item: bool>", + Arrow::ListDataType.new(data_type).to_s) + end + + test("String") do + assert_equal("list<item: bool>", + Arrow::ListDataType.new("boolean").to_s) + end + + test("Symbol") do + assert_equal("list<item: bool>", + Arrow::ListDataType.new(:boolean).to_s) + end + + test("[data type name, additional information]") do + assert_equal("list<item: time32[ms]>", + Arrow::ListDataType.new([:time32, :milli]).to_s) + end + + test("type: Symbol") do + assert_equal("list<item: bool>", + Arrow::ListDataType.new(type: :boolean).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-map-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-map-array-builder.rb new file mode 100644 index 000000000..80e571448 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-map-array-builder.rb @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class MapArrayBuilderTest < Test::Unit::TestCase + def setup + key_type = Arrow::StringDataType.new + item_type = Arrow::Int16DataType.new + data_type = Arrow::MapDataType.new(key_type, item_type) + @builder = Arrow::MapArrayBuilder.new(data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal([nil], array.collect {|value| value}) + end + + test("Hash") do + @builder.append_value({"a" => 0, "b" => 1}) + @builder.append_value({"c" => 0, "d" => 1}) + array = @builder.finish + assert_equal([ + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1} + ], + array.collect {|value| value}) + end + + test("#each") do + @builder.append_value([["a", 0], ["b", 1]]) + @builder.append_value([["c", 0], ["d", 1]]) + array = @builder.finish + assert_equal([ + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1} + ], + array.collect {|value| value}) + end + end + + sub_test_case("#append_values") do + test("[nil]") do + @builder.append_values([nil]) + array = @builder.finish + assert_equal([nil], array.collect {|value| value}) + end + + test("[Hash]") do + @builder.append_values([{"a" => 0, "b" => 1}, {"c" => 0, "d" => 1}]) + array = @builder.finish + assert_equal([ + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1} + ], + array.collect {|value| value}) + end + + test("[#each]") do + @builder.append_values([[["a", 0], ["b", 1]], [["c", 0], ["d", 1]]]) + array = @builder.finish + assert_equal([ + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1} + ], + array.collect {|value| value}) + end + + test("[nil, Hash, #each]") do + @builder.append_values([nil, {"a" => 0, "b" => 1}, [["c", 0], ["d", 1]]]) + array = @builder.finish + assert_equal([ + nil, + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1} + ], + array.collect {|value| value}) + end + + test("is_valids") do + @builder.append_values([ + {"a" => 0, "b" => 1}, + {"c" => 0, "d" => 1}, + {"e" => 0, "f" => 1} + ], + [true, false, true]) + array = @builder.finish + assert_equal([ + {"a" => 0, "b" => 1}, + nil, + {"e" => 0, "f" => 1} + ], + array.collect {|value| value}) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-map-array.rb b/src/arrow/ruby/red-arrow/test/test-map-array.rb new file mode 100644 index 000000000..9f4c1ff57 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-map-array.rb @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class MapArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + key_type = Arrow::StringDataType.new + item_type = Arrow::Int16DataType.new + data_type = Arrow::MapDataType.new(key_type, item_type) + values = [ + {"a" => 0, "b" => 1}, + nil, + {"c" => 0, "d" => 1} + ] + array = Arrow::MapArray.new(data_type, values) + assert_equal(values, array.collect {|value| value}) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-map-data-type.rb b/src/arrow/ruby/red-arrow/test/test-map-data-type.rb new file mode 100644 index 000000000..cdbbc2ed1 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-map-data-type.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class MapDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + def setup + @key = :int8 + @item = :string + end + + test("ordered arguments") do + assert_equal("map<int8, string>", + Arrow::MapDataType.new(@key, @item).to_s) + end + + test("description") do + assert_equal("map<int8, string>", + Arrow::MapDataType.new(key: @key, + item: @item).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-memory-view.rb b/src/arrow/ruby/red-arrow/test/test-memory-view.rb new file mode 100644 index 000000000..0b9c98c40 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-memory-view.rb @@ -0,0 +1,434 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class MemoryViewTest < Test::Unit::TestCase + def setup + unless Fiddle.const_defined?(:MemoryView) + omit("Fiddle::MemoryView is needed") + end + unless Fiddle::MemoryView.respond_to?(:export) + omit("Fiddle::MemoryView.export is needed") + end + end + + def little_endian? + [1].pack("s") == [1].pack("s<") + end + + test("BooleanArray") do + array = Arrow::BooleanArray.new([true] * 9) + Fiddle::MemoryView.export(array) do |memory_view| + if little_endian? + template = "b" + else + template = "B" + end + assert_equal([ + "#{template}8", + 1, + 2, + [(("1" * 9) + ("0" * 7))].pack("#{template}*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Int8Array") do + values = [-(2 ** 7), 0, (2 ** 7) - 1] + array = Arrow::Int8Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "c", + 1, + values.size, + values.pack("c*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Int16Array") do + values = [-(2 ** 15), 0, (2 ** 15) - 1] + array = Arrow::Int16Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "s", + 2, + 2 * values.size, + values.pack("s*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Int32Array") do + values = [-(2 ** 31), 0, (2 ** 31) - 1] + array = Arrow::Int32Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "l", + 4, + 4 * values.size, + values.pack("l*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Int64Array") do + values = [-(2 ** 63), 0, (2 ** 63) - 1] + array = Arrow::Int64Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q", + 8, + 8 * values.size, + values.pack("q*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("UInt8Array") do + values = [0, (2 ** 8) - 1] + array = Arrow::UInt8Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "C", + 1, + values.size, + values.pack("C*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("UInt16Array") do + values = [0, (2 ** 16) - 1] + array = Arrow::UInt16Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "S", + 2, + 2 * values.size, + values.pack("S*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("UInt32Array") do + values = [0, (2 ** 32) - 1] + array = Arrow::UInt32Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "L", + 4, + 4 * values.size, + values.pack("L*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("UInt64Array") do + values = [(2 ** 64) - 1] + array = Arrow::UInt64Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "Q", + 8, + 8 * values.size, + values.pack("Q*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("FloatArray") do + values = [-1.1, 0.0, 1.1] + array = Arrow::FloatArray.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "f", + 4, + 4 * values.size, + values.pack("f*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("DoubleArray") do + values = [-1.1, 0.0, 1.1] + array = Arrow::DoubleArray.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "d", + 8, + 8 * values.size, + values.pack("d*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("FixedSizeBinaryArray") do + values = ["\x01\x02", "\x03\x04", "\x05\x06"] + data_type = Arrow::FixedSizeBinaryDataType.new(2) + array = Arrow::FixedSizeBinaryArray.new(data_type, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "C2", + 2, + 2 * values.size, + values.join("").b, + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Date32Array") do + n_days_since_epoch = 17406 # 2017-08-28 + values = [n_days_since_epoch] + array = Arrow::Date32Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "l", + 4, + 4 * values.size, + values.pack("l*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Date64Array") do + n_msecs_since_epoch = 1503878400000 # 2017-08-28T00:00:00Z + values = [n_msecs_since_epoch] + array = Arrow::Date64Array.new(values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q", + 8, + 8 * values.size, + values.pack("q*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Time32Array") do + values = [1, 2, 3] + array = Arrow::Time32Array.new(:milli, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "l", + 4, + 4 * values.size, + values.pack("l*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Time64Array") do + values = [1, 2, 3] + array = Arrow::Time64Array.new(:nano, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q", + 8, + 8 * values.size, + values.pack("q*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("TimestampArray") do + values = [1, 2, 3] + array = Arrow::TimestampArray.new(:micro, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q", + 8, + 8 * values.size, + values.pack("q*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Decimal128Array") do + values = [ + Arrow::Decimal128.new("10.1"), + Arrow::Decimal128.new("11.1"), + Arrow::Decimal128.new("10.2"), + ] + data_type = Arrow::Decimal128DataType.new(3, 1) + array = Arrow::Decimal128Array.new(data_type, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q2", + 16, + 16 * values.size, + values.collect {|value| value.to_bytes.to_s}.join(""), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Decimal256Array") do + values = [ + Arrow::Decimal256.new("10.1"), + Arrow::Decimal256.new("11.1"), + Arrow::Decimal256.new("10.2"), + ] + data_type = Arrow::Decimal256DataType.new(3, 1) + array = Arrow::Decimal256Array.new(data_type, values) + Fiddle::MemoryView.export(array) do |memory_view| + assert_equal([ + "q4", + 32, + 32 * values.size, + values.collect {|value| value.to_bytes.to_s}.join(""), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end + + test("Buffer") do + values = [0, nil, nil] * 3 + array = Arrow::Int8Array.new(values) + buffer = array.null_bitmap + Fiddle::MemoryView.export(buffer) do |memory_view| + if little_endian? + template = "b" + else + template = "B" + end + assert_equal([ + "#{template}8", + 1, + 2, + ["100" * 3].pack("#{template}*"), + ], + [ + memory_view.format, + memory_view.item_size, + memory_view.byte_size, + memory_view.to_s, + ]) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-null-array.rb b/src/arrow/ruby/red-arrow/test/test-null-array.rb new file mode 100644 index 000000000..c5d061636 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-null-array.rb @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class NullArrayTest < Test::Unit::TestCase + test("#[]") do + array = Arrow::NullArray.new(1) + assert_nil(array[0]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-orc.rb b/src/arrow/ruby/red-arrow/test/test-orc.rb new file mode 100644 index 000000000..b882da0a1 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-orc.rb @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ORCTest < Test::Unit::TestCase + include Helper::Fixture + + def setup + omit("Require Apache Arrow ORC") unless Arrow.const_defined?(:ORCFileReader) + @orc_path = fixture_path("TestOrcFile.test1.orc") + end + + def pp_values(values) + "[\n " + values.collect(&:inspect).join(",\n ") + "\n]" + end + + sub_test_case("load") do + test("default") do + table = Arrow::Table.load(@orc_path) + dump = table.columns.collect do |column| + [ + column.field.to_s, + column.data.chunks.collect(&:to_s), + ] + end + assert_equal([ + ["boolean1: bool", [pp_values([false, true])]], + ["byte1: int8", [pp_values([1, 100])]], + ["short1: int16", [pp_values([1024, 2048])]], + ["int1: int32", [pp_values([65536, 65536])]], + [ + "long1: int64", + [pp_values([9223372036854775807, 9223372036854775807])], + ], + ["float1: float", [pp_values([1, 2])]], + ["double1: double", [pp_values([-15, -5])]], + ["bytes1: binary", ["[\n 0001020304,\n \n]"]], + ["string1: string", [pp_values(["hi", "bye"])]], + [ + "middle: " + + "struct<list: " + + "list<item: struct<int1: int32, string1: string>>>", + [ + <<-STRUCT.chomp +-- is_valid: all not null +-- child 0 type: list<item: struct<int1: int32, string1: string>> + [ + -- is_valid: all not null + -- child 0 type: int32 + [ + 1, + 2 + ] + -- child 1 type: string + [ + "bye", + "sigh" + ], + -- is_valid: all not null + -- child 0 type: int32 + [ + 1, + 2 + ] + -- child 1 type: string + [ + "bye", + "sigh" + ] + ] + STRUCT + ] + ], + [ + "list: list<item: struct<int1: int32, string1: string>>", + [ + <<-LIST.chomp +[ + -- is_valid: all not null + -- child 0 type: int32 + [ + 3, + 4 + ] + -- child 1 type: string + [ + "good", + "bad" + ], + -- is_valid: all not null + -- child 0 type: int32 + [ + 100000000, + -100000, + 1234 + ] + -- child 1 type: string + [ + "cat", + "in", + "hat" + ] +] + LIST + ] + ], + [ + "map: map<string, struct<int1: int32, string1: string>>", + [ + <<-MAP.chomp +[ + keys: + [] + values: + -- is_valid: all not null + -- child 0 type: int32 + [] + -- child 1 type: string + [], + keys: + [ + "chani", + "mauddib" + ] + values: + -- is_valid: all not null + -- child 0 type: int32 + [ + 5, + 1 + ] + -- child 1 type: string + [ + "chani", + "mauddib" + ] +] + MAP + ], + ], + ], + dump) + end + + test(":field_indexes") do + table = Arrow::Table.load(@orc_path, field_indexes: [1, 3]) + dump = table.columns.collect do |column| + [ + column.field.to_s, + column.data.chunks.collect(&:to_s), + ] + end + assert_equal([ + ["boolean1: bool", [pp_values([false, true])]], + ["short1: int16", [pp_values([1024, 2048])]], + ], + dump) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-record-batch-builder.rb b/src/arrow/ruby/red-arrow/test/test-record-batch-builder.rb new file mode 100644 index 000000000..988e02043 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-record-batch-builder.rb @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RecordBatchBuilderTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Schema") do + schema = Arrow::Schema.new(visible: :boolean, + count: :uint32) + builder = Arrow::RecordBatchBuilder.new(schema) + assert_equal(schema, + builder.schema) + end + + test("Hash") do + builder = Arrow::RecordBatchBuilder.new(visible: :boolean, + count: :uint32) + assert_equal(Arrow::Schema.new(visible: :boolean, + count: :uint32), + builder.schema) + end + end + + sub_test_case("instance methods") do + def setup + @schema = Arrow::Schema.new(visible: :boolean, + count: :uint32) + @builder = Arrow::RecordBatchBuilder.new(@schema) + end + + sub_test_case("#[]") do + test("String") do + assert_equal(Arrow::BooleanDataType.new, + @builder["visible"].value_data_type) + end + + test("Symbol") do + assert_equal(Arrow::BooleanDataType.new, + @builder[:visible].value_data_type) + end + + test("Integer") do + assert_equal(Arrow::UInt32DataType.new, + @builder[1].value_data_type) + end + end + + test("#append") do + records = [ + {visible: true, count: 1}, + ] + columns = { + visible: [false], + count: [2], + } + arrays = [ + Arrow::BooleanArray.new([true, false]), + Arrow::UInt32Array.new([1, 2]), + ] + @builder.append(records, columns) + assert_equal(Arrow::RecordBatch.new(@schema, + arrays[0].length, + arrays), + @builder.flush) + end + + test("#append_records") do + records = [ + {visible: true, count: 1}, + {visible: true, count: 2, garbage: "garbage"}, + {visible: true}, + [false, 4], + nil, + [true], + ] + arrays = [ + Arrow::BooleanArray.new([true, true, true, false, nil, true]), + Arrow::UInt32Array.new([1, 2, nil, 4, nil, nil]), + ] + @builder.append_records(records) + assert_equal(Arrow::RecordBatch.new(@schema, + arrays[0].length, + arrays), + @builder.flush) + end + + test("#append_columns") do + columns = { + visible: [true, true, true, false, nil, true], + count: [1, 2, nil, 4, nil, nil], + } + arrays = [ + Arrow::BooleanArray.new(columns[:visible]), + Arrow::UInt32Array.new(columns[:count]), + ] + @builder.append_columns(columns) + assert_equal(Arrow::RecordBatch.new(@schema, + arrays[0].length, + arrays), + @builder.flush) + end + + test("#column_builders") do + column_builders = [ + @builder.get_column_builder(0), + @builder.get_column_builder(1), + ] + assert_equal(column_builders, + @builder.column_builders) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-record-batch-file-reader.rb b/src/arrow/ruby/red-arrow/test/test-record-batch-file-reader.rb new file mode 100644 index 000000000..57b02abf9 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-record-batch-file-reader.rb @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RecordBatchFileReaderTest < Test::Unit::TestCase + test("write/read") do + fields = [ + Arrow::Field.new("uint8", :uint8), + Arrow::Field.new("uint16", :uint16), + Arrow::Field.new("uint32", :uint32), + Arrow::Field.new("uint64", :uint64), + Arrow::Field.new("int8", :int8), + Arrow::Field.new("int16", :int16), + Arrow::Field.new("int32", :int32), + Arrow::Field.new("int64", :int64), + Arrow::Field.new("float", :float), + Arrow::Field.new("double", :double), + ] + schema = Arrow::Schema.new(fields) + + tempfile = Tempfile.new(["batch", ".arrow"]) + Arrow::FileOutputStream.open(tempfile.path, false) do |output| + Arrow::RecordBatchFileWriter.open(output, schema) do |writer| + uints = [1, 2, 4, 8] + ints = [1, -2, 4, -8] + floats = [1.1, -2.2, 4.4, -8.8] + columns = [ + Arrow::UInt8Array.new(uints), + Arrow::UInt16Array.new(uints), + Arrow::UInt32Array.new(uints), + Arrow::UInt64Array.new(uints), + Arrow::Int8Array.new(ints), + Arrow::Int16Array.new(ints), + Arrow::Int32Array.new(ints), + Arrow::Int64Array.new(ints), + Arrow::FloatArray.new(floats), + Arrow::DoubleArray.new(floats), + ] + + record_batch = Arrow::RecordBatch.new(schema, 4, columns) + writer.write_record_batch(record_batch) + end + end + + Arrow::MemoryMappedInputStream.open(tempfile.path) do |input| + reader = Arrow::RecordBatchFileReader.new(input) + reader.each do |record_batch| + assert_equal([ + { + "uint8" => 1, + "uint16" => 1, + "uint32" => 1, + "uint64" => 1, + "int8" => 1, + "int16" => 1, + "int32" => 1, + "int64" => 1, + "float" => 1.100000023841858, + "double" => 1.1, + }, + { + "uint8" => 2, + "uint16" => 2, + "uint32" => 2, + "uint64" => 2, + "int8" => -2, + "int16" => -2, + "int32" => -2, + "int64" => -2, + "float" => -2.200000047683716, + "double" => -2.2, + }, + { + "uint8" => 4, + "uint16" => 4, + "uint32" => 4, + "uint64" => 4, + "int8" => 4, + "int16" => 4, + "int32" => 4, + "int64" => 4, + "float" => 4.400000095367432, + "double" => 4.4, + }, + { + "uint8" => 8, + "uint16" => 8, + "uint32" => 8, + "uint64" => 8, + "int8" => -8, + "int16" => -8, + "int32" => -8, + "int64" => -8, + "float" => -8.800000190734863, + "double" => -8.8, + }, + ], + record_batch.collect(&:to_h)) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-record-batch-iterator.rb b/src/arrow/ruby/red-arrow/test/test-record-batch-iterator.rb new file mode 100644 index 000000000..88f3ecaac --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-record-batch-iterator.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RecordBatchIteratorTest < Test::Unit::TestCase + def setup + @schema = Arrow::Schema.new(visible: :boolean, + count: :uint32) + @record_batches = [ + Arrow::RecordBatch.new(@schema, + visible: [true], + count: [1]), + Arrow::RecordBatch.new(@schema, + visible: [false, nil], + count: [nil, 3]), + ] + @iterator = Arrow::RecordBatchIterator.new(@record_batches) + end + + def test_to_a + assert_equal(@record_batches, + @iterator.to_a) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-record-batch-reader.rb b/src/arrow/ruby/red-arrow/test/test-record-batch-reader.rb new file mode 100644 index 000000000..1becdf5b6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-record-batch-reader.rb @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestRecordBatchReader < Test::Unit::TestCase + sub_test_case(".try_convert") do + test("Arrow::RecordBatch") do + record_batch = + Arrow::RecordBatch.new("count" => [1, 2, 3], + "private" => [true, false, true]) + reader = Arrow::RecordBatchReader.try_convert(record_batch) + assert_equal(record_batch, + reader.read_next) + end + + test("[Arrow::RecordBatch]") do + record_batch = + Arrow::RecordBatch.new("count" => [1, 2, 3], + "private" => [true, false, true]) + reader = Arrow::RecordBatchReader.try_convert([record_batch]) + assert_equal(record_batch, + reader.read_next) + end + + test("Arrow::Table") do + table = Arrow::Table.new("count" => [1, 2, 3], + "private" => [true, false, true]) + reader = Arrow::RecordBatchReader.try_convert(table) + assert_equal(table, + reader.read_all) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-record-batch.rb b/src/arrow/ruby/red-arrow/test/test-record-batch.rb new file mode 100644 index 000000000..e94c26f2e --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-record-batch.rb @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RecordBatchTest < Test::Unit::TestCase + sub_test_case(".new") do + def setup + @schema = Arrow::Schema.new(visible: :boolean, + count: :uint32) + end + + test("[raw_table]") do + raw_table = { + visible: [true, nil, false], + count: [1, nil, 3], + } + record_batch = Arrow::RecordBatch.new(raw_table) + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => nil, "count" => nil}, + {"visible" => false, "count" => 3}, + ], + record_batch.each_record.collect(&:to_h)) + end + + test("[Schema, records]") do + records = [ + {visible: true, count: 1}, + nil, + [false, 3], + ] + record_batch = Arrow::RecordBatch.new(@schema, records) + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => nil, "count" => nil}, + {"visible" => false, "count" => 3}, + ], + record_batch.each_record.collect(&:to_h)) + end + + test("[Schema, columns]") do + columns = { + visible: [true, nil, false], + count: [1, 2, nil], + } + record_batch = Arrow::RecordBatch.new(@schema, columns) + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => nil, "count" => 2}, + {"visible" => false, "count" => nil}, + ], + record_batch.each_record.collect(&:to_h)) + end + + test("[Schema, n_rows, columns]") do + columns = [ + Arrow::BooleanArray.new([true, nil, false]), + Arrow::UInt32Array.new([1, 2, nil]), + ] + n_rows = columns[0].length + record_batch = Arrow::RecordBatch.new(@schema, n_rows, columns) + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => nil, "count" => 2}, + {"visible" => false, "count" => nil}, + ], + record_batch.each_record.collect(&:to_h)) + end + end + + sub_test_case("instance methods") do + def setup + @schema = Arrow::Schema.new(count: :uint32) + @counts = Arrow::UInt32Array.new([1, 2, 4, 8]) + @record_batch = Arrow::RecordBatch.new(@schema, @counts.length, [@counts]) + end + + sub_test_case("#each") do + test("default") do + records = [] + @record_batch.each do |record| + records << [record, record.index] + end + assert_equal([ + [0, 0], + [1, 1], + [2, 2], + [3, 3], + ], + records.collect {|record, i| [record.index, i]}) + end + + test("reuse_record: true") do + records = [] + @record_batch.each(reuse_record: true) do |record| + records << [record, record.index] + end + assert_equal([ + [3, 0], + [3, 1], + [3, 2], + [3, 3], + ], + records.collect {|record, i| [record.index, i]}) + end + end + + test("#to_table") do + assert_equal(Arrow::Table.new(@schema, [@counts]), + @record_batch.to_table) + end + + sub_test_case("#==") do + test("Arrow::RecordBatch") do + assert do + @record_batch == @record_batch + end + end + + test("not Arrow::RecordBatch") do + assert do + not (@record_batch == 29) + end + end + end + + sub_test_case("#[]") do + def setup + @record_batch = Arrow::RecordBatch.new(a: [true], + b: [true], + c: [true], + d: [true], + e: [true], + f: [true], + g: [true]) + end + + test("[String]") do + assert_equal(Arrow::Column.new(@record_batch, 0), + @record_batch["a"]) + end + + test("[Symbol]") do + assert_equal(Arrow::Column.new(@record_batch, 1), + @record_batch[:b]) + end + + test("[Integer]") do + assert_equal(Arrow::Column.new(@record_batch, 6), + @record_batch[-1]) + end + + test("[Range]") do + assert_equal(Arrow::RecordBatch.new(d: [true], + e: [true]), + @record_batch[3..4]) + end + + test("[[Symbol, String, Integer, Range]]") do + assert_equal(Arrow::RecordBatch.new(c: [true], + a: [true], + g: [true], + d: [true], + e: [true]), + @record_batch[[:c, "a", -1, 3..4]]) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-rolling-window.rb b/src/arrow/ruby/red-arrow/test/test-rolling-window.rb new file mode 100644 index 000000000..4158ad162 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-rolling-window.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class RollingWindowTest < Test::Unit::TestCase + include Helper::Fixture + + def setup + raw_table = { + :number => Arrow::Int32Array.new([1, -2, nil, 4, 6, 3]), + } + @table = Arrow::Table.new(raw_table) + end + + test("#lag") do + assert_equal(<<-ARRAY.chomp, @table.window.lag(:number).to_s) +[ + null, + -3, + null, + null, + 2, + -3 +] + ARRAY + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-schema.rb b/src/arrow/ruby/red-arrow/test/test-schema.rb new file mode 100644 index 000000000..20d73b272 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-schema.rb @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SchemaTest < Test::Unit::TestCase + include Helper::Omittable + + def setup + @count_field = Arrow::Field.new("count", :uint32) + @visible_field = Arrow::Field.new("visible", :boolean) + end + + sub_test_case(".new") do + test("[Arrow::Field]") do + fields = [ + @count_field, + @visible_field, + ] + assert_equal("count: uint32\n" + + "visible: bool", + Arrow::Schema.new(fields).to_s) + end + + test("[Arrow::Field, Hash]") do + fields = [ + @count_field, + {name: "visible", type: :boolean}, + ] + assert_equal("count: uint32\n" + + "visible: bool", + Arrow::Schema.new(fields).to_s) + end + + test("{String, Symbol => Arrow::DataType}") do + fields = { + "count" => Arrow::UInt32DataType.new, + :visible => :boolean, + } + assert_equal("count: uint32\n" + + "visible: bool", + Arrow::Schema.new(fields).to_s) + end + + test("{String, Symbol => Hash}") do + fields = { + "count" => {type: :uint32}, + :tags => { + type: :list, + field: { + name: "tag", + type: :string, + }, + }, + } + assert_equal("count: uint32\n" + + "tags: list<tag: string>", + Arrow::Schema.new(fields).to_s) + end + end + + sub_test_case("instance methods") do + def setup + super + @schema = Arrow::Schema.new([@count_field, @visible_field]) + end + + sub_test_case("#[]") do + test("[String]") do + assert_equal([@count_field, @visible_field], + [@schema["count"], @schema["visible"]]) + end + + test("[Symbol]") do + assert_equal([@count_field, @visible_field], + [@schema[:count], @schema[:visible]]) + end + + test("[Integer]") do + assert_equal([@count_field, @visible_field], + [@schema[0], @schema[1]]) + end + + test("[invalid]") do + invalid = [] + message = "field name or index must be String, Symbol or Integer" + message << ": <#{invalid.inspect}>" + assert_raise(ArgumentError.new(message)) do + @schema[invalid] + end + end + end + + sub_test_case("#==") do + test("Arrow::Schema") do + assert do + @schema == @schema + end + end + + test("not Arrow::Schema") do + assert do + not (@schema == 29) + end + end + end + + sub_test_case("#to_s") do + test("show_metadata") do + require_gi_bindings(3, 4, 2) + + schema = @schema.with_metadata("key" => "value") + assert_equal(<<-SCHEMA.chomp, schema.to_s(show_metadata: true)) +count: uint32 +visible: bool +-- metadata -- +key: value + SCHEMA + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-slicer.rb b/src/arrow/ruby/red-arrow/test/test-slicer.rb new file mode 100644 index 000000000..420086690 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-slicer.rb @@ -0,0 +1,487 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SlicerTest < Test::Unit::TestCase + def setup + @count_field = Arrow::Field.new("count", :uint32) + @visible_field = Arrow::Field.new("visible", :boolean) + schema = Arrow::Schema.new([@count_field, @visible_field]) + count_arrays = [ + Arrow::UInt32Array.new([0, 1, 2]), + Arrow::UInt32Array.new([4, 8, 16]), + Arrow::UInt32Array.new([32, 64, nil]), + Arrow::UInt32Array.new([256]), + ] + visible_arrays = [ + Arrow::BooleanArray.new([nil, true, false, nil]), + Arrow::BooleanArray.new([true]), + Arrow::BooleanArray.new([true, false]), + Arrow::BooleanArray.new([nil]), + Arrow::BooleanArray.new([nil]), + Arrow::BooleanArray.new([true]), + ] + @count_array = Arrow::ChunkedArray.new(count_arrays) + @visible_array = Arrow::ChunkedArray.new(visible_arrays) + @table = Arrow::Table.new(schema, [@count_array, @visible_array]) + end + + sub_test_case("column") do + test("BooleanArray") do + sliced_table = @table.slice do |slicer| + slicer.visible + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 1 true +2 (null) (null) +3 8 true +4 16 true +5 (null) (null) +6 (null) (null) +7 256 true + TABLE + end + + test("not BooleanArray") do + sliced_table = @table.slice do |slicer| + slicer.count + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 1 true +1 2 false +2 4 (null) +3 8 true +4 16 true +5 32 false +6 64 (null) +7 (null) (null) +8 256 true + TABLE + end + end + + sub_test_case("!column") do + test("BooleanArray") do + sliced_table = @table.slice do |slicer| + !slicer.visible + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 2 false +2 (null) (null) +3 32 false +4 (null) (null) +5 (null) (null) + TABLE + end + + test("not BooleanArray") do + sliced_table = @table.slice do |slicer| + !slicer.count + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 (null) (null) + TABLE + end + end + + test("column.null?") do + sliced_table = @table.slice do |slicer| + slicer.visible.null? + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 4 (null) +2 64 (null) +3 (null) (null) + TABLE + end + + test("column.valid?") do + sliced_table = @table.slice do |slicer| + slicer.visible.valid? + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 1 true +1 2 false +2 8 true +3 16 true +4 32 false +5 256 true + TABLE + end + + sub_test_case("column ==") do + test("nil") do + sliced_table = @table.slice do |slicer| + slicer.visible == nil + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 4 (null) +2 64 (null) +3 (null) (null) + TABLE + end + + test("value") do + sliced_table = @table.slice do |slicer| + slicer.visible == true + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 1 true +2 (null) (null) +3 8 true +4 16 true +5 (null) (null) +6 (null) (null) +7 256 true + TABLE + end + end + + sub_test_case("!(column ==)") do + test("nil") do + sliced_table = @table.slice do |slicer| + !(slicer.visible == nil) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 1 true +1 2 false +2 8 true +3 16 true +4 32 false +5 256 true + TABLE + end + + test("value") do + sliced_table = @table.slice do |slicer| + !(slicer.visible == true) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 2 false +2 (null) (null) +3 32 false +4 (null) (null) +5 (null) (null) + TABLE + end + end + + sub_test_case("column !=") do + test("nil") do + sliced_table = @table.slice do |slicer| + slicer.visible != nil + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 1 true +1 2 false +2 8 true +3 16 true +4 32 false +5 256 true + TABLE + end + + test("value") do + sliced_table = @table.slice do |slicer| + slicer.visible != true + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 2 false +2 (null) (null) +3 32 false +4 (null) (null) +5 (null) (null) + TABLE + end + end + + test("column < value") do + sliced_table = @table.slice do |slicer| + slicer.count < 16 + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 2 false +3 4 (null) +4 8 true +5 (null) (null) + TABLE + end + + test("!(column < value)") do + sliced_table = @table.slice do |slicer| + !(slicer.count < 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 16 true +1 32 false +2 64 (null) +3 (null) (null) +4 256 true + TABLE + end + + test("column <= value") do + sliced_table = @table.slice do |slicer| + slicer.count <= 16 + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 2 false +3 4 (null) +4 8 true +5 16 true +6 (null) (null) + TABLE + end + + test("!(column <= value)") do + sliced_table = @table.slice do |slicer| + !(slicer.count <= 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 32 false +1 64 (null) +2 (null) (null) +3 256 true + TABLE + end + + test("column > value") do + sliced_table = @table.slice do |slicer| + slicer.count > 16 + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 32 false +1 64 (null) +2 (null) (null) +3 256 true + TABLE + end + + test("!(column > value)") do + sliced_table = @table.slice do |slicer| + !(slicer.count > 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 2 false +3 4 (null) +4 8 true +5 16 true +6 (null) (null) + TABLE + end + + test("column >= value") do + sliced_table = @table.slice do |slicer| + slicer.count >= 16 + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 16 true +1 32 false +2 64 (null) +3 (null) (null) +4 256 true + TABLE + end + + test("!(column >= value)") do + sliced_table = @table.slice do |slicer| + !(slicer.count >= 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 2 false +3 4 (null) +4 8 true +5 (null) (null) + TABLE + end + + test("column.in") do + sliced_table = @table.slice do |slicer| + slicer.count.in?([1, 4, 16, 64]) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 1 true +1 4 (null) +2 16 true +3 64 (null) + TABLE + end + + test("!column.in") do + sliced_table = @table.slice do |slicer| + !slicer.count.in?([1, 4, 16, 64]) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 2 false +2 8 true +3 32 false +4 (null) (null) +5 256 true + TABLE + end + + test("condition & condition") do + sliced_table = @table.slice do |slicer| + slicer.visible & (slicer.count >= 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 (null) (null) +2 16 true +3 (null) (null) +4 (null) (null) +5 256 true + TABLE + end + + test("condition | condition") do + sliced_table = @table.slice do |slicer| + slicer.visible | (slicer.count >= 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 1 true +2 (null) (null) +3 8 true +4 16 true +5 32 false +6 (null) (null) +7 (null) (null) +8 256 true + TABLE + end + + test("condition ^ condition") do + sliced_table = @table.slice do |slicer| + slicer.visible ^ (slicer.count >= 16) + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 (null) (null) +1 1 true +2 (null) (null) +3 8 true +4 32 false +5 (null) (null) +6 (null) (null) + TABLE + end + + test("select") do + sliced_table = @table.slice do |slicer| + slicer.visible.select do |value| + value.nil? or value + end + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 4 (null) +3 8 true +4 16 true +5 64 (null) +6 (null) (null) +7 256 true + TABLE + end + + test("!select") do + sliced_table = @table.slice do |slicer| + !slicer.visible.select do |value| + value.nil? or value + end + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 2 false +1 32 false + TABLE + end + + test("reject") do + sliced_table = @table.slice do |slicer| + slicer.visible.reject do |value| + value.nil? or value + end + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 2 false +1 32 false + TABLE + end + + test("!reject") do + sliced_table = @table.slice do |slicer| + !slicer.visible.reject do |value| + value.nil? or value + end + end + assert_equal(<<-TABLE, sliced_table.to_s) + count visible +0 0 (null) +1 1 true +2 4 (null) +3 8 true +4 16 true +5 64 (null) +6 (null) (null) +7 256 true + TABLE + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-sort-indices.rb b/src/arrow/ruby/red-arrow/test/test-sort-indices.rb new file mode 100644 index 000000000..b177831fe --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-sort-indices.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SortIndicesTest < Test::Unit::TestCase + def setup + @table = Arrow::Table.new(number1: [16, -1, 2, 32, -4, -4, -8], + number2: [32, 2, -16, 8, 1, 4, 1]) + end + + sub_test_case("Table") do + test("Symbol") do + assert_equal(Arrow::UInt64Array.new([6, 4, 5, 1, 2, 0, 3]), + @table.sort_indices(:number1)) + end + + test("-String") do + assert_equal(Arrow::UInt64Array.new([3, 0, 2, 1, 4, 5, 6]), + @table.sort_indices("-number1")) + end + + test("Symbol, -String") do + assert_equal(Arrow::UInt64Array.new([6, 5, 4, 1, 2, 0, 3]), + @table.sort_indices([:number1, "-number2"])) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-sort-key.rb b/src/arrow/ruby/red-arrow/test/test-sort-key.rb new file mode 100644 index 000000000..0a31f8461 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-sort-key.rb @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SortKeyTest < Test::Unit::TestCase + sub_test_case(".resolve") do + test("SortKey") do + assert_equal(Arrow::SortKey.new("-count"), + Arrow::SortKey.resolve(Arrow::SortKey.new("-count"))) + end + + test("-String") do + assert_equal(Arrow::SortKey.new("-count"), + Arrow::SortKey.resolve("-count")) + end + + test("Symbol, Symbol") do + assert_equal(Arrow::SortKey.new("-count"), + Arrow::SortKey.resolve(:count, :desc)) + end + end + + sub_test_case("#initialize") do + test("String") do + assert_equal("+count", + Arrow::SortKey.new("count").to_s) + end + + test("+String") do + assert_equal("+count", + Arrow::SortKey.new("+count").to_s) + end + + test("-String") do + assert_equal("-count", + Arrow::SortKey.new("-count").to_s) + end + + test("Symbol") do + assert_equal("+-count", + Arrow::SortKey.new(:"-count").to_s) + end + + test("String, Symbol") do + assert_equal("--count", + Arrow::SortKey.new("-count", :desc).to_s) + end + + test("String, String") do + assert_equal("--count", + Arrow::SortKey.new("-count", "desc").to_s) + end + + test("String, SortOrder") do + assert_equal("--count", + Arrow::SortKey.new("-count", + Arrow::SortOrder::DESCENDING).to_s) + end + end + + sub_test_case("#to_s") do + test("recreatable") do + key = Arrow::SortKey.new("-count", :desc) + assert_equal(key, + Arrow::SortKey.new(key.to_s)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-sort-options.rb b/src/arrow/ruby/red-arrow/test/test-sort-options.rb new file mode 100644 index 000000000..0afd65b0f --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-sort-options.rb @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SortOptionsTest < Test::Unit::TestCase + sub_test_case("#initialize") do + test("none") do + options = Arrow::SortOptions.new + assert_equal([], + options.sort_keys.collect(&:to_s)) + end + + test("-String, Symbol") do + options = Arrow::SortOptions.new("-count", :age) + assert_equal(["-count", "+age"], + options.sort_keys.collect(&:to_s)) + end + end + + sub_test_case("instance methods") do + setup do + @options = Arrow::SortOptions.new + end + + sub_test_case("#add_sort_key") do + test("-String") do + @options.add_sort_key("-count") + assert_equal(["-count"], + @options.sort_keys.collect(&:to_s)) + end + + test("-String, Symbol") do + @options.add_sort_key("-count", :desc) + assert_equal(["--count"], + @options.sort_keys.collect(&:to_s)) + end + + test("SortKey") do + @options.add_sort_key(Arrow::SortKey.new("-count")) + assert_equal(["-count"], + @options.sort_keys.collect(&:to_s)) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-sparse-union-data-type.rb b/src/arrow/ruby/red-arrow/test/test-sparse-union-data-type.rb new file mode 100644 index 000000000..e672f82d4 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-sparse-union-data-type.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class SparseUnionDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + def setup + @fields = [ + Arrow::Field.new("visible", :boolean), + { + name: "count", + type: :int32, + }, + ] + end + + test("ordered arguments") do + assert_equal("sparse_union<visible: bool=2, count: int32=9>", + Arrow::SparseUnionDataType.new(@fields, [2, 9]).to_s) + end + + test("description") do + assert_equal("sparse_union<visible: bool=2, count: int32=9>", + Arrow::SparseUnionDataType.new(fields: @fields, + type_codes: [2, 9]).to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-string-dictionary-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-string-dictionary-array-builder.rb new file mode 100644 index 000000000..d6df509ed --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-string-dictionary-array-builder.rb @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class StringDictionaryArrayBuilderTest < Test::Unit::TestCase + def setup + @builder = Arrow::StringDictionaryArrayBuilder.new + end + + sub_test_case("#append_values") do + test("[nil]") do + @builder.append_values([nil]) + array = @builder.finish + assert_equal([ + [], + [nil], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[String]") do + @builder.append_values(["hello"]) + array = @builder.finish + assert_equal([ + ["hello"], + [0], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[Symbol]") do + @builder.append_values([:hello]) + array = @builder.finish + assert_equal([ + ["hello"], + [0], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("[nil, String, Symbol]") do + @builder.append_values([ + nil, + "Hello", + :world, + "world", + ]) + array = @builder.finish + assert_equal([ + ["Hello", "world"], + [nil, 0, 1, 1], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + + test("is_valids") do + @builder.append_values([ + "Hello", + :world, + :goodbye, + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + ["Hello", "goodbye"], + [0, nil, 1], + ], + [ + array.dictionary.to_a, + array.indices.to_a, + ]) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-struct-array-builder.rb b/src/arrow/ruby/red-arrow/test/test-struct-array-builder.rb new file mode 100644 index 000000000..ab0aa5edf --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-struct-array-builder.rb @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class StructArrayBuilderTest < Test::Unit::TestCase + def setup + @data_type = Arrow::StructDataType.new(visible: {type: :boolean}, + count: {type: :uint64}) + @builder = Arrow::StructArrayBuilder.new(@data_type) + end + + sub_test_case("#append_value") do + test("nil") do + @builder.append_value(nil) + array = @builder.finish + assert_equal([ + [false], + [0], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("Array") do + @builder.append_value([true, 1]) + @builder.append_value([]) + @builder.append_value([false]) + array = @builder.finish + assert_equal([ + [true, nil, false], + [1, nil, nil], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("Arrow::Struct") do + source_array = Arrow::StructArray.new(@data_type, [[true, 1]]) + struct = source_array.get_value(0) + @builder.append_value(struct) + array = @builder.finish + assert_equal([ + [true], + [1], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("Hash") do + @builder.append_value(count: 1, visible: true) + @builder.append_value(visible: false) + @builder.append_value(count: 2) + array = @builder.finish + assert_equal([ + [true, false, nil], + [1, nil, 2], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + end + + sub_test_case("#append_values") do + test("[nil]") do + @builder.append_values([nil]) + array = @builder.finish + assert_equal([ + [false], + [0], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("[Array]") do + @builder.append_values([[true, 1]]) + array = @builder.finish + assert_equal([ + [true], + [1], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("[Hash]") do + @builder.append_values([{count: 1, visible: true}]) + array = @builder.finish + assert_equal([ + [true], + [1], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("[nil, Array, Hash]") do + @builder.append_values([ + nil, + [true, 1], + {count: 2, visible: false}, + ]) + array = @builder.finish + assert_equal([ + [false, true, false], + [0, 1, 2], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + + test("is_valids") do + @builder.append_values([ + [true, 1], + [false, 2], + [true, 3], + ], + [ + true, + false, + true, + ]) + array = @builder.finish + assert_equal([ + [true, false, true], + [1, 0, 3], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + end + + sub_test_case("#append") do + test("backward compatibility") do + @builder.append + @builder.get_field_builder(0).append(true) + @builder.get_field_builder(1).append(1) + @builder.append + @builder.get_field_builder(0).append(false) + @builder.get_field_builder(1).append(2) + array = @builder.finish + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => false, "count" => 2}, + ], + [ + array.get_value(0), + array.get_value(1), + ]) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-struct-array.rb b/src/arrow/ruby/red-arrow/test/test-struct-array.rb new file mode 100644 index 000000000..2c01f33ef --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-struct-array.rb @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class StructArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + test("build") do + data_type = Arrow::StructDataType.new(visible: :boolean, + count: :uint64) + values = [ + [true, 1], + nil, + [false, 2], + ] + array = Arrow::StructArray.new(data_type, values) + assert_equal([ + [true, false, false], + [1, 0, 2], + ], + [ + array.find_field(0).to_a, + array.find_field(1).to_a, + ]) + end + end + + sub_test_case("instance methods") do + def setup + @data_type = Arrow::StructDataType.new(visible: {type: :boolean}, + count: {type: :uint64}) + @values = [ + [true, 1], + [false, 2], + ] + @array = Arrow::StructArray.new(@data_type, @values) + end + + test("#[]") do + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => false, "count" => 2}, + ], + @array.to_a) + end + + test("#get_value") do + assert_equal([ + {"visible" => true, "count" => 1}, + {"visible" => false, "count" => 2}, + ], + [ + @array.get_value(0), + @array.get_value(1), + ]) + end + + sub_test_case("#find_field") do + test("Integer") do + assert_equal([ + [true, false], + [1, 2], + ], + [ + @array.find_field(0).to_a, + @array.find_field(1).to_a, + ]) + end + + test("String, Symbol") do + assert_equal([ + [true, false], + [1, 2], + ], + [ + @array.find_field("visible").to_a, + @array.find_field(:count).to_a, + ]) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-struct-data-type.rb b/src/arrow/ruby/red-arrow/test/test-struct-data-type.rb new file mode 100644 index 000000000..d106e38b1 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-struct-data-type.rb @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class StructDataTypeTest < Test::Unit::TestCase + def setup + @count_field = Arrow::Field.new("count", :uint32) + @visible_field = Arrow::Field.new("visible", :boolean) + end + + sub_test_case(".new") do + test("[Arrow::Field]") do + fields = [ + @count_field, + @visible_field, + ] + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + + test("[Hash]") do + fields = [ + {name: "count", data_type: :uint32}, + {name: "visible", data_type: :boolean}, + ] + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + + test("[Arrow::Field, Hash]") do + fields = [ + @count_field, + {name: "visible", data_type: :boolean}, + ] + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + + test("{Arrow::DataType}") do + fields = { + "count" => Arrow::UInt32DataType.new, + "visible" => Arrow::BooleanDataType.new, + } + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + + test("{Hash}") do + fields = { + "count" => {type: :uint32}, + "visible" => {type: :boolean}, + } + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + + test("{String, Symbol}") do + fields = { + "count" => "uint32", + "visible" => :boolean, + } + assert_equal("struct<count: uint32, visible: bool>", + Arrow::StructDataType.new(fields).to_s) + end + end + + sub_test_case("instance methods") do + def setup + super + @data_type = Arrow::StructDataType.new([@count_field, @visible_field]) + end + + sub_test_case("#[]") do + test("[String]") do + assert_equal([@count_field, @visible_field], + [@data_type["count"], @data_type["visible"]]) + end + + test("[Symbol]") do + assert_equal([@count_field, @visible_field], + [@data_type[:count], @data_type[:visible]]) + end + + test("[Integer]") do + assert_equal([@count_field, @visible_field], + [@data_type[0], @data_type[1]]) + end + + test("[invalid]") do + invalid = [] + message = "field name or index must be String, Symbol or Integer" + message << ": <#{invalid.inspect}>" + assert_raise(ArgumentError.new(message)) do + @data_type[invalid] + end + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-table.rb b/src/arrow/ruby/red-arrow/test/test-table.rb new file mode 100644 index 000000000..78361a824 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-table.rb @@ -0,0 +1,925 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TableTest < Test::Unit::TestCase + include Helper::Fixture + + def setup + @count_field = Arrow::Field.new("count", :uint8) + @visible_field = Arrow::Field.new("visible", :boolean) + schema = Arrow::Schema.new([@count_field, @visible_field]) + count_arrays = [ + Arrow::UInt8Array.new([1, 2]), + Arrow::UInt8Array.new([4, 8, 16]), + Arrow::UInt8Array.new([32, 64]), + Arrow::UInt8Array.new([128]), + ] + visible_arrays = [ + Arrow::BooleanArray.new([true, false, nil]), + Arrow::BooleanArray.new([true]), + Arrow::BooleanArray.new([true, false]), + Arrow::BooleanArray.new([nil]), + Arrow::BooleanArray.new([nil]), + ] + @count_array = Arrow::ChunkedArray.new(count_arrays) + @visible_array = Arrow::ChunkedArray.new(visible_arrays) + @table = Arrow::Table.new(schema, [@count_array, @visible_array]) + end + + test("#columns") do + assert_equal([ + Arrow::Column.new(@table, 0), + Arrow::Column.new(@table, 1), + ], + @table.columns) + end + + sub_test_case("#slice") do + test("Arrow::BooleanArray") do + target_rows_raw = [nil, true, true, false, true, false, true, true] + target_rows = Arrow::BooleanArray.new(target_rows_raw) + assert_equal(<<-TABLE, @table.slice(target_rows).to_s) + count visible +0 (null) (null) +1 2 false +2 4 (null) +3 16 true +4 64 (null) +5 128 (null) + TABLE + end + + test("Array: boolean") do + target_rows_raw = [nil, true, true, false, true, false, true, true] + assert_equal(<<-TABLE, @table.slice(target_rows_raw).to_s) + count visible +0 (null) (null) +1 2 false +2 4 (null) +3 16 true +4 64 (null) +5 128 (null) + TABLE + end + + test("Integer: positive") do + assert_equal({"count" => 128, "visible" => nil}, + @table.slice(@table.n_rows - 1).to_h) + end + + test("Integer: negative") do + assert_equal({"count" => 1, "visible" => true}, + @table.slice(-@table.n_rows).to_h) + end + + test("Integer: out of index") do + assert_equal([ + nil, + nil, + ], + [ + @table.slice(@table.n_rows), + @table.slice(-(@table.n_rows + 1)), + ]) + end + + test("Range: positive: include end") do + assert_equal(<<-TABLE, @table.slice(2..4).to_s) + count visible +0 4 (null) +1 8 true +2 16 true + TABLE + end + + test("Range: positive: exclude end") do + assert_equal(<<-TABLE, @table.slice(2...4).to_s) + count visible +0 4 (null) +1 8 true + TABLE + end + + test("Range: negative: include end") do + assert_equal(<<-TABLE, @table.slice(-4..-2).to_s) + count visible +0 16 true +1 32 false +2 64 (null) + TABLE + end + + test("Range: negative: exclude end") do + assert_equal(<<-TABLE, @table.slice(-4...-2).to_s) + count visible +0 16 true +1 32 false + TABLE + end + + test("[from, to]: positive") do + assert_equal(<<-TABLE, @table.slice(0, 2).to_s) + count visible +0 1 true +1 2 false + TABLE + end + + test("[from, to]: negative") do + assert_equal(<<-TABLE, @table.slice(-4, 2).to_s) + count visible +0 16 true +1 32 false + TABLE + end + + test("{key: Number}") do + assert_equal(<<-TABLE, @table.slice(count: 16).to_s) + count visible +0 16 true + TABLE + end + + test("{key: String}") do + table = Arrow::Table.new(name: Arrow::StringArray.new(["a", "b", "c"])) + assert_equal(<<-TABLE, table.slice(name: 'b').to_s) + name +0 b + TABLE + end + + test("{key: true}") do + assert_equal(<<-TABLE, @table.slice(visible: true).to_s) + count visible +0 1 true +1 (null) (null) +2 8 true +3 16 true +4 (null) (null) +5 (null) (null) + TABLE + end + + test("{key: false}") do + assert_equal(<<-TABLE, @table.slice(visible: false).to_s) + count visible +0 2 false +1 (null) (null) +2 32 false +3 (null) (null) +4 (null) (null) + TABLE + end + + test("{key: Range}: beginless include end") do + assert_equal(<<-TABLE, @table.slice(count: ..8).to_s) + count visible +0 1 true +1 2 false +2 4 (null) +3 8 true + TABLE + end + + test("{key: Range}: beginless exclude end") do + assert_equal(<<-TABLE, @table.slice(count: ...8).to_s) + count visible +0 1 true +1 2 false +2 4 (null) + TABLE + end + + test("{key: Range}: endless") do + assert_equal(<<-TABLE, @table.slice(count: 16..).to_s) + count visible +0 16 true +1 32 false +2 64 (null) +3 128 (null) + TABLE + end + + test("{key: Range}: include end") do + assert_equal(<<-TABLE, @table.slice(count: 1..16).to_s) + count visible +0 1 true +1 2 false +2 4 (null) +3 8 true +4 16 true + TABLE + end + + test("{key: Range}: exclude end") do + assert_equal(<<-TABLE, @table.slice(count: 1...16).to_s) + count visible +0 1 true +1 2 false +2 4 (null) +3 8 true + TABLE + end + + test("{key1: Range, key2: true}") do + assert_equal(<<-TABLE, @table.slice(count: 0..8, visible: false).to_s) + count visible +0 2 false +1 (null) (null) +2 (null) (null) +3 (null) (null) + TABLE + end + + sub_test_case("wrong argument") do + test("no arguments") do + message = "wrong number of arguments (given 0, expected 1..2)" + assert_raise(ArgumentError.new(message)) do + @table.slice + end + end + + test("too many arguments") do + message = "wrong number of arguments (given 3, expected 1..2)" + assert_raise(ArgumentError.new(message)) do + @table.slice(1, 2, 3) + end + end + + test("arguments: with block") do + message = "must not specify both arguments and block" + assert_raise(ArgumentError.new(message)) do + @table.slice(1, 2) {} + end + end + + test("offset: too small") do + n_rows = @table.n_rows + offset = -(n_rows + 1) + message = "offset is out of range (-#{n_rows + 1},#{n_rows}): #{offset}" + assert_raise(ArgumentError.new(message)) do + @table.slice(offset, 1) + end + end + + test("offset: too large") do + n_rows = @table.n_rows + offset = n_rows + message = "offset is out of range (-#{n_rows + 1},#{n_rows}): #{offset}" + assert_raise(ArgumentError.new(message)) do + @table.slice(offset, 1) + end + end + end + end + + sub_test_case("#[]") do + def setup + @table = Arrow::Table.new(a: [true], + b: [true], + c: [true], + d: [true], + e: [true], + f: [true], + g: [true]) + end + + test("[String]") do + assert_equal(Arrow::Column.new(@table, 0), + @table["a"]) + end + + test("[Symbol]") do + assert_equal(Arrow::Column.new(@table, 1), + @table[:b]) + end + + test("[Integer]") do + assert_equal(Arrow::Column.new(@table, 6), + @table[-1]) + end + + test("[Range]") do + assert_equal(Arrow::Table.new(d: [true], + e: [true]), + @table[3..4]) + end + + test("[[Symbol, String, Integer, Range]]") do + assert_equal(Arrow::Table.new(c: [true], + a: [true], + g: [true], + d: [true], + e: [true]), + @table[[:c, "a", -1, 3..4]]) + end + end + + sub_test_case("#merge") do + sub_test_case("Hash") do + test("add") do + name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"]) + assert_equal(<<-TABLE, @table.merge(:name => name_array).to_s) + count visible name +0 1 true a +1 2 false b +2 4 (null) c +3 8 true d +4 16 true e +5 32 false f +6 64 (null) g +7 128 (null) h + TABLE + end + + test("remove") do + assert_equal(<<-TABLE, @table.merge(:visible => nil).to_s) + count +0 1 +1 2 +2 4 +3 8 +4 16 +5 32 +6 64 +7 128 + TABLE + end + + test("replace") do + visible_array = Arrow::Int32Array.new([1] * @visible_array.length) + assert_equal(<<-TABLE, @table.merge(:visible => visible_array).to_s) + count visible +0 1 1 +1 2 1 +2 4 1 +3 8 1 +4 16 1 +5 32 1 +6 64 1 +7 128 1 + TABLE + end + end + + sub_test_case("Arrow::Table") do + test("add") do + name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"]) + table = Arrow::Table.new("name" => name_array) + assert_equal(<<-TABLE, @table.merge(table).to_s) + count visible name +0 1 true a +1 2 false b +2 4 (null) c +3 8 true d +4 16 true e +5 32 false f +6 64 (null) g +7 128 (null) h + TABLE + end + + test("replace") do + visible_array = Arrow::Int32Array.new([1] * @visible_array.length) + table = Arrow::Table.new("visible" => visible_array) + assert_equal(<<-TABLE, @table.merge(table).to_s) + count visible +0 1 1 +1 2 1 +2 4 1 +3 8 1 +4 16 1 +5 32 1 +6 64 1 +7 128 1 + TABLE + end + end + end + + test("column name getter") do + assert_equal(Arrow::Column.new(@table, 1), + @table.visible) + end + + sub_test_case("#remove_column") do + test("String") do + assert_equal(<<-TABLE, @table.remove_column("visible").to_s) + count +0 1 +1 2 +2 4 +3 8 +4 16 +5 32 +6 64 +7 128 + TABLE + end + + test("Symbol") do + assert_equal(<<-TABLE, @table.remove_column(:visible).to_s) + count +0 1 +1 2 +2 4 +3 8 +4 16 +5 32 +6 64 +7 128 + TABLE + end + + test("unknown column name") do + assert_raise(KeyError) do + @table.remove_column(:nonexistent) + end + end + + test("Integer") do + assert_equal(<<-TABLE, @table.remove_column(1).to_s) + count +0 1 +1 2 +2 4 +3 8 +4 16 +5 32 +6 64 +7 128 + TABLE + end + + test("negative integer") do + assert_equal(<<-TABLE, @table.remove_column(-1).to_s) + count +0 1 +1 2 +2 4 +3 8 +4 16 +5 32 +6 64 +7 128 + TABLE + end + + test("too small index") do + assert_raise(IndexError) do + @table.remove_column(-3) + end + end + + test("too large index") do + assert_raise(IndexError) do + @table.remove_column(2) + end + end + end + + sub_test_case("#select_columns") do + def setup + raw_table = { + :a => Arrow::UInt8Array.new([1]), + :b => Arrow::UInt8Array.new([1]), + :c => Arrow::UInt8Array.new([1]), + :d => Arrow::UInt8Array.new([1]), + :e => Arrow::UInt8Array.new([1]), + } + @table = Arrow::Table.new(raw_table) + end + + test("names") do + assert_equal(<<-TABLE, @table.select_columns(:c, :a).to_s) + c a +0 1 1 + TABLE + end + + test("range") do + assert_equal(<<-TABLE, @table.select_columns(2...4).to_s) + c d +0 1 1 + TABLE + end + + test("indexes") do + assert_equal(<<-TABLE, @table.select_columns(0, -1, 2).to_s) + a e c +0 1 1 1 + TABLE + end + + test("mixed") do + assert_equal(<<-TABLE, @table.select_columns(:a, -1, 2..3).to_s) + a e c d +0 1 1 1 1 + TABLE + end + + test("block") do + selected_table = @table.select_columns.with_index do |column, i| + column.name == "a" or i.odd? + end + assert_equal(<<-TABLE, selected_table.to_s) + a b d +0 1 1 1 + TABLE + end + + test("names, indexes and block") do + selected_table = @table.select_columns(:a, -1) do |column| + column.name == "a" + end + assert_equal(<<-TABLE, selected_table.to_s) + a +0 1 + TABLE + end + end + + sub_test_case("#save and .load") do + module SaveLoadFormatTests + def test_default + output = create_output(".arrow") + @table.save(output) + assert_equal(@table, Arrow::Table.load(output)) + end + + def test_arrow_file + output = create_output(".arrow") + @table.save(output, format: :arrow_file) + assert_equal(@table, Arrow::Table.load(output, format: :arrow_file)) + end + + def test_batch + output = create_output(".arrow") + @table.save(output, format: :batch) + assert_equal(@table, Arrow::Table.load(output, format: :batch)) + end + + def test_arrow_streaming + output = create_output(".arrow") + @table.save(output, format: :arrow_streaming) + assert_equal(@table, Arrow::Table.load(output, format: :arrow_streaming)) + end + + def test_stream + output = create_output(".arrow") + @table.save(output, format: :stream) + assert_equal(@table, Arrow::Table.load(output, format: :stream)) + end + + def test_csv + output = create_output(".csv") + @table.save(output, format: :csv) + assert_equal(@table, + Arrow::Table.load(output, + format: :csv, + schema: @table.schema)) + end + + def test_csv_gz + output = create_output(".csv.gz") + @table.save(output, + format: :csv, + compression: :gzip) + assert_equal(@table, + Arrow::Table.load(output, + format: :csv, + compression: :gzip, + schema: @table.schema)) + end + + def test_tsv + output = create_output(".tsv") + @table.save(output, format: :tsv) + assert_equal(@table, + Arrow::Table.load(output, + format: :tsv, + schema: @table.schema)) + end + end + + sub_test_case("path") do + sub_test_case(":format") do + include SaveLoadFormatTests + + def create_output(extension) + @file = Tempfile.new(["red-arrow", extension]) + @file.path + end + + sub_test_case("save: auto detect") do + test("csv") do + output = create_output(".csv") + @table.save(output) + assert_equal(@table, + Arrow::Table.load(output, + format: :csv, + schema: @table.schema)) + end + + test("csv.gz") do + output = create_output(".csv.gz") + @table.save(output) + assert_equal(@table, + Arrow::Table.load(output, + format: :csv, + compression: :gzip, + schema: @table.schema)) + end + + test("tsv") do + output = create_output(".tsv") + @table.save(output) + assert_equal(@table, + Arrow::Table.load(output, + format: :tsv, + schema: @table.schema)) + end + end + + sub_test_case("load: auto detect") do + test("arrow: file") do + output = create_output(".arrow") + @table.save(output, format: :arrow_file) + assert_equal(@table, Arrow::Table.load(output)) + end + + test("arrow: streaming") do + output = create_output(".arrow") + @table.save(output, format: :arrow_streaming) + assert_equal(@table, Arrow::Table.load(output)) + end + + test("csv") do + path = fixture_path("with-header.csv") + table = Arrow::Table.load(path, skip_lines: /^\#/) + assert_equal(<<-TABLE, table.to_s) + name score +0 alice 10 +1 bob 29 +2 chris -1 + TABLE + end + + test("csv.gz") do + file = Tempfile.new(["red-arrow", ".csv.gz"]) + file.close + Zlib::GzipWriter.open(file.path) do |gz| + gz.write(<<-CSV) +name,score +alice,10 +bob,29 +chris,-1 + CSV + end + assert_equal(<<-TABLE, Arrow::Table.load(file.path).to_s) + name score +0 alice 10 +1 bob 29 +2 chris -1 + TABLE + end + + test("tsv") do + file = Tempfile.new(["red-arrow", ".tsv"]) + file.puts(<<-TSV) +name\tscore +alice\t10 +bob\t29 +chris\t-1 + TSV + file.close + table = Arrow::Table.load(file.path) + assert_equal(<<-TABLE, table.to_s) + name score +0 alice 10 +1 bob 29 +2 chris -1 + TABLE + end + end + end + end + + sub_test_case("Buffer") do + sub_test_case(":format") do + include SaveLoadFormatTests + + def create_output(extension) + Arrow::ResizableBuffer.new(1024) + end + end + end + end + + test("#pack") do + packed_table = @table.pack + column_n_chunks = packed_table.columns.collect {|c| c.data.n_chunks} + assert_equal([[1, 1], <<-TABLE], [column_n_chunks, packed_table.to_s]) + count visible +0 1 true +1 2 false +2 4 (null) +3 8 true +4 16 true +5 32 false +6 64 (null) +7 128 (null) + TABLE + end + + sub_test_case("#to_s") do + sub_test_case(":format") do + def setup + columns = { + "count" => Arrow::UInt8Array.new([1, 2]), + "visible" => Arrow::BooleanArray.new([true, false]), + } + @table = Arrow::Table.new(columns) + end + + test(":column") do + assert_equal(<<-TABLE, @table.to_s(format: :column)) +count: uint8 +visible: bool +---- +count: + [ + [ + 1, + 2 + ] + ] +visible: + [ + [ + true, + false + ] + ] + TABLE + end + + test(":list") do + assert_equal(<<-TABLE, @table.to_s(format: :list)) +==================== 0 ==================== +count: 1 +visible: true +==================== 1 ==================== +count: 2 +visible: false + TABLE + end + + test(":table") do + assert_equal(<<-TABLE, @table.to_s(format: :table)) + count visible +0 1 true +1 2 false + TABLE + end + + test("invalid") do + message = ":format must be :column, :list, :table or nil: <:invalid>" + assert_raise(ArgumentError.new(message)) do + @table.to_s(format: :invalid) + end + end + end + + sub_test_case("#==") do + test("Arrow::Table") do + assert do + @table == @table + end + end + + test("not Arrow::Table") do + assert do + not (@table == 29) + end + end + end + end + + sub_test_case("#filter") do + def setup + super + @options = Arrow::FilterOptions.new + @options.null_selection_behavior = :emit_null + end + + test("Array: boolean") do + filter = [nil, true, true, false, true, false, true, true] + assert_equal(<<-TABLE, @table.filter(filter, @options).to_s) + count visible +0 (null) (null) +1 2 false +2 4 (null) +3 16 true +4 64 (null) +5 128 (null) + TABLE + end + + test("Arrow::BooleanArray") do + array = [nil, true, true, false, true, false, true, true] + filter = Arrow::BooleanArray.new(array) + assert_equal(<<-TABLE, @table.filter(filter, @options).to_s) + count visible +0 (null) (null) +1 2 false +2 4 (null) +3 16 true +4 64 (null) +5 128 (null) + TABLE + end + + test("Arrow::ChunkedArray") do + filter_chunks = [ + Arrow::BooleanArray.new([nil, true, true]), + Arrow::BooleanArray.new([false, true, false]), + Arrow::BooleanArray.new([true, true]), + ] + filter = Arrow::ChunkedArray.new(filter_chunks) + assert_equal(<<-TABLE, @table.filter(filter, @options).to_s) + count visible +0 (null) (null) +1 2 false +2 4 (null) +3 16 true +4 64 (null) +5 128 (null) + TABLE + end + end + + sub_test_case("#take") do + test("Arrow: boolean") do + indices = [1, 0, 2] + assert_equal(<<-TABLE, @table.take(indices).to_s) + count visible +0 2 false +1 1 true +2 4 (null) + TABLE + end + + test("Arrow::Array") do + indices = Arrow::Int16Array.new([1, 0, 2]) + assert_equal(<<-TABLE, @table.take(indices).to_s) + count visible +0 2 false +1 1 true +2 4 (null) + TABLE + end + + test("Arrow::ChunkedArray") do + chunks = [ + Arrow::Int16Array.new([1, 0]), + Arrow::Int16Array.new([2]) + ] + indices = Arrow::ChunkedArray.new(chunks) + assert_equal(<<-TABLE, @table.take(indices).to_s) + count visible +0 2 false +1 1 true +2 4 (null) + TABLE + end + end + + sub_test_case("#concatenate") do + test("options: :unify_schemas") do + table1 = Arrow::Table.new(a: [true], + b: [false]) + table2 = Arrow::Table.new(b: [false]) + concatenated = table1.concatenate([table2], unify_schemas: true) + assert_equal(<<-TABLE, concatenated.to_s) + a b +0 true false +1 (null) false + TABLE + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-tensor.rb b/src/arrow/ruby/red-arrow/test/test-tensor.rb new file mode 100644 index 000000000..ffa1e3241 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-tensor.rb @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TensorTest < Test::Unit::TestCase + sub_test_case("instance methods") do + def setup + raw_data = [ + 1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12, + ] + data = Arrow::Buffer.new(raw_data.pack("c*")) + shape = [3, 2, 2] + strides = [] + names = ["a", "b", "c"] + @tensor = Arrow::Tensor.new(Arrow::Int8DataType.new, + data, + shape, + strides, + names) + end + + sub_test_case("#==") do + test("Arrow::Tensor") do + assert do + @tensor == @tensor + end + end + + test("not Arrow::Tensor") do + assert do + not (@tensor == 29) + end + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-time.rb b/src/arrow/ruby/red-arrow/test/test-time.rb new file mode 100644 index 000000000..37c098c69 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-time.rb @@ -0,0 +1,288 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TimeTest < Test::Unit::TestCase + sub_test_case("#==") do + test("same unit") do + assert do + Arrow::Time.new(:second, 10) == Arrow::Time.new(:second, 10) + end + end + + test("different unit") do + assert do + Arrow::Time.new(:second, 10) == Arrow::Time.new(:milli, 10 * 1000) + end + end + + test("false") do + assert do + not(Arrow::Time.new(:second, 10) == Arrow::Time.new(:second, 11)) + end + end + end + + sub_test_case("#cast") do + test("same unit") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + casted_time = time.cast(Arrow::TimeUnit::SECOND) + assert_equal([time.unit, time.value], + [casted_time.unit, casted_time.value]) + end + + test("second -> milli") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + casted_time = time.cast(Arrow::TimeUnit::MILLI) + assert_equal([ + Arrow::TimeUnit::MILLI, + time.value * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("second -> micro") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + casted_time = time.cast(Arrow::TimeUnit::MICRO) + assert_equal([ + Arrow::TimeUnit::MICRO, + time.value * 1000 * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("second -> nano") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + casted_time = time.cast(Arrow::TimeUnit::NANO) + assert_equal([ + Arrow::TimeUnit::NANO, + time.value * 1000 * 1000 * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("milli -> second") do + time = Arrow::Time.new(Arrow::TimeUnit::MILLI, 10_200) + casted_time = time.cast(Arrow::TimeUnit::SECOND) + assert_equal([ + Arrow::TimeUnit::SECOND, + 10, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("milli -> micro") do + time = Arrow::Time.new(Arrow::TimeUnit::MILLI, 10_200) + casted_time = time.cast(Arrow::TimeUnit::MICRO) + assert_equal([ + Arrow::TimeUnit::MICRO, + time.value * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("milli -> nano") do + time = Arrow::Time.new(Arrow::TimeUnit::MILLI, 10_200) + casted_time = time.cast(Arrow::TimeUnit::NANO) + assert_equal([ + Arrow::TimeUnit::NANO, + time.value * 1000 * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("micro -> second") do + time = Arrow::Time.new(Arrow::TimeUnit::MICRO, 10_200_300) + casted_time = time.cast(Arrow::TimeUnit::SECOND) + assert_equal([ + Arrow::TimeUnit::SECOND, + 10, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("micro -> milli") do + time = Arrow::Time.new(Arrow::TimeUnit::MICRO, 10_200_300) + casted_time = time.cast(Arrow::TimeUnit::MILLI) + assert_equal([ + Arrow::TimeUnit::MILLI, + 10_200, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("micro -> nano") do + time = Arrow::Time.new(Arrow::TimeUnit::MICRO, 10_200_300) + casted_time = time.cast(Arrow::TimeUnit::NANO) + assert_equal([ + Arrow::TimeUnit::NANO, + time.value * 1000, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("nano -> second") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, 10_200_300_400) + casted_time = time.cast(Arrow::TimeUnit::SECOND) + assert_equal([ + Arrow::TimeUnit::SECOND, + 10, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("nano -> milli") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, 10_200_300_400) + casted_time = time.cast(Arrow::TimeUnit::MILLI) + assert_equal([ + Arrow::TimeUnit::MILLI, + 10_200, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + + test("nano -> micro") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, 10_200_300_400) + casted_time = time.cast(Arrow::TimeUnit::MICRO) + assert_equal([ + Arrow::TimeUnit::MICRO, + 10_200_300, + ], + [ + casted_time.unit, + casted_time.value, + ]) + end + end + + sub_test_case("#to_f") do + test("second") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + assert_in_delta(10.0, time.to_f) + end + + test("milli") do + time = Arrow::Time.new(Arrow::TimeUnit::MILLI, 10_200) + assert_in_delta(10.2, time.to_f) + end + + test("micro") do + time = Arrow::Time.new(Arrow::TimeUnit::MICRO, 10_200_300) + assert_in_delta(10.2003, time.to_f) + end + + test("nano") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, 10_200_300_400) + assert_in_delta(10.2003004, time.to_f) + end + end + + sub_test_case("#positive?") do + test("true") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + assert do + time.positive? + end + end + + test("false") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, -10) + assert do + not time.positive? + end + end + end + + sub_test_case("#negative?") do + test("true") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, -10) + assert do + time.negative? + end + end + + test("false") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, 10) + assert do + not time.negative? + end + end + end + + test("#hour") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, + (5 * 60 * 60) + (12 * 60) + 10) + assert_equal(5, time.hour) + end + + test("#minute") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, + (5 * 60 * 60) + (12 * 60) + 10) + assert_equal(12, time.minute) + end + + test("#second") do + time = Arrow::Time.new(Arrow::TimeUnit::SECOND, + (5 * 60 * 60) + (12 * 60) + 10) + assert_equal(10, time.second) + end + + test("#nano_second") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, 1234) + assert_equal(1234, time.nano_second) + end + + test("#to_s") do + time = Arrow::Time.new(Arrow::TimeUnit::NANO, + -(((5 * 60 * 60) + (12 * 60) + 10) * 1_000_000_000 + + 1234)) + assert_equal("-05:12:10.000001234", + time.to_s) + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-time32-array.rb b/src/arrow/ruby/red-arrow/test/test-time32-array.rb new file mode 100644 index 000000000..b8bb4eb94 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-time32-array.rb @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Time32ArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + sub_test_case("unit") do + test("Arrow::TimeUnit") do + values = [1000 * 10, nil] + array = Arrow::Time32Array.new(Arrow::TimeUnit::MILLI, values) + assert_equal([ + "time32[ms]", + [ + Arrow::Time.new(Arrow::TimeUnit::MILLI, + 1000 * 10), + nil, + ], + ], + [ + array.value_data_type.to_s, + array.to_a, + ]) + end + + test("Symbol") do + values = [60 * 10, nil] + array = Arrow::Time32Array.new(:second, values) + assert_equal([ + "time32[s]", + [ + Arrow::Time.new(Arrow::TimeUnit::SECOND, + 60 * 10), + nil, + ], + ], + [ + array.value_data_type.to_s, + array.to_a, + ]) + end + end + + sub_test_case("values") do + test("Arrow::Time") do + data_type = Arrow::Time32DataType.new(:second) + values = [ + Arrow::Time.new(Arrow::TimeUnit::SECOND, + 60 * 10), + nil, + ] + array = Arrow::Time32Array.new(data_type, values) + assert_equal(values, array.to_a) + end + + test("Integer") do + data_type = Arrow::Time32DataType.new(:second) + values = [60 * 10, nil] + array = Arrow::Time32Array.new(data_type, values) + assert_equal([ + Arrow::Time.new(Arrow::TimeUnit::SECOND, + 60 * 10), + nil, + ], + array.to_a) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-time32-data-type.rb b/src/arrow/ruby/red-arrow/test/test-time32-data-type.rb new file mode 100644 index 000000000..26f17359a --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-time32-data-type.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Time32DataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Arrow::TimeUnit") do + assert_equal("time32[ms]", + Arrow::Time32DataType.new(Arrow::TimeUnit::MILLI).to_s) + end + + test("Symbol") do + assert_equal("time32[ms]", + Arrow::Time32DataType.new(:milli).to_s) + end + + test("unit: Arrow::TimeUnit") do + data_type = Arrow::Time32DataType.new(unit: Arrow::TimeUnit::MILLI) + assert_equal("time32[ms]", + data_type.to_s) + end + + test("unit: Symbol") do + data_type = Arrow::Time32DataType.new(unit: :milli) + assert_equal("time32[ms]", + data_type.to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-time64-array.rb b/src/arrow/ruby/red-arrow/test/test-time64-array.rb new file mode 100644 index 000000000..831af1e35 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-time64-array.rb @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Time64ArrayTest < Test::Unit::TestCase + sub_test_case(".new") do + sub_test_case("unit") do + test("Arrow::TimeUnit") do + values = [1000 * 10, nil] + array = Arrow::Time64Array.new(Arrow::TimeUnit::NANO, values) + assert_equal([ + "time64[ns]", + [ + Arrow::Time.new(Arrow::TimeUnit::NANO, + 1000 * 10), + nil, + ], + ], + [ + array.value_data_type.to_s, + array.to_a, + ]) + end + + test("Symbol") do + values = [1000 * 10, nil] + array = Arrow::Time64Array.new(:micro, values) + assert_equal([ + "time64[us]", + [ + Arrow::Time.new(Arrow::TimeUnit::MICRO, + 1000 * 10), + nil, + ], + ], + [ + array.value_data_type.to_s, + array.to_a, + ]) + end + end + + sub_test_case("values") do + test("Arrow::Time") do + data_type = Arrow::Time64DataType.new(:nano) + values = [ + Arrow::Time.new(Arrow::TimeUnit::NANO, + 1000 * 10), + nil, + ] + array = Arrow::Time64Array.new(data_type, values) + assert_equal(values, array.to_a) + end + + test("Integer") do + data_type = Arrow::Time64DataType.new(:nano) + values = [1000 * 10, nil] + array = Arrow::Time64Array.new(data_type, values) + assert_equal([ + Arrow::Time.new(Arrow::TimeUnit::NANO, + 1000 * 10), + nil, + ], + array.to_a) + end + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-time64-data-type.rb b/src/arrow/ruby/red-arrow/test/test-time64-data-type.rb new file mode 100644 index 000000000..a5f341753 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-time64-data-type.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Time64DataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Arrow::TimeUnit") do + assert_equal("time64[ns]", + Arrow::Time64DataType.new(Arrow::TimeUnit::NANO).to_s) + end + + test("Symbol") do + assert_equal("time64[ns]", + Arrow::Time64DataType.new(:nano).to_s) + end + + test("unit: Arrow::TimeUnit") do + data_type = Arrow::Time64DataType.new(unit: Arrow::TimeUnit::NANO) + assert_equal("time64[ns]", + data_type.to_s) + end + + test("unit: Symbol") do + data_type = Arrow::Time64DataType.new(unit: :nano) + assert_equal("time64[ns]", + data_type.to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-timestamp-array.rb b/src/arrow/ruby/red-arrow/test/test-timestamp-array.rb new file mode 100644 index 000000000..248a2531e --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-timestamp-array.rb @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TimestampArrayTest < Test::Unit::TestCase + test("#[]") do + sec = 1513267750 + usec = 914509 + array = Arrow::TimestampArray.new(:micro, [sec * (10 ** 6) + usec]) + time = Time.at(sec, usec) + assert_equal(time, array[0]) + end + + sub_test_case("#is_in") do + def setup + values = [ + Time.parse("2019-11-18T00:09:11"), + Time.parse("2019-11-18T00:09:12"), + Time.parse("2019-11-18T00:09:13"), + ] + @array = Arrow::TimestampArray.new(:micro, values) + end + + test("Arrow: Array") do + right = [ + Time.parse("2019-11-18T00:09:12"), + ] + assert_equal(Arrow::BooleanArray.new([false, true, false]), + @array.is_in(right)) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/test-timestamp-data-type.rb b/src/arrow/ruby/red-arrow/test/test-timestamp-data-type.rb new file mode 100644 index 000000000..f8ccd3d8b --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/test-timestamp-data-type.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TimestampDataTypeTest < Test::Unit::TestCase + sub_test_case(".new") do + test("Arrow::TimeUnit") do + assert_equal("timestamp[ms]", + Arrow::TimestampDataType.new(Arrow::TimeUnit::MILLI).to_s) + end + + test("Symbol") do + assert_equal("timestamp[ms]", + Arrow::TimestampDataType.new(:milli).to_s) + end + + test("unit: Arrow::TimeUnit") do + data_type = Arrow::TimestampDataType.new(unit: Arrow::TimeUnit::MILLI) + assert_equal("timestamp[ms]", + data_type.to_s) + end + + test("unit: Symbol") do + data_type = Arrow::TimestampDataType.new(unit: :milli) + assert_equal("timestamp[ms]", + data_type.to_s) + end + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-basic-arrays.rb b/src/arrow/ruby/red-arrow/test/values/test-basic-arrays.rb new file mode 100644 index 000000000..c54c7f62d --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-basic-arrays.rb @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesBasicArraysTests + def test_null + target = build(Arrow::NullArray.new(4)) + assert_equal([nil] * 4, target.values) + end + + def test_boolean + values = [true, nil, false] + target = build(Arrow::BooleanArray.new(values)) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + -(2 ** 7), + nil, + (2 ** 7) - 1, + ] + target = build(Arrow::Int8Array.new(values)) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + 0, + nil, + (2 ** 8) - 1, + ] + target = build(Arrow::UInt8Array.new(values)) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + -(2 ** 15), + nil, + (2 ** 15) - 1, + ] + target = build(Arrow::Int16Array.new(values)) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + 0, + nil, + (2 ** 16) - 1, + ] + target = build(Arrow::UInt16Array.new(values)) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + -(2 ** 31), + nil, + (2 ** 31) - 1, + ] + target = build(Arrow::Int32Array.new(values)) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + 0, + nil, + (2 ** 32) - 1, + ] + target = build(Arrow::UInt32Array.new(values)) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + -(2 ** 63), + nil, + (2 ** 63) - 1, + ] + target = build(Arrow::Int64Array.new(values)) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + 0, + nil, + (2 ** 64) - 1, + ] + target = build(Arrow::UInt64Array.new(values)) + assert_equal(values, target.values) + end + + def test_float + values = [ + -1.0, + nil, + 1.0, + ] + target = build(Arrow::FloatArray.new(values)) + assert_equal(values, target.values) + end + + def test_double + values = [ + -1.0, + nil, + 1.0, + ] + target = build(Arrow::DoubleArray.new(values)) + assert_equal(values, target.values) + end + + def test_binary + values = [ + "\x00".b, + nil, + "\xff".b, + ] + target = build(Arrow::BinaryArray.new(values)) + assert_equal(values, target.values) + end + + def test_tring + values = [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + ] + target = build(Arrow::StringArray.new(values)) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + ] + target = build(Arrow::Date32Array.new(values)) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + ] + target = build(Arrow::Date64Array.new(values)) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + ] + target = build(Arrow::TimestampArray.new(:second, values)) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + ] + target = build(Arrow::TimestampArray.new(:milli, values)) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + ] + target = build(Arrow::TimestampArray.new(:micro, values)) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + ] + target = build(Arrow::TimestampArray.new(:nano, values)) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + Arrow::Time.new(unit, 60 * 10), # 00:10:00 + nil, + Arrow::Time.new(unit, 60 * 60 * 2 + 9), # 02:00:09 + ] + target = build(Arrow::Time32Array.new(:second, values)) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), # 00:10:00.123 + nil, + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), # 02:00:09.987 + ] + target = build(Arrow::Time32Array.new(:milli, values)) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + ] + target = build(Arrow::Time64Array.new(:micro, values)) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + ] + target = build(Arrow::Time64Array.new(:nano, values)) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ] + data_type = Arrow::Decimal128DataType.new(8, 2) + target = build(Arrow::Decimal128Array.new(data_type, values)) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ] + data_type = Arrow::Decimal256DataType.new(38, 2) + target = build(Arrow::Decimal256Array.new(data_type, values)) + assert_equal(values, target.values) + end +end + +class ValuesArrayBasicArraysTest < Test::Unit::TestCase + include ValuesBasicArraysTests + + def build(array) + array + end +end + +class ValuesChunkedArrayBasicArraysTest < Test::Unit::TestCase + include ValuesBasicArraysTests + + def build(array) + Arrow::ChunkedArray.new([array]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-dense-union-array.rb b/src/arrow/ruby/red-arrow/test/values/test-dense-union-array.rb new file mode 100644 index 000000000..465ffb9e6 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-dense-union-array.rb @@ -0,0 +1,482 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesDenseUnionArrayTests + def build_data_type(type, type_codes) + field_description = {} + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::DenseUnionDataType.new(fields: [ + field_description.merge(name: "0"), + field_description.merge(name: "1"), + ], + type_codes: type_codes) + end + + def build_array(type, values) + type_codes = [0, 1] + data_type = build_data_type(type, type_codes) + type_ids = [] + offsets = [] + arrays = data_type.fields.collect do |field| + sub_schema = Arrow::Schema.new([field]) + sub_records = [] + values.each do |value| + next if value.nil? + next unless value.key?(field.name) + sub_records << [value[field.name]] + end + sub_record_batch = Arrow::RecordBatch.new(sub_schema, + sub_records) + sub_record_batch.columns[0].data + end + values.each do |value| + if value.key?("0") + type_id = type_codes[0] + type_ids << type_id + offsets << (type_ids.count(type_id) - 1) + elsif value.key?("1") + type_id = type_codes[1] + type_ids << type_id + offsets << (type_ids.count(type_id) - 1) + end + end + Arrow::DenseUnionArray.new(data_type, + Arrow::Int8Array.new(type_ids), + Arrow::Int32Array.new(offsets), + arrays) + end + + def test_null + values = [ + {"0" => nil}, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + {"0" => true}, + {"1" => nil}, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + {"0" => -(2 ** 7)}, + {"1" => nil}, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + {"0" => (2 ** 8) - 1}, + {"1" => nil}, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + {"0" => -(2 ** 15)}, + {"1" => nil}, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + {"0" => (2 ** 16) - 1}, + {"1" => nil}, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + {"0" => -(2 ** 31)}, + {"1" => nil}, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + {"0" => (2 ** 32) - 1}, + {"1" => nil}, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + {"0" => -(2 ** 63)}, + {"1" => nil}, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + {"0" => (2 ** 64) - 1}, + {"1" => nil}, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + {"0" => -1.0}, + {"1" => nil}, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + {"0" => -1.0}, + {"1" => nil}, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + {"0" => "\xff".b}, + {"1" => nil}, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + {"0" => "Ruby"}, + {"1" => nil}, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + {"0" => Date.new(1960, 1, 1)}, + {"1" => nil}, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}, + {"1" => nil}, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + {"0" => Time.parse("1960-01-01T02:09:30Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123456Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + # 00:10:00 + {"0" => Arrow::Time.new(unit, 60 * 10)}, + {"1" => nil}, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + # 00:10:00.123 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}, + {"1" => nil}, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + # 00:10:00.123456 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}, + {"1" => nil}, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + # 00:10:00.123456789 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}, + {"1" => nil}, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + {"0" => BigDecimal("92.92")}, + {"1" => nil}, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + {"0" => BigDecimal("92.92")}, + {"1" => nil}, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_list + values = [ + {"0" => [true, nil, false]}, + {"1" => nil}, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + {"0" => {"sub_field" => true}}, + {"1" => nil}, + {"0" => {"sub_field" => nil}}, + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + {"0" => {"key1" => true, "key2" => nil}}, + {"1" => nil}, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + values = [ + {"0" => {"field1" => true}}, + {"1" => nil}, + {"0" => {"field2" => nil}}, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + values = [ + {"0" => {"field1" => true}}, + {"1" => nil}, + {"0" => {"field2" => nil}}, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + values = [ + {"0" => "Ruby"}, + {"1" => nil}, + {"0" => "GLib"}, + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayDenseUnionArrayTest < Test::Unit::TestCase + include ValuesDenseUnionArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArrayDenseUnionArrayTest < Test::Unit::TestCase + include ValuesDenseUnionArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-list-array.rb b/src/arrow/ruby/red-arrow/test/values/test-list-array.rb new file mode 100644 index 000000000..d2905b36b --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-list-array.rb @@ -0,0 +1,532 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesListArrayTests + def build_data_type(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::ListDataType.new(field: field_description) + end + + def build_array(type, values) + Arrow::ListArray.new(build_data_type(type), values) + end + + def test_null + values = [ + [nil, nil, nil], + nil, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + [true, nil, false], + nil, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + [-(2 ** 7), nil, (2 ** 7) - 1], + nil, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + [0, nil, (2 ** 8) - 1], + nil, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + [-(2 ** 15), nil, (2 ** 15) - 1], + nil, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + [0, nil, (2 ** 16) - 1], + nil, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + [-(2 ** 31), nil, (2 ** 31) - 1], + nil, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + [0, nil, (2 ** 32) - 1], + nil, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + [-(2 ** 63), nil, (2 ** 63) - 1], + nil, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + [0, nil, (2 ** 64) - 1], + nil, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + [-1.0, nil, 1.0], + nil, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + [-1.0, nil, 1.0], + nil, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + ["\x00".b, nil, "\xff".b], + nil, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + ], + nil, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + ], + nil, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + ], + nil, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + ], + nil, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + ], + nil, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + ], + nil, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + ], + nil, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + nil, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + nil, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + [ + {"field" => true}, + nil, + {"field" => nil}, + ], + nil, + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + [ + {"key1" => true, "key2" => nil}, + nil, + ], + nil, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def test_sparse + omit("Need to add support for SparseUnionArrayBuilder") + values = [ + [ + {"field1" => true}, + nil, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dense + omit("Need to add support for DenseUnionArrayBuilder") + values = [ + [ + {"field1" => true}, + nil, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + values = [ + [ + "Ruby", + nil, + "GLib", + ], + nil, + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayListArrayTest < Test::Unit::TestCase + include ValuesListArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArrayListArrayTest < Test::Unit::TestCase + include ValuesListArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-map-array.rb b/src/arrow/ruby/red-arrow/test/values/test-map-array.rb new file mode 100644 index 000000000..14b5bf6c3 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-map-array.rb @@ -0,0 +1,433 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesMapArrayTests + def build_data_type(item_type) + Arrow::MapDataType.new( + key: :string, + item: item_type + ) + end + + def build_array(item_type, values) + Arrow::MapArray.new(build_data_type(item_type), values) + end + + def test_null + values = [ + {"key1" => nil}, + nil, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + {"key1" => false, "key2" => nil}, + nil, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + {"key1" => (2 ** 7) - 1, "key2" => nil}, + nil, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + {"key1" => (2 ** 8) - 1, "key2" => nil}, + nil, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + {"key1" => (2 ** 16) - 1, "key2" => nil}, + nil, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + {"key1" => -(2 ** 31), "key2" => nil}, + nil, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + {"key1" => (2 ** 32) - 1, "key2" => nil}, + nil, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + {"key1" => -(2 ** 63), "key2" => nil}, + nil, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + {"key1" => (2 ** 64) - 1, "key2" => nil}, + nil, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + {"key1" => -1.0, "key2" => nil}, + nil, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + {"key1" => -1.0, "key2" => nil}, + nil, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + {"key1" => "\xff".b, "key2" => nil}, + nil, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + {"key1" => "Ruby", "key2" => nil}, + nil, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + {"key1" => Date.new(1960, 1, 1), "key2" => nil}, + nil, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + {"key1" => DateTime.new(1960, 1, 1, 2, 9, 30), "key2" => nil}, + nil, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + {"key1" => Time.parse("1960-01-01T02:09:30Z"), "key2" => nil}, + nil, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + {"key1" => Time.parse("1960-01-01T02:09:30.123Z"), "key2" => nil}, + nil, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + {"key1" => Time.parse("1960-01-01T02:09:30.123456Z"), "key2" => nil}, + nil, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + {"key1" => Time.parse("1960-01-01T02:09:30.123456789Z"), "key2" => nil}, + nil, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + # 00:10:00 + {"key1" => Arrow::Time.new(unit, 60 * 10), "key2" => nil}, + nil, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + # 00:10:00.123 + {"key1" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123), "key2" => nil}, + nil, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + # 00:10:00.123456 + {"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), "key2" => nil}, + nil, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + # 00:10:00.123456789 + {"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), "key2" => nil}, + nil, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + {"key1" => BigDecimal("92.92"), "key2" => nil}, + nil, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + {"key1" => BigDecimal("92.92"), "key2" => nil}, + nil, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_list + values = [ + {"key1" => [true, nil, false], "key2" => nil}, + nil, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + {"key1" => {"field" => true}, "key2" => nil, "key3" => {"field" => nil}}, + nil, + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + {"key1" => {"sub_key1" => true, "sub_key2" => nil}, "key2" => nil}, + nil, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + values = [ + {"key1" => {"field1" => true}, "key2" => nil, "key3" => {"field2" => nil}}, + nil, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + values = [ + {"key1" => {"field1" => true}, "key2" => nil, "key3" => {"field2" => nil}}, + nil, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + values = [ + {"key1" => "Ruby", "key2" => nil, "key3" => "GLib"}, + nil, + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayMapArrayTest < Test::Unit::TestCase + include ValuesMapArrayTests + + def build(item_type, values) + build_array(item_type, values) + end +end + +class ValuesChunkedArrayMapArrayTest < Test::Unit::TestCase + include ValuesMapArrayTests + + def build(item_type, values) + Arrow::ChunkedArray.new([build_array(item_type, values)]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-sparse-union-array.rb b/src/arrow/ruby/red-arrow/test/values/test-sparse-union-array.rb new file mode 100644 index 000000000..909d67e61 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-sparse-union-array.rb @@ -0,0 +1,473 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesSparseUnionArrayTests + def build_data_type(type, type_codes) + field_description = {} + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::SparseUnionDataType.new(fields: [ + field_description.merge(name: "0"), + field_description.merge(name: "1"), + ], + type_codes: type_codes) + end + + def build_array(type, values) + type_codes = [0, 1] + data_type = build_data_type(type, type_codes) + type_ids = [] + arrays = data_type.fields.collect do |field| + sub_schema = Arrow::Schema.new([field]) + sub_records = values.collect do |value| + [value.nil? ? nil : value[field.name]] + end + sub_record_batch = Arrow::RecordBatch.new(sub_schema, + sub_records) + sub_record_batch.columns[0].data + end + values.each do |value| + if value.key?("0") + type_ids << type_codes[0] + elsif value.key?("1") + type_ids << type_codes[1] + end + end + Arrow::SparseUnionArray.new(data_type, + Arrow::Int8Array.new(type_ids), + arrays) + end + + def test_null + values = [ + {"0" => nil}, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + {"0" => true}, + {"1" => nil}, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + {"0" => -(2 ** 7)}, + {"1" => nil}, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + {"0" => (2 ** 8) - 1}, + {"1" => nil}, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + {"0" => -(2 ** 15)}, + {"1" => nil}, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + {"0" => (2 ** 16) - 1}, + {"1" => nil}, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + {"0" => -(2 ** 31)}, + {"1" => nil}, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + {"0" => (2 ** 32) - 1}, + {"1" => nil}, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + {"0" => -(2 ** 63)}, + {"1" => nil}, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + {"0" => (2 ** 64) - 1}, + {"1" => nil}, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + {"0" => -1.0}, + {"1" => nil}, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + {"0" => -1.0}, + {"1" => nil}, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + {"0" => "\xff".b}, + {"1" => nil}, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + {"0" => "Ruby"}, + {"1" => nil}, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + {"0" => Date.new(1960, 1, 1)}, + {"1" => nil}, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}, + {"1" => nil}, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + {"0" => Time.parse("1960-01-01T02:09:30Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123456Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}, + {"1" => nil}, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + # 00:10:00 + {"0" => Arrow::Time.new(unit, 60 * 10)}, + {"1" => nil}, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + # 00:10:00.123 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}, + {"1" => nil}, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + # 00:10:00.123456 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}, + {"1" => nil}, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + # 00:10:00.123456789 + {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}, + {"1" => nil}, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + {"0" => BigDecimal("92.92")}, + {"1" => nil}, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + {"0" => BigDecimal("92.92")}, + {"1" => nil}, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_list + values = [ + {"0" => [true, nil, false]}, + {"1" => nil}, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + {"0" => {"sub_field" => true}}, + {"1" => nil}, + {"0" => {"sub_field" => nil}}, + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + {"0" => {"key1" => true, "key2" => nil}}, + {"1" => nil}, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + values = [ + {"0" => {"field1" => true}}, + {"1" => nil}, + {"0" => {"field2" => nil}}, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + values = [ + {"0" => {"field1" => true}}, + {"1" => nil}, + {"0" => {"field2" => nil}}, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + values = [ + {"0" => "Ruby"}, + {"1" => nil}, + {"0" => "GLib"}, + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArraySparseUnionArrayTest < Test::Unit::TestCase + include ValuesSparseUnionArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArraySparseUnionArrayTest < Test::Unit::TestCase + include ValuesSparseUnionArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end diff --git a/src/arrow/ruby/red-arrow/test/values/test-struct-array.rb b/src/arrow/ruby/red-arrow/test/values/test-struct-array.rb new file mode 100644 index 000000000..4e3396796 --- /dev/null +++ b/src/arrow/ruby/red-arrow/test/values/test-struct-array.rb @@ -0,0 +1,482 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesStructArrayTests + def build_data_type(type) + field_description = { + name: :field, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::StructDataType.new([field_description]) + end + + def build_array(type, values) + Arrow::StructArray.new(build_data_type(type), values) + end + + def test_null + values = [ + {"field" => nil}, + nil, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + {"field" => true}, + nil, + {"field" => nil}, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + {"field" => -(2 ** 7)}, + nil, + {"field" => nil}, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + {"field" => (2 ** 8) - 1}, + nil, + {"field" => nil}, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + {"field" => -(2 ** 15)}, + nil, + {"field" => nil}, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + {"field" => (2 ** 16) - 1}, + nil, + {"field" => nil}, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + {"field" => -(2 ** 31)}, + nil, + {"field" => nil}, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + {"field" => (2 ** 32) - 1}, + nil, + {"field" => nil}, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + {"field" => -(2 ** 63)}, + nil, + {"field" => nil}, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + {"field" => (2 ** 64) - 1}, + nil, + {"field" => nil}, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + {"field" => -1.0}, + nil, + {"field" => nil}, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + {"field" => -1.0}, + nil, + {"field" => nil}, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + {"field" => "\xff".b}, + nil, + {"field" => nil}, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + {"field" => "Ruby"}, + nil, + {"field" => nil}, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + {"field" => Date.new(1960, 1, 1)}, + nil, + {"field" => nil}, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + {"field" => DateTime.new(1960, 1, 1, 2, 9, 30)}, + nil, + {"field" => nil}, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + {"field" => Time.parse("1960-01-01T02:09:30Z")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + {"field" => Time.parse("1960-01-01T02:09:30.123Z")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + {"field" => Time.parse("1960-01-01T02:09:30.123456Z")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + {"field" => Time.parse("1960-01-01T02:09:30.123456789Z")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + # 00:10:00 + {"field" => Arrow::Time.new(unit, 60 * 10)}, + nil, + {"field" => nil}, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + # 00:10:00.123 + {"field" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}, + nil, + {"field" => nil}, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + # 00:10:00.123456 + {"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}, + nil, + {"field" => nil}, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + # 00:10:00.123456789 + {"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}, + nil, + {"field" => nil}, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + {"field" => BigDecimal("92.92")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + {"field" => BigDecimal("92.92")}, + nil, + {"field" => nil}, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_list + values = [ + {"field" => [true, nil, false]}, + nil, + {"field" => nil}, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + {"field" => {"sub_field" => true}}, + nil, + {"field" => nil}, + {"field" => {"sub_field" => nil}}, + ] + target = build({ + type: :struct, + fields: [ + { + name: :sub_field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + {"field" => {"key1" => true, "key2" => nil}}, + nil, + {"field" => nil}, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def test_sparse_union + omit("Need to add support for SparseUnionArrayBuilder") + values = [ + {"field" => {"field1" => true}}, + nil, + {"field" => nil}, + {"field" => {"field2" => nil}}, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dense_union + omit("Need to add support for DenseUnionArrayBuilder") + values = [ + {"field" => {"field1" => true}}, + nil, + {"field" => nil}, + {"field" => {"field2" => nil}}, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(values, target.values) + end + + def test_dictionary + omit("Need to add support for DictionaryArrayBuilder") + values = [ + {"field" => "Ruby"}, + nil, + {"field" => nil}, + {"field" => "GLib"}, + ] + dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) + target = build({ + type: :dictionary, + index_data_type: :int8, + dictionary: dictionary, + ordered: true, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayStructArrayTest < Test::Unit::TestCase + include ValuesStructArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArrayStructArrayTest < Test::Unit::TestCase + include ValuesStructArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end |