Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 653 insertions, 0 deletions

diff --git a/src/arrow/c_glib/test/helper/buildable.rb b/src/arrow/c_glib/test/helper/buildable.rb
new file mode 100644
index 000000000..3a1240cfa
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/buildable.rb
@@ -0,0 +1,263 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Buildable
+    def build_schema(fields)
+      fields = fields.collect do |name, data_type|
+        Arrow::Field.new(name, data_type)
+      end
+      Arrow::Schema.new(fields)
+    end
+
+    def build_null_array(values)
+      build_array(Arrow::NullArrayBuilder.new, values)
+    end
+
+    def build_boolean_array(values)
+      build_array(Arrow::BooleanArrayBuilder.new, values)
+    end
+
+    def build_int_array(values)
+      build_array(Arrow::IntArrayBuilder.new, values)
+    end
+
+    def build_uint_array(values)
+      build_array(Arrow::UIntArrayBuilder.new, values)
+    end
+
+    def build_int8_array(values)
+      build_array(Arrow::Int8ArrayBuilder.new, values)
+    end
+
+    def build_uint8_array(values)
+      build_array(Arrow::UInt8ArrayBuilder.new, values)
+    end
+
+    def build_int16_array(values)
+      build_array(Arrow::Int16ArrayBuilder.new, values)
+    end
+
+    def build_uint16_array(values)
+      build_array(Arrow::UInt16ArrayBuilder.new, values)
+    end
+
+    def build_int32_array(values)
+      build_array(Arrow::Int32ArrayBuilder.new, values)
+    end
+
+    def build_uint32_array(values)
+      build_array(Arrow::UInt32ArrayBuilder.new, values)
+    end
+
+    def build_int64_array(values)
+      build_array(Arrow::Int64ArrayBuilder.new, values)
+    end
+
+    def build_uint64_array(values)
+      build_array(Arrow::UInt64ArrayBuilder.new, values)
+    end
+
+    def build_float_array(values)
+      build_array(Arrow::FloatArrayBuilder.new, values)
+    end
+
+    def build_double_array(values)
+      build_array(Arrow::DoubleArrayBuilder.new, values)
+    end
+
+    def build_date32_array(values)
+      build_array(Arrow::Date32ArrayBuilder.new, values)
+    end
+
+    def build_date64_array(values)
+      build_array(Arrow::Date64ArrayBuilder.new, values)
+    end
+
+    def build_timestamp_array(unit, values)
+      data_type = Arrow::TimestampDataType.new(unit)
+      build_array(Arrow::TimestampArrayBuilder.new(data_type),
+                  values)
+    end
+
+    def build_time32_array(unit, values)
+      build_array(Arrow::Time32ArrayBuilder.new(Arrow::Time32DataType.new(unit)),
+                  values)
+    end
+
+    def build_time64_array(unit, values)
+      build_array(Arrow::Time64ArrayBuilder.new(Arrow::Time64DataType.new(unit)),
+                  values)
+    end
+
+    def build_binary_array(values)
+      build_array(Arrow::BinaryArrayBuilder.new, values)
+    end
+
+    def build_large_binary_array(values)
+      build_array(Arrow::LargeBinaryArrayBuilder.new, values)
+    end
+
+    def build_fixed_size_binary_array(data_type, values)
+      build_array(Arrow::FixedSizeBinaryArrayBuilder.new(data_type),
+                  values)
+    end
+
+    def build_string_array(values)
+      build_array(Arrow::StringArrayBuilder.new, values)
+    end
+
+    def build_large_string_array(values)
+      build_array(Arrow::LargeStringArrayBuilder.new, values)
+    end
+
+    def build_decimal128_array(value_data_type, values)
+      values = values.collect do |value|
+        case value
+        when String
+          Arrow::Decimal128.new(value)
+        else
+          value
+        end
+      end
+      build_array(Arrow::Decimal128ArrayBuilder.new(value_data_type),
+                  values)
+    end
+
+    def build_list_array(value_data_type, values_list, field_name: "value")
+      value_field = Arrow::Field.new(field_name, value_data_type)
+      data_type = Arrow::ListDataType.new(value_field)
+      builder = Arrow::ListArrayBuilder.new(data_type)
+      values_list.each do |values|
+        append_to_builder(builder, values)
+      end
+      builder.finish
+    end
+
+    def build_large_list_array(value_data_type, values_list, field_name: "value")
+      value_field = Arrow::Field.new(field_name, value_data_type)
+      data_type = Arrow::LargeListDataType.new(value_field)
+      builder = Arrow::LargeListArrayBuilder.new(data_type)
+      values_list.each do |values|
+        append_to_builder(builder, values)
+      end
+      builder.finish
+    end
+
+    def build_map_array(key_data_type, item_data_type, maps)
+      data_type = Arrow::MapDataType.new(key_data_type, item_data_type)
+      builder = Arrow::MapArrayBuilder.new(data_type)
+      maps.each do |map|
+        append_to_builder(builder, map)
+      end
+      builder.finish
+    end
+
+    def build_struct_array(fields, structs)
+      data_type = Arrow::StructDataType.new(fields)
+      builder = Arrow::StructArrayBuilder.new(data_type)
+      structs.each do |struct|
+        append_to_builder(builder, struct)
+      end
+      builder.finish
+    end
+
+    def append_to_builder(builder, value)
+      if value.nil?
+        builder.append_null
+      else
+        data_type = builder.value_data_type
+        case data_type
+        when Arrow::MapDataType
+          builder.append_value
+          key_builder = builder.key_builder
+          item_builder = builder.item_builder
+          value.each do |k, v|
+            append_to_builder(key_builder, k)
+            append_to_builder(item_builder, v)
+          end
+        when Arrow::ListDataType, Arrow::LargeListDataType
+          builder.append_value
+          value_builder = builder.value_builder
+          value.each do |v|
+            append_to_builder(value_builder, v)
+          end
+        when Arrow::StructDataType
+          builder.append_value
+          value.each do |name, v|
+            field_index = data_type.get_field_index(name)
+            field_builder = builder.get_field_builder(field_index)
+            append_to_builder(field_builder, v)
+          end
+        else
+          builder.append_value(value)
+        end
+      end
+    end
+
+    def build_table(columns)
+      fields = []
+      chunked_arrays = []
+      columns.each do |name, data|
+        case data
+        when Arrow::Array
+          chunked_array = Arrow::ChunkedArray.new([data])
+        when Array
+          chunked_array = Arrow::ChunkedArray.new(data)
+        else
+          chunked_array = data
+        end
+        fields << Arrow::Field.new(name, chunked_array.value_data_type)
+        chunked_arrays << chunked_array
+      end
+      schema = Arrow::Schema.new(fields)
+      Arrow::Table.new(schema, chunked_arrays)
+    end
+
+    def build_record_batch(columns)
+      n_rows = columns.collect {|_, array| array.length}.min || 0
+      fields = columns.collect do |name, array|
+        Arrow::Field.new(name, array.value_data_type)
+      end
+      schema = Arrow::Schema.new(fields)
+      Arrow::RecordBatch.new(schema, n_rows, columns.values)
+    end
+
+    def build_file_uri(path)
+      absolute_path = File.expand_path(path)
+      if absolute_path.start_with?("/")
+        "file://#{absolute_path}"
+      else
+        "file:///#{absolute_path}"
+      end
+    end
+
+    private
+    def build_array(builder, values)
+      values.each do |value|
+        if value.nil?
+          builder.append_null
+        elsif builder.respond_to?(:append_string)
+          builder.append_string(value)
+        else
+          builder.append_value(value)
+        end
+      end
+      builder.finish
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/data-type.rb b/src/arrow/c_glib/test/helper/data-type.rb
new file mode 100644
index 000000000..bbe6866f5
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/data-type.rb
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module DataType
+    def boolean_data_type
+      Arrow::BooleanDataType.new
+    end
+
+    def int8_data_type
+      Arrow::Int8DataType.new
+    end
+
+    def int16_data_type
+      Arrow::Int16DataType.new
+    end
+
+    def int32_data_type
+      Arrow::Int32DataType.new
+    end
+
+    def int64_data_type
+      Arrow::Int64DataType.new
+    end
+
+    def uint8_data_type
+      Arrow::UInt8DataType.new
+    end
+
+    def uint16_data_type
+      Arrow::UInt16DataType.new
+    end
+
+    def uint32_data_type
+      Arrow::UInt32DataType.new
+    end
+
+    def uint64_data_type
+      Arrow::UInt64DataType.new
+    end
+
+    def string_data_type
+      Arrow::StringDataType.new
+    end
+
+    def date64_data_type
+      Arrow::Date64DataType.new
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/fixture.rb b/src/arrow/c_glib/test/helper/fixture.rb
new file mode 100644
index 000000000..f07afd0e4
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/fixture.rb
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Fixture
+    def fixture_path(*components)
+      File.join(__dir__, "..", "fixture", *components)
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/flight-info-generator.rb b/src/arrow/c_glib/test/helper/flight-info-generator.rb
new file mode 100644
index 000000000..c57530879
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/flight-info-generator.rb
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative "buildable"
+require_relative "data-type"
+require_relative "writable"
+
+module Helper
+  class FlightInfoGenerator
+    include Buildable
+    include DataType
+    include Writable
+
+    def page_view_table
+      build_table("count" => build_uint64_array([1, 2, 3]),
+                  "private" => build_boolean_array([true, false, true]))
+    end
+
+    def page_view_descriptor
+      ArrowFlight::PathDescriptor.new(["page-view"])
+    end
+
+    def page_view_ticket
+      ArrowFlight::Ticket.new("page-view")
+    end
+
+    def page_view_endpoints
+      locations = [
+        ArrowFlight::Location.new("grpc+tcp://127.0.0.1:10000"),
+        ArrowFlight::Location.new("grpc+tcp://127.0.0.1:10001"),
+      ]
+      [
+        ArrowFlight::Endpoint.new(page_view_ticket, locations),
+      ]
+    end
+
+    def page_view
+      table = page_view_table
+      descriptor = page_view_descriptor
+      endpoints = page_view_endpoints
+      output = Arrow::ResizableBuffer.new(0)
+      write_table(table, output, type: :stream)
+      ArrowFlight::Info.new(table.schema,
+                            descriptor,
+                            endpoints,
+                            table.n_rows,
+                            output.size)
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/flight-server.rb b/src/arrow/c_glib/test/helper/flight-server.rb
new file mode 100644
index 000000000..89fd13b42
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/flight-server.rb
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative "flight-info-generator"
+
+module Helper
+  class FlightServer < ArrowFlight::Server
+    type_register
+
+    private
+    def virtual_do_list_flights(context, criteria)
+      generator = FlightInfoGenerator.new
+      [generator.page_view]
+    end
+
+    def virtual_do_do_get(context, ticket)
+      generator = FlightInfoGenerator.new
+      unless ticket == generator.page_view_ticket
+        raise Arrow::Error::Invalid.new("invalid ticket")
+      end
+      table = generator.page_view_table
+      reader = Arrow::TableBatchReader.new(table)
+      ArrowFlight::RecordBatchStream.new(reader)
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/omittable.rb b/src/arrow/c_glib/test/helper/omittable.rb
new file mode 100644
index 000000000..a1c0334b6
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/omittable.rb
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Omittable
+    def require_gi_bindings(major, minor, micro)
+      return if GLib.check_binding_version?(major, minor, micro)
+      message =
+        "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " +
+        GLib::BINDING_VERSION.join(".")
+      omit(message)
+    end
+
+    def require_gi(major, minor, micro)
+      return if GObjectIntrospection::Version.or_later?(major, minor, micro)
+      message =
+        "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " +
+        GObjectIntrospection::Version::STRING
+      omit(message)
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/plasma-store.rb b/src/arrow/c_glib/test/helper/plasma-store.rb
new file mode 100644
index 000000000..dcf1f47ae
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/plasma-store.rb
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  class PlasmaStore
+    def initialize(options={})
+      @path = `pkg-config --variable=plasma_store_server plasma`.chomp
+      @memory_size = options[:memory_size] || 1024 * 1024
+      @socket_file = Tempfile.new(["plasma-store", ".sock"])
+      @socket_file.close
+      @pid = nil
+      FileUtils.rm_f(socket_path)
+    end
+
+    def socket_path
+      @socket_file.path
+    end
+
+    def start
+      @pid = spawn(@path,
+                   "-m", @memory_size.to_s,
+                   "-s", socket_path)
+      until File.exist?(socket_path)
+        if Process.waitpid(@pid, Process::WNOHANG)
+          raise "Failed to run plasma-store-server: #{@path}"
+        end
+      end
+    end
+
+    def stop
+      return if @pid.nil?
+      Process.kill(:TERM, @pid)
+      timeout = 1
+      limit = Time.now + timeout
+      while Time.now < limit
+        return if Process.waitpid(@pid, Process::WNOHANG)
+        sleep(0.1)
+      end
+      Process.kill(:KILL, @pid)
+      Process.waitpid(@pid)
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/readable.rb b/src/arrow/c_glib/test/helper/readable.rb
new file mode 100644
index 000000000..81bf0795c
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/readable.rb
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Readable
+    def read_table(input, type: :file)
+      if input.is_a?(Arrow::Buffer)
+        input_stream = Arrow::BufferIntputStream.new(input)
+      else
+        input_stream = Arrow::FileInputStream.new(input)
+      end
+      begin
+        if type == :file
+          reader = Arrow::RecordBatchFileReader.new(input_stream)
+          record_batches = []
+          reader.n_record_batches.times do |i|
+            record_batches << reader.read_record_batch(i)
+          end
+          yield(Arrow::Table.new(record_batches[0].schema, record_batches))
+        else
+          reader = Arrow::RecordBatchStreamReader.new(input_stream)
+          begin
+            yield(reader.read_all)
+          ensure
+            reader.close
+          end
+        end
+      ensure
+        input_stream.close
+      end
+    end
+  end
+end
diff --git a/src/arrow/c_glib/test/helper/writable.rb b/src/arrow/c_glib/test/helper/writable.rb
new file mode 100644
index 000000000..1c8db756c
--- /dev/null
+++ b/src/arrow/c_glib/test/helper/writable.rb
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Writable
+    def write_table(table, output, type: :file)
+      if output.is_a?(Arrow::Buffer)
+        output_stream = Arrow::BufferOutputStream.new(output)
+      else
+        output_stream = Arrow::FileOutputStream.new(output, false)
+      end
+      begin
+        if type == :file
+          writer_class = Arrow::RecordBatchFileWriter
+        else
+          writer_class = Arrow::RecordBatchStreamWriter
+        end
+        writer = writer_class.new(output_stream, table.schema)
+        begin
+          writer.write_table(table)
+        ensure
+          writer.close
+        end
+      ensure
+        output_stream.close
+      end
+    end
+  end
+end