summaryrefslogtreecommitdiffstats
path: root/src/arrow/dev/archery
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
commite6918187568dbd01842d8d1d2c808ce16a894239 (patch)
tree64f88b554b444a49f656b6c656111a145cbbaa28 /src/arrow/dev/archery
parentInitial commit. (diff)
downloadceph-upstream/18.2.2.tar.xz
ceph-upstream/18.2.2.zip
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/arrow/dev/archery')
-rw-r--r--src/arrow/dev/archery/MANIFEST.in4
-rw-r--r--src/arrow/dev/archery/README.md49
-rw-r--r--src/arrow/dev/archery/archery/__init__.py16
-rw-r--r--src/arrow/dev/archery/archery/benchmark/__init__.py16
-rw-r--r--src/arrow/dev/archery/archery/benchmark/codec.py97
-rw-r--r--src/arrow/dev/archery/archery/benchmark/compare.py173
-rw-r--r--src/arrow/dev/archery/archery/benchmark/core.py57
-rw-r--r--src/arrow/dev/archery/archery/benchmark/google.py174
-rw-r--r--src/arrow/dev/archery/archery/benchmark/jmh.py201
-rw-r--r--src/arrow/dev/archery/archery/benchmark/runner.py313
-rw-r--r--src/arrow/dev/archery/archery/bot.py267
-rw-r--r--src/arrow/dev/archery/archery/cli.py943
-rw-r--r--src/arrow/dev/archery/archery/compat.py59
-rw-r--r--src/arrow/dev/archery/archery/crossbow/__init__.py19
-rw-r--r--src/arrow/dev/archery/archery/crossbow/cli.py365
-rw-r--r--src/arrow/dev/archery/archery/crossbow/core.py1172
-rw-r--r--src/arrow/dev/archery/archery/crossbow/reports.py315
-rw-r--r--src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml51
-rw-r--r--src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md10
-rw-r--r--src/arrow/dev/archery/archery/crossbow/tests/test_core.py25
-rw-r--r--src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py43
-rw-r--r--src/arrow/dev/archery/archery/crossbow/tests/test_reports.py35
-rw-r--r--src/arrow/dev/archery/archery/docker.py402
-rw-r--r--src/arrow/dev/archery/archery/docker/__init__.py18
-rw-r--r--src/arrow/dev/archery/archery/docker/cli.py261
-rw-r--r--src/arrow/dev/archery/archery/docker/core.py417
-rw-r--r--src/arrow/dev/archery/archery/docker/tests/test_docker.py531
-rw-r--r--src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py201
-rw-r--r--src/arrow/dev/archery/archery/integration/__init__.py16
-rw-r--r--src/arrow/dev/archery/archery/integration/datagen.py1662
-rw-r--r--src/arrow/dev/archery/archery/integration/runner.py429
-rw-r--r--src/arrow/dev/archery/archery/integration/scenario.py29
-rw-r--r--src/arrow/dev/archery/archery/integration/tester.py62
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_cpp.py116
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_csharp.py67
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_go.py119
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_java.py140
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_js.py73
-rw-r--r--src/arrow/dev/archery/archery/integration/tester_rust.py115
-rw-r--r--src/arrow/dev/archery/archery/integration/util.py166
-rw-r--r--src/arrow/dev/archery/archery/lang/__init__.py16
-rw-r--r--src/arrow/dev/archery/archery/lang/cpp.py296
-rw-r--r--src/arrow/dev/archery/archery/lang/java.py77
-rw-r--r--src/arrow/dev/archery/archery/lang/python.py223
-rw-r--r--src/arrow/dev/archery/archery/linking.py75
-rw-r--r--src/arrow/dev/archery/archery/release.py535
-rw-r--r--src/arrow/dev/archery/archery/templates/release_changelog.md.j229
-rw-r--r--src/arrow/dev/archery/archery/templates/release_curation.txt.j241
-rw-r--r--src/arrow/dev/archery/archery/testing.py83
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl6
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl4
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json212
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json212
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json212
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json212
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json217
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json206
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json445
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/issue-19.json64
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/issue-26.json70
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json31
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json31
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json158
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json170
-rw-r--r--src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json329
-rw-r--r--src/arrow/dev/archery/archery/tests/test_benchmarks.py383
-rw-r--r--src/arrow/dev/archery/archery/tests/test_bot.py215
-rw-r--r--src/arrow/dev/archery/archery/tests/test_cli.py39
-rw-r--r--src/arrow/dev/archery/archery/tests/test_release.py333
-rw-r--r--src/arrow/dev/archery/archery/tests/test_testing.py62
-rw-r--r--src/arrow/dev/archery/archery/utils/__init__.py16
-rw-r--r--src/arrow/dev/archery/archery/utils/cache.py80
-rw-r--r--src/arrow/dev/archery/archery/utils/cli.py73
-rw-r--r--src/arrow/dev/archery/archery/utils/cmake.py215
-rw-r--r--src/arrow/dev/archery/archery/utils/command.py100
-rw-r--r--src/arrow/dev/archery/archery/utils/git.py100
-rw-r--r--src/arrow/dev/archery/archery/utils/lint.py429
-rw-r--r--src/arrow/dev/archery/archery/utils/logger.py29
-rw-r--r--src/arrow/dev/archery/archery/utils/maven.py204
-rw-r--r--src/arrow/dev/archery/archery/utils/rat.py70
-rw-r--r--src/arrow/dev/archery/archery/utils/report.py64
-rw-r--r--src/arrow/dev/archery/archery/utils/source.py211
-rw-r--r--src/arrow/dev/archery/archery/utils/tmpdir.py28
-rw-r--r--src/arrow/dev/archery/conftest.py70
-rwxr-xr-xsrc/arrow/dev/archery/generate_files_for_endian_test.sh43
-rw-r--r--src/arrow/dev/archery/requirements.txt4
-rwxr-xr-xsrc/arrow/dev/archery/setup.py59
87 files changed, 16009 insertions, 0 deletions
diff --git a/src/arrow/dev/archery/MANIFEST.in b/src/arrow/dev/archery/MANIFEST.in
new file mode 100644
index 000000000..90fe034c2
--- /dev/null
+++ b/src/arrow/dev/archery/MANIFEST.in
@@ -0,0 +1,4 @@
+include ../../LICENSE.txt
+include ../../NOTICE.txt
+
+include archery/reports/*
diff --git a/src/arrow/dev/archery/README.md b/src/arrow/dev/archery/README.md
new file mode 100644
index 000000000..eff654416
--- /dev/null
+++ b/src/arrow/dev/archery/README.md
@@ -0,0 +1,49 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+# Developing with Archery
+
+Archery is documented on the Arrow website:
+
+* [Daily development using Archery](https://arrow.apache.org/docs/developers/archery.html)
+* [Using Archery and Crossbow](https://arrow.apache.org/docs/developers/crossbow.html)
+* [Using Archer and Docker](https://arrow.apache.org/docs/developers/docker.html)
+
+# Installing Archery
+
+See the pages linked aboved for more details. As a general overview, Archery
+comes in a number of subpackages, each needing to be installed if you want
+to use the functionality of it:
+
+* lint – lint (and in some cases auto-format) code in the Arrow repo
+ To install: `pip install -e "arrow/dev/archery[lint]"`
+* benchmark – to run Arrow benchmarks using Archery
+ To install: `pip install -e "arrow/dev/archery[benchmark]"`
+* docker – to run docker-compose based tasks more easily
+ To install: `pip install -e "arrow/dev/archery[docker]"`
+* release – release related helpers
+ To install: `pip install -e "arrow/dev/archery[release]"`
+* crossbow – to trigger + interact with the crossbow build system
+ To install: `pip install -e "arrow/dev/archery[crossbow]"`
+* crossbow-upload
+ To install: `pip install -e "arrow/dev/archery[crossbow-upload]"`
+
+Additionally, if you would prefer to install everything at once,
+`pip install -e "arrow/dev/archery[all]"` is an alias for all of
+the above subpackages. \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/__init__.py b/src/arrow/dev/archery/archery/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/arrow/dev/archery/archery/benchmark/__init__.py b/src/arrow/dev/archery/archery/benchmark/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/arrow/dev/archery/archery/benchmark/codec.py b/src/arrow/dev/archery/archery/benchmark/codec.py
new file mode 100644
index 000000000..4157890d1
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/codec.py
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import json
+
+from ..benchmark.core import Benchmark, BenchmarkSuite
+from ..benchmark.runner import BenchmarkRunner, StaticBenchmarkRunner
+from ..benchmark.compare import BenchmarkComparator
+
+
+class JsonEncoder(json.JSONEncoder):
+ def default(self, o):
+ if isinstance(o, Benchmark):
+ return BenchmarkCodec.encode(o)
+
+ if isinstance(o, BenchmarkSuite):
+ return BenchmarkSuiteCodec.encode(o)
+
+ if isinstance(o, BenchmarkRunner):
+ return BenchmarkRunnerCodec.encode(o)
+
+ if isinstance(o, BenchmarkComparator):
+ return BenchmarkComparatorCodec.encode(o)
+
+ return json.JSONEncoder.default(self, o)
+
+
+class BenchmarkCodec:
+ @staticmethod
+ def encode(b):
+ return {
+ "name": b.name,
+ "unit": b.unit,
+ "less_is_better": b.less_is_better,
+ "values": b.values,
+ "time_unit": b.time_unit,
+ "times": b.times,
+ "counters": b.counters,
+ }
+
+ @staticmethod
+ def decode(dct, **kwargs):
+ return Benchmark(**dct, **kwargs)
+
+
+class BenchmarkSuiteCodec:
+ @staticmethod
+ def encode(bs):
+ return {
+ "name": bs.name,
+ "benchmarks": [BenchmarkCodec.encode(b) for b in bs.benchmarks]
+ }
+
+ @staticmethod
+ def decode(dct, **kwargs):
+ benchmarks = [BenchmarkCodec.decode(b)
+ for b in dct.pop("benchmarks", [])]
+ return BenchmarkSuite(benchmarks=benchmarks, **dct, **kwargs)
+
+
+class BenchmarkRunnerCodec:
+ @staticmethod
+ def encode(br):
+ return {"suites": [BenchmarkSuiteCodec.encode(s) for s in br.suites]}
+
+ @staticmethod
+ def decode(dct, **kwargs):
+ suites = [BenchmarkSuiteCodec.decode(s)
+ for s in dct.pop("suites", [])]
+ return StaticBenchmarkRunner(suites=suites, **dct, **kwargs)
+
+
+class BenchmarkComparatorCodec:
+ @staticmethod
+ def encode(bc):
+ comparator = bc.formatted
+
+ suite_name = bc.suite_name
+ if suite_name:
+ comparator["suite"] = suite_name
+
+ return comparator
diff --git a/src/arrow/dev/archery/archery/benchmark/compare.py b/src/arrow/dev/archery/archery/benchmark/compare.py
new file mode 100644
index 000000000..622b80179
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/compare.py
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# Define a global regression threshold as 5%. This is purely subjective and
+# flawed. This does not track cumulative regression.
+DEFAULT_THRESHOLD = 0.05
+
+
+def items_per_seconds_fmt(value):
+ if value < 1000:
+ return "{} items/sec".format(value)
+ if value < 1000**2:
+ return "{:.3f}K items/sec".format(value / 1000)
+ if value < 1000**3:
+ return "{:.3f}M items/sec".format(value / 1000**2)
+ else:
+ return "{:.3f}G items/sec".format(value / 1000**3)
+
+
+def bytes_per_seconds_fmt(value):
+ if value < 1024:
+ return "{} bytes/sec".format(value)
+ if value < 1024**2:
+ return "{:.3f} KiB/sec".format(value / 1024)
+ if value < 1024**3:
+ return "{:.3f} MiB/sec".format(value / 1024**2)
+ if value < 1024**4:
+ return "{:.3f} GiB/sec".format(value / 1024**3)
+ else:
+ return "{:.3f} TiB/sec".format(value / 1024**4)
+
+
+def change_fmt(value):
+ return "{:.3%}".format(value)
+
+
+def formatter_for_unit(unit):
+ if unit == "bytes_per_second":
+ return bytes_per_seconds_fmt
+ elif unit == "items_per_second":
+ return items_per_seconds_fmt
+ else:
+ return lambda x: x
+
+
+class BenchmarkComparator:
+ """ Compares two benchmarks.
+
+ Encodes the logic of comparing two benchmarks and taking a decision on
+ if it induce a regression.
+ """
+
+ def __init__(self, contender, baseline, threshold=DEFAULT_THRESHOLD,
+ suite_name=None):
+ self.contender = contender
+ self.baseline = baseline
+ self.threshold = threshold
+ self.suite_name = suite_name
+
+ @property
+ def name(self):
+ return self.baseline.name
+
+ @property
+ def less_is_better(self):
+ return self.baseline.less_is_better
+
+ @property
+ def unit(self):
+ return self.baseline.unit
+
+ @property
+ def change(self):
+ new = self.contender.value
+ old = self.baseline.value
+
+ if old == 0 and new == 0:
+ return 0.0
+ if old == 0:
+ return 0.0
+
+ return float(new - old) / abs(old)
+
+ @property
+ def confidence(self):
+ """ Indicate if a comparison of benchmarks should be trusted. """
+ return True
+
+ @property
+ def regression(self):
+ change = self.change
+ adjusted_change = change if self.less_is_better else -change
+ return (self.confidence and adjusted_change > self.threshold)
+
+ @property
+ def formatted(self):
+ fmt = formatter_for_unit(self.unit)
+ return {
+ "benchmark": self.name,
+ "change": change_fmt(self.change),
+ "regression": self.regression,
+ "baseline": fmt(self.baseline.value),
+ "contender": fmt(self.contender.value),
+ "unit": self.unit,
+ "less_is_better": self.less_is_better,
+ "counters": str(self.baseline.counters)
+ }
+
+ def compare(self, comparator=None):
+ return {
+ "benchmark": self.name,
+ "change": self.change,
+ "regression": self.regression,
+ "baseline": self.baseline.value,
+ "contender": self.contender.value,
+ "unit": self.unit,
+ "less_is_better": self.less_is_better,
+ "counters": self.baseline.counters
+ }
+
+ def __call__(self, **kwargs):
+ return self.compare(**kwargs)
+
+
+def pairwise_compare(contender, baseline):
+ dict_contender = {e.name: e for e in contender}
+ dict_baseline = {e.name: e for e in baseline}
+
+ for name in (dict_contender.keys() & dict_baseline.keys()):
+ yield name, (dict_contender[name], dict_baseline[name])
+
+
+class RunnerComparator:
+ """ Compares suites/benchmarks from runners.
+
+ It is up to the caller that ensure that runners are compatible (both from
+ the same language implementation).
+ """
+
+ def __init__(self, contender, baseline, threshold=DEFAULT_THRESHOLD):
+ self.contender = contender
+ self.baseline = baseline
+ self.threshold = threshold
+
+ @property
+ def comparisons(self):
+ contender = self.contender.suites
+ baseline = self.baseline.suites
+ suites = pairwise_compare(contender, baseline)
+
+ for suite_name, (suite_cont, suite_base) in suites:
+ benchmarks = pairwise_compare(
+ suite_cont.benchmarks, suite_base.benchmarks)
+
+ for _, (bench_cont, bench_base) in benchmarks:
+ yield BenchmarkComparator(bench_cont, bench_base,
+ threshold=self.threshold,
+ suite_name=suite_name)
diff --git a/src/arrow/dev/archery/archery/benchmark/core.py b/src/arrow/dev/archery/archery/benchmark/core.py
new file mode 100644
index 000000000..5a92271a3
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/core.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+def median(values):
+ n = len(values)
+ if n == 0:
+ raise ValueError("median requires at least one value")
+ elif n % 2 == 0:
+ return (values[(n // 2) - 1] + values[n // 2]) / 2
+ else:
+ return values[n // 2]
+
+
+class Benchmark:
+ def __init__(self, name, unit, less_is_better, values, time_unit,
+ times, counters=None):
+ self.name = name
+ self.unit = unit
+ self.less_is_better = less_is_better
+ self.values = sorted(values)
+ self.time_unit = time_unit
+ self.times = sorted(times)
+ self.median = median(self.values)
+ self.counters = counters or {}
+
+ @property
+ def value(self):
+ return self.median
+
+ def __repr__(self):
+ return "Benchmark[name={},value={}]".format(self.name, self.value)
+
+
+class BenchmarkSuite:
+ def __init__(self, name, benchmarks):
+ self.name = name
+ self.benchmarks = benchmarks
+
+ def __repr__(self):
+ return "BenchmarkSuite[name={}, benchmarks={}]".format(
+ self.name, self.benchmarks
+ )
diff --git a/src/arrow/dev/archery/archery/benchmark/google.py b/src/arrow/dev/archery/archery/benchmark/google.py
new file mode 100644
index 000000000..ebcc52636
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/google.py
@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from itertools import filterfalse, groupby, tee
+import json
+import subprocess
+from tempfile import NamedTemporaryFile
+
+from .core import Benchmark
+from ..utils.command import Command
+
+
+def partition(pred, iterable):
+ # adapted from python's examples
+ t1, t2 = tee(iterable)
+ return list(filter(pred, t1)), list(filterfalse(pred, t2))
+
+
+class GoogleBenchmarkCommand(Command):
+ """ Run a google benchmark binary.
+
+ This assumes the binary supports the standard command line options,
+ notably `--benchmark_filter`, `--benchmark_format`, etc...
+ """
+
+ def __init__(self, benchmark_bin, benchmark_filter=None):
+ self.bin = benchmark_bin
+ self.benchmark_filter = benchmark_filter
+
+ def list_benchmarks(self):
+ argv = ["--benchmark_list_tests"]
+ if self.benchmark_filter:
+ argv.append("--benchmark_filter={}".format(self.benchmark_filter))
+ result = self.run(*argv, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ return str.splitlines(result.stdout.decode("utf-8"))
+
+ def results(self, repetitions=1):
+ with NamedTemporaryFile() as out:
+ argv = ["--benchmark_repetitions={}".format(repetitions),
+ "--benchmark_out={}".format(out.name),
+ "--benchmark_out_format=json"]
+
+ if self.benchmark_filter:
+ argv.append(
+ "--benchmark_filter={}".format(self.benchmark_filter)
+ )
+
+ self.run(*argv, check=True)
+ return json.load(out)
+
+
+class GoogleBenchmarkObservation:
+ """ Represents one run of a single (google c++) benchmark.
+
+ Aggregates are reported by Google Benchmark executables alongside
+ other observations whenever repetitions are specified (with
+ `--benchmark_repetitions` on the bare benchmark, or with the
+ archery option `--repetitions`). Aggregate observations are not
+ included in `GoogleBenchmark.runs`.
+
+ RegressionSumKernel/32768/0 1 us 1 us 25.8077GB/s
+ RegressionSumKernel/32768/0 1 us 1 us 25.7066GB/s
+ RegressionSumKernel/32768/0 1 us 1 us 25.1481GB/s
+ RegressionSumKernel/32768/0 1 us 1 us 25.846GB/s
+ RegressionSumKernel/32768/0 1 us 1 us 25.6453GB/s
+ RegressionSumKernel/32768/0_mean 1 us 1 us 25.6307GB/s
+ RegressionSumKernel/32768/0_median 1 us 1 us 25.7066GB/s
+ RegressionSumKernel/32768/0_stddev 0 us 0 us 288.046MB/s
+ """
+
+ def __init__(self, name, real_time, cpu_time, time_unit, run_type,
+ size=None, bytes_per_second=None, items_per_second=None,
+ **counters):
+ self._name = name
+ self.real_time = real_time
+ self.cpu_time = cpu_time
+ self.time_unit = time_unit
+ self.run_type = run_type
+ self.size = size
+ self.bytes_per_second = bytes_per_second
+ self.items_per_second = items_per_second
+ self.counters = counters
+
+ @property
+ def is_aggregate(self):
+ """ Indicate if the observation is a run or an aggregate. """
+ return self.run_type == "aggregate"
+
+ @property
+ def is_realtime(self):
+ """ Indicate if the preferred value is realtime instead of cputime. """
+ return self.name.find("/real_time") != -1
+
+ @property
+ def name(self):
+ name = self._name
+ return name.rsplit("_", maxsplit=1)[0] if self.is_aggregate else name
+
+ @property
+ def time(self):
+ return self.real_time if self.is_realtime else self.cpu_time
+
+ @property
+ def value(self):
+ """ Return the benchmark value."""
+ return self.bytes_per_second or self.items_per_second or self.time
+
+ @property
+ def unit(self):
+ if self.bytes_per_second:
+ return "bytes_per_second"
+ elif self.items_per_second:
+ return "items_per_second"
+ else:
+ return self.time_unit
+
+ def __repr__(self):
+ return str(self.value)
+
+
+class GoogleBenchmark(Benchmark):
+ """ A set of GoogleBenchmarkObservations. """
+
+ def __init__(self, name, runs):
+ """ Initialize a GoogleBenchmark.
+
+ Parameters
+ ----------
+ name: str
+ Name of the benchmark
+ runs: list(GoogleBenchmarkObservation)
+ Repetitions of GoogleBenchmarkObservation run.
+
+ """
+ self.name = name
+ # exclude google benchmark aggregate artifacts
+ _, runs = partition(lambda b: b.is_aggregate, runs)
+ self.runs = sorted(runs, key=lambda b: b.value)
+ unit = self.runs[0].unit
+ time_unit = self.runs[0].time_unit
+ less_is_better = not unit.endswith("per_second")
+ values = [b.value for b in self.runs]
+ times = [b.real_time for b in self.runs]
+ # Slight kludge to extract the UserCounters for each benchmark
+ counters = self.runs[0].counters
+ super().__init__(name, unit, less_is_better, values, time_unit, times,
+ counters)
+
+ def __repr__(self):
+ return "GoogleBenchmark[name={},runs={}]".format(self.names, self.runs)
+
+ @classmethod
+ def from_json(cls, payload):
+ def group_key(x):
+ return x.name
+
+ benchmarks = map(lambda x: GoogleBenchmarkObservation(**x), payload)
+ groups = groupby(sorted(benchmarks, key=group_key), group_key)
+ return [cls(k, list(bs)) for k, bs in groups]
diff --git a/src/arrow/dev/archery/archery/benchmark/jmh.py b/src/arrow/dev/archery/archery/benchmark/jmh.py
new file mode 100644
index 000000000..f531b6de1
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/jmh.py
@@ -0,0 +1,201 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from itertools import filterfalse, groupby, tee
+import json
+import subprocess
+from tempfile import NamedTemporaryFile
+
+from .core import Benchmark
+from ..utils.command import Command
+from ..utils.maven import Maven
+
+
+def partition(pred, iterable):
+ # adapted from python's examples
+ t1, t2 = tee(iterable)
+ return list(filter(pred, t1)), list(filterfalse(pred, t2))
+
+
+class JavaMicrobenchmarkHarnessCommand(Command):
+ """ Run a Java Micro Benchmark Harness
+
+ This assumes the binary supports the standard command line options,
+ notably `-Dbenchmark_filter`
+ """
+
+ def __init__(self, build, benchmark_filter=None):
+ self.benchmark_filter = benchmark_filter
+ self.build = build
+ self.maven = Maven()
+
+ """ Extract benchmark names from output between "Benchmarks:" and "[INFO]".
+ Assume the following output:
+ ...
+ Benchmarks:
+ org.apache.arrow.vector.IntBenchmarks.setIntDirectly
+ ...
+ org.apache.arrow.vector.IntBenchmarks.setWithValueHolder
+ org.apache.arrow.vector.IntBenchmarks.setWithWriter
+ ...
+ [INFO]
+ """
+
+ def list_benchmarks(self):
+ argv = []
+ if self.benchmark_filter:
+ argv.append("-Dbenchmark.filter={}".format(self.benchmark_filter))
+ result = self.build.list(
+ *argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ lists = []
+ benchmarks = False
+ for line in str.splitlines(result.stdout.decode("utf-8")):
+ if not benchmarks:
+ if line.startswith("Benchmarks:"):
+ benchmarks = True
+ else:
+ if line.startswith("org.apache.arrow"):
+ lists.append(line)
+ if line.startswith("[INFO]"):
+ break
+ return lists
+
+ def results(self, repetitions):
+ with NamedTemporaryFile(suffix=".json") as out:
+ argv = ["-Dbenchmark.runs={}".format(repetitions),
+ "-Dbenchmark.resultfile={}".format(out.name),
+ "-Dbenchmark.resultformat=json"]
+ if self.benchmark_filter:
+ argv.append(
+ "-Dbenchmark.filter={}".format(self.benchmark_filter)
+ )
+
+ self.build.benchmark(*argv, check=True)
+ return json.load(out)
+
+
+class JavaMicrobenchmarkHarnessObservation:
+ """ Represents one run of a single Java Microbenchmark Harness
+ """
+
+ def __init__(self, benchmark, primaryMetric,
+ forks, warmupIterations, measurementIterations, **counters):
+ self.name = benchmark
+ self.primaryMetric = primaryMetric
+ self.score = primaryMetric["score"]
+ self.score_unit = primaryMetric["scoreUnit"]
+ self.forks = forks
+ self.warmups = warmupIterations
+ self.runs = measurementIterations
+ self.counters = {
+ "mode": counters["mode"],
+ "threads": counters["threads"],
+ "warmups": warmupIterations,
+ "warmupTime": counters["warmupTime"],
+ "measurements": measurementIterations,
+ "measurementTime": counters["measurementTime"],
+ "jvmArgs": counters["jvmArgs"]
+ }
+ self.reciprocal_value = True if self.score_unit.endswith(
+ "/op") else False
+ if self.score_unit.startswith("ops/"):
+ idx = self.score_unit.find("/")
+ self.normalizePerSec(self.score_unit[idx+1:])
+ elif self.score_unit.endswith("/op"):
+ idx = self.score_unit.find("/")
+ self.normalizePerSec(self.score_unit[:idx])
+ else:
+ self.normalizeFactor = 1
+
+ @property
+ def value(self):
+ """ Return the benchmark value."""
+ val = 1 / self.score if self.reciprocal_value else self.score
+ return val * self.normalizeFactor
+
+ def normalizePerSec(self, unit):
+ if unit == "ns":
+ self.normalizeFactor = 1000 * 1000 * 1000
+ elif unit == "us":
+ self.normalizeFactor = 1000 * 1000
+ elif unit == "ms":
+ self.normalizeFactor = 1000
+ elif unit == "min":
+ self.normalizeFactor = 1 / 60
+ elif unit == "hr":
+ self.normalizeFactor = 1 / (60 * 60)
+ elif unit == "day":
+ self.normalizeFactor = 1 / (60 * 60 * 24)
+ else:
+ self.normalizeFactor = 1
+
+ @property
+ def unit(self):
+ if self.score_unit.startswith("ops/"):
+ return "items_per_second"
+ elif self.score_unit.endswith("/op"):
+ return "items_per_second"
+ else:
+ return "?"
+
+ def __repr__(self):
+ return str(self.value)
+
+
+class JavaMicrobenchmarkHarness(Benchmark):
+ """ A set of JavaMicrobenchmarkHarnessObservations. """
+
+ def __init__(self, name, runs):
+ """ Initialize a JavaMicrobenchmarkHarness.
+
+ Parameters
+ ----------
+ name: str
+ Name of the benchmark
+ forks: int
+ warmups: int
+ runs: int
+ runs: list(JavaMicrobenchmarkHarnessObservation)
+ Repetitions of JavaMicrobenchmarkHarnessObservation run.
+
+ """
+ self.name = name
+ self.runs = sorted(runs, key=lambda b: b.value)
+ unit = self.runs[0].unit
+ time_unit = "N/A"
+ less_is_better = not unit.endswith("per_second")
+ values = [b.value for b in self.runs]
+ times = []
+ # Slight kludge to extract the UserCounters for each benchmark
+ counters = self.runs[0].counters
+ super().__init__(name, unit, less_is_better, values, time_unit, times,
+ counters)
+
+ def __repr__(self):
+ return "JavaMicrobenchmark[name={},runs={}]".format(
+ self.name, self.runs)
+
+ @classmethod
+ def from_json(cls, payload):
+ def group_key(x):
+ return x.name
+
+ benchmarks = map(
+ lambda x: JavaMicrobenchmarkHarnessObservation(**x), payload)
+ groups = groupby(sorted(benchmarks, key=group_key), group_key)
+ return [cls(k, list(bs)) for k, bs in groups]
diff --git a/src/arrow/dev/archery/archery/benchmark/runner.py b/src/arrow/dev/archery/archery/benchmark/runner.py
new file mode 100644
index 000000000..fc6d354b1
--- /dev/null
+++ b/src/arrow/dev/archery/archery/benchmark/runner.py
@@ -0,0 +1,313 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import glob
+import json
+import os
+import re
+
+from .core import BenchmarkSuite
+from .google import GoogleBenchmarkCommand, GoogleBenchmark
+from .jmh import JavaMicrobenchmarkHarnessCommand, JavaMicrobenchmarkHarness
+from ..lang.cpp import CppCMakeDefinition, CppConfiguration
+from ..lang.java import JavaMavenDefinition, JavaConfiguration
+from ..utils.cmake import CMakeBuild
+from ..utils.maven import MavenBuild
+from ..utils.logger import logger
+
+
+def regex_filter(re_expr):
+ if re_expr is None:
+ return lambda s: True
+ re_comp = re.compile(re_expr)
+ return lambda s: re_comp.search(s)
+
+
+DEFAULT_REPETITIONS = 1
+
+
+class BenchmarkRunner:
+ def __init__(self, suite_filter=None, benchmark_filter=None,
+ repetitions=DEFAULT_REPETITIONS):
+ self.suite_filter = suite_filter
+ self.benchmark_filter = benchmark_filter
+ self.repetitions = repetitions
+
+ @property
+ def suites(self):
+ raise NotImplementedError("BenchmarkRunner must implement suites")
+
+ @staticmethod
+ def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs):
+ raise NotImplementedError(
+ "BenchmarkRunner must implement from_rev_or_path")
+
+
+class StaticBenchmarkRunner(BenchmarkRunner):
+ """ Run suites from a (static) set of suites. """
+
+ def __init__(self, suites, **kwargs):
+ self._suites = suites
+ super().__init__(**kwargs)
+
+ @property
+ def list_benchmarks(self):
+ for suite in self._suites:
+ for benchmark in suite.benchmarks:
+ yield "{}.{}".format(suite.name, benchmark.name)
+
+ @property
+ def suites(self):
+ suite_fn = regex_filter(self.suite_filter)
+ benchmark_fn = regex_filter(self.benchmark_filter)
+
+ for suite in (s for s in self._suites if suite_fn(s.name)):
+ benchmarks = [b for b in suite.benchmarks if benchmark_fn(b.name)]
+ yield BenchmarkSuite(suite.name, benchmarks)
+
+ @classmethod
+ def is_json_result(cls, path_or_str):
+ builder = None
+ try:
+ builder = cls.from_json(path_or_str)
+ except BaseException:
+ pass
+
+ return builder is not None
+
+ @staticmethod
+ def from_json(path_or_str, **kwargs):
+ # .codec imported here to break recursive imports
+ from .codec import BenchmarkRunnerCodec
+ if os.path.isfile(path_or_str):
+ with open(path_or_str) as f:
+ loaded = json.load(f)
+ else:
+ loaded = json.loads(path_or_str)
+ return BenchmarkRunnerCodec.decode(loaded, **kwargs)
+
+ def __repr__(self):
+ return "BenchmarkRunner[suites={}]".format(list(self.suites))
+
+
+class CppBenchmarkRunner(BenchmarkRunner):
+ """ Run suites from a CMakeBuild. """
+
+ def __init__(self, build, **kwargs):
+ """ Initialize a CppBenchmarkRunner. """
+ self.build = build
+ super().__init__(**kwargs)
+
+ @staticmethod
+ def default_configuration(**kwargs):
+ """ Returns the default benchmark configuration. """
+ return CppConfiguration(
+ build_type="release", with_tests=False, with_benchmarks=True,
+ with_compute=True,
+ with_csv=True,
+ with_dataset=True,
+ with_json=True,
+ with_parquet=True,
+ with_python=False,
+ with_brotli=True,
+ with_bz2=True,
+ with_lz4=True,
+ with_snappy=True,
+ with_zlib=True,
+ with_zstd=True,
+ **kwargs)
+
+ @property
+ def suites_binaries(self):
+ """ Returns a list of benchmark binaries for this build. """
+ # Ensure build is up-to-date to run benchmarks
+ self.build()
+ # Not the best method, but works for now
+ glob_expr = os.path.join(self.build.binaries_dir, "*-benchmark")
+ return {os.path.basename(b): b for b in glob.glob(glob_expr)}
+
+ def suite(self, name, suite_bin):
+ """ Returns the resulting benchmarks for a given suite. """
+ suite_cmd = GoogleBenchmarkCommand(suite_bin, self.benchmark_filter)
+
+ # Ensure there will be data
+ benchmark_names = suite_cmd.list_benchmarks()
+ if not benchmark_names:
+ return None
+
+ results = suite_cmd.results(repetitions=self.repetitions)
+ benchmarks = GoogleBenchmark.from_json(results.get("benchmarks"))
+ return BenchmarkSuite(name, benchmarks)
+
+ @property
+ def list_benchmarks(self):
+ for suite_name, suite_bin in self.suites_binaries.items():
+ suite_cmd = GoogleBenchmarkCommand(suite_bin)
+ for benchmark_name in suite_cmd.list_benchmarks():
+ yield "{}.{}".format(suite_name, benchmark_name)
+
+ @property
+ def suites(self):
+ """ Returns all suite for a runner. """
+ suite_matcher = regex_filter(self.suite_filter)
+
+ suite_and_binaries = self.suites_binaries
+ for suite_name in suite_and_binaries:
+ if not suite_matcher(suite_name):
+ logger.debug("Ignoring suite {}".format(suite_name))
+ continue
+
+ suite_bin = suite_and_binaries[suite_name]
+ suite = self.suite(suite_name, suite_bin)
+
+ # Filter may exclude all benchmarks
+ if not suite:
+ logger.debug("Suite {} executed but no results"
+ .format(suite_name))
+ continue
+
+ yield suite
+
+ @staticmethod
+ def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs):
+ """ Returns a BenchmarkRunner from a path or a git revision.
+
+ First, it checks if `rev_or_path` is a valid path (or string) of a json
+ object that can deserialize to a BenchmarkRunner. If so, it initialize
+ a StaticBenchmarkRunner from it. This allows memoizing the result of a
+ run in a file or a string.
+
+ Second, it checks if `rev_or_path` points to a valid CMake build
+ directory. If so, it creates a CppBenchmarkRunner with this existing
+ CMakeBuild.
+
+ Otherwise, it assumes `rev_or_path` is a revision and clone/checkout
+ the given revision and create a fresh CMakeBuild.
+ """
+ build = None
+ if StaticBenchmarkRunner.is_json_result(rev_or_path):
+ return StaticBenchmarkRunner.from_json(rev_or_path, **kwargs)
+ elif CMakeBuild.is_build_dir(rev_or_path):
+ build = CMakeBuild.from_path(rev_or_path)
+ return CppBenchmarkRunner(build, **kwargs)
+ else:
+ # Revisions can references remote via the `/` character, ensure
+ # that the revision is path friendly
+ path_rev = rev_or_path.replace("/", "_")
+ root_rev = os.path.join(root, path_rev)
+ os.mkdir(root_rev)
+
+ clone_dir = os.path.join(root_rev, "arrow")
+ # Possibly checkout the sources at given revision, no need to
+ # perform cleanup on cloned repository as root_rev is reclaimed.
+ src_rev, _ = src.at_revision(rev_or_path, clone_dir)
+ cmake_def = CppCMakeDefinition(src_rev.cpp, cmake_conf)
+ build_dir = os.path.join(root_rev, "build")
+ return CppBenchmarkRunner(cmake_def.build(build_dir), **kwargs)
+
+
+class JavaBenchmarkRunner(BenchmarkRunner):
+ """ Run suites for Java. """
+
+ # default repetitions is 5 for Java microbenchmark harness
+ def __init__(self, build, **kwargs):
+ """ Initialize a JavaBenchmarkRunner. """
+ self.build = build
+ super().__init__(**kwargs)
+
+ @staticmethod
+ def default_configuration(**kwargs):
+ """ Returns the default benchmark configuration. """
+ return JavaConfiguration(**kwargs)
+
+ def suite(self, name):
+ """ Returns the resulting benchmarks for a given suite. """
+ # update .m2 directory, which installs target jars
+ self.build.build()
+
+ suite_cmd = JavaMicrobenchmarkHarnessCommand(
+ self.build, self.benchmark_filter)
+
+ # Ensure there will be data
+ benchmark_names = suite_cmd.list_benchmarks()
+ if not benchmark_names:
+ return None
+
+ results = suite_cmd.results(repetitions=self.repetitions)
+ benchmarks = JavaMicrobenchmarkHarness.from_json(results)
+ return BenchmarkSuite(name, benchmarks)
+
+ @property
+ def list_benchmarks(self):
+ """ Returns all suite names """
+ # Ensure build is up-to-date to run benchmarks
+ self.build.build()
+
+ suite_cmd = JavaMicrobenchmarkHarnessCommand(self.build)
+ benchmark_names = suite_cmd.list_benchmarks()
+ for benchmark_name in benchmark_names:
+ yield "{}".format(benchmark_name)
+
+ @property
+ def suites(self):
+ """ Returns all suite for a runner. """
+ suite_name = "JavaBenchmark"
+ suite = self.suite(suite_name)
+
+ # Filter may exclude all benchmarks
+ if not suite:
+ logger.debug("Suite {} executed but no results"
+ .format(suite_name))
+ return
+
+ yield suite
+
+ @staticmethod
+ def from_rev_or_path(src, root, rev_or_path, maven_conf, **kwargs):
+ """ Returns a BenchmarkRunner from a path or a git revision.
+
+ First, it checks if `rev_or_path` is a valid path (or string) of a json
+ object that can deserialize to a BenchmarkRunner. If so, it initialize
+ a StaticBenchmarkRunner from it. This allows memoizing the result of a
+ run in a file or a string.
+
+ Second, it checks if `rev_or_path` points to a valid Maven build
+ directory. If so, it creates a JavaBenchmarkRunner with this existing
+ MavenBuild.
+
+ Otherwise, it assumes `rev_or_path` is a revision and clone/checkout
+ the given revision and create a fresh MavenBuild.
+ """
+ if StaticBenchmarkRunner.is_json_result(rev_or_path):
+ return StaticBenchmarkRunner.from_json(rev_or_path, **kwargs)
+ elif MavenBuild.is_build_dir(rev_or_path):
+ maven_def = JavaMavenDefinition(rev_or_path, maven_conf)
+ return JavaBenchmarkRunner(maven_def.build(rev_or_path), **kwargs)
+ else:
+ # Revisions can references remote via the `/` character, ensure
+ # that the revision is path friendly
+ path_rev = rev_or_path.replace("/", "_")
+ root_rev = os.path.join(root, path_rev)
+ os.mkdir(root_rev)
+
+ clone_dir = os.path.join(root_rev, "arrow")
+ # Possibly checkout the sources at given revision, no need to
+ # perform cleanup on cloned repository as root_rev is reclaimed.
+ src_rev, _ = src.at_revision(rev_or_path, clone_dir)
+ maven_def = JavaMavenDefinition(src_rev.java, maven_conf)
+ build_dir = os.path.join(root_rev, "arrow/java")
+ return JavaBenchmarkRunner(maven_def.build(build_dir), **kwargs)
diff --git a/src/arrow/dev/archery/archery/bot.py b/src/arrow/dev/archery/archery/bot.py
new file mode 100644
index 000000000..e8fbbdd04
--- /dev/null
+++ b/src/arrow/dev/archery/archery/bot.py
@@ -0,0 +1,267 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shlex
+from pathlib import Path
+from functools import partial
+import tempfile
+
+import click
+import github
+
+from .utils.git import git
+from .utils.logger import logger
+from .crossbow import Repo, Queue, Config, Target, Job, CommentReport
+
+
+class EventError(Exception):
+ pass
+
+
+class CommandError(Exception):
+
+ def __init__(self, message):
+ self.message = message
+
+
+class _CommandMixin:
+
+ def get_help_option(self, ctx):
+ def show_help(ctx, param, value):
+ if value and not ctx.resilient_parsing:
+ raise click.UsageError(ctx.get_help())
+ option = super().get_help_option(ctx)
+ option.callback = show_help
+ return option
+
+ def __call__(self, message, **kwargs):
+ args = shlex.split(message)
+ try:
+ with self.make_context(self.name, args=args, obj=kwargs) as ctx:
+ return self.invoke(ctx)
+ except click.ClickException as e:
+ raise CommandError(e.format_message())
+
+
+class Command(_CommandMixin, click.Command):
+ pass
+
+
+class Group(_CommandMixin, click.Group):
+
+ def command(self, *args, **kwargs):
+ kwargs.setdefault('cls', Command)
+ return super().command(*args, **kwargs)
+
+ def group(self, *args, **kwargs):
+ kwargs.setdefault('cls', Group)
+ return super().group(*args, **kwargs)
+
+ def parse_args(self, ctx, args):
+ if not args and self.no_args_is_help and not ctx.resilient_parsing:
+ raise click.UsageError(ctx.get_help())
+ return super().parse_args(ctx, args)
+
+
+command = partial(click.command, cls=Command)
+group = partial(click.group, cls=Group)
+
+
+class CommentBot:
+
+ def __init__(self, name, handler, token=None):
+ # TODO(kszucs): validate
+ assert isinstance(name, str)
+ assert callable(handler)
+ self.name = name
+ self.handler = handler
+ self.github = github.Github(token)
+
+ def parse_command(self, payload):
+ # only allow users of apache org to submit commands, for more see
+ # https://developer.github.com/v4/enum/commentauthorassociation/
+ allowed_roles = {'OWNER', 'MEMBER', 'CONTRIBUTOR'}
+ mention = '@{}'.format(self.name)
+ comment = payload['comment']
+
+ if payload['sender']['login'] == self.name:
+ raise EventError("Don't respond to itself")
+ elif payload['action'] not in {'created', 'edited'}:
+ raise EventError("Don't respond to comment deletion")
+ elif comment['author_association'] not in allowed_roles:
+ raise EventError(
+ "Don't respond to comments from non-authorized users"
+ )
+ elif not comment['body'].lstrip().startswith(mention):
+ raise EventError("The bot is not mentioned")
+
+ # Parse the comment, removing the bot mentioned (and everything
+ # before it)
+ command = payload['comment']['body'].split(mention)[-1]
+
+ # then split on newlines and keep only the first line
+ # (ignoring all other lines)
+ return command.split("\n")[0].strip()
+
+ def handle(self, event, payload):
+ try:
+ command = self.parse_command(payload)
+ except EventError as e:
+ logger.error(e)
+ # see the possible reasons in the validate method
+ return
+
+ if event == 'issue_comment':
+ return self.handle_issue_comment(command, payload)
+ elif event == 'pull_request_review_comment':
+ return self.handle_review_comment(command, payload)
+ else:
+ raise ValueError("Unexpected event type {}".format(event))
+
+ def handle_issue_comment(self, command, payload):
+ repo = self.github.get_repo(payload['repository']['id'], lazy=True)
+ issue = repo.get_issue(payload['issue']['number'])
+
+ try:
+ pull = issue.as_pull_request()
+ except github.GithubException:
+ return issue.create_comment(
+ "The comment bot only listens to pull request comments!"
+ )
+
+ comment = pull.get_issue_comment(payload['comment']['id'])
+ try:
+ self.handler(command, issue=issue, pull_request=pull,
+ comment=comment)
+ except CommandError as e:
+ logger.error(e)
+ pull.create_issue_comment("```\n{}\n```".format(e.message))
+ except Exception as e:
+ logger.exception(e)
+ comment.create_reaction('-1')
+ else:
+ comment.create_reaction('+1')
+
+ def handle_review_comment(self, payload):
+ raise NotImplementedError()
+
+
+@group(name='@github-actions')
+@click.pass_context
+def actions(ctx):
+ """Ursabot"""
+ ctx.ensure_object(dict)
+
+
+@actions.group()
+@click.option('--crossbow', '-c', default='ursacomputing/crossbow',
+ help='Crossbow repository on github to use')
+@click.pass_obj
+def crossbow(obj, crossbow):
+ """
+ Trigger crossbow builds for this pull request
+ """
+ obj['crossbow_repo'] = crossbow
+
+
+def _clone_arrow_and_crossbow(dest, crossbow_repo, pull_request):
+ """
+ Clone the repositories and initialize crossbow objects.
+
+ Parameters
+ ----------
+ dest : Path
+ Filesystem path to clone the repositories to.
+ crossbow_repo : str
+ Github repository name, like kszucs/crossbow.
+ pull_request : pygithub.PullRequest
+ Object containing information about the pull request the comment bot
+ was triggered from.
+ """
+ arrow_path = dest / 'arrow'
+ queue_path = dest / 'crossbow'
+
+ # clone arrow and checkout the pull request's branch
+ pull_request_ref = 'pull/{}/head:{}'.format(
+ pull_request.number, pull_request.head.ref
+ )
+ git.clone(pull_request.base.repo.clone_url, str(arrow_path))
+ git.fetch('origin', pull_request_ref, git_dir=arrow_path)
+ git.checkout(pull_request.head.ref, git_dir=arrow_path)
+
+ # clone crossbow repository
+ crossbow_url = 'https://github.com/{}'.format(crossbow_repo)
+ git.clone(crossbow_url, str(queue_path))
+
+ # initialize crossbow objects
+ github_token = os.environ['CROSSBOW_GITHUB_TOKEN']
+ arrow = Repo(arrow_path)
+ queue = Queue(queue_path, github_token=github_token, require_https=True)
+
+ return (arrow, queue)
+
+
+@crossbow.command()
+@click.argument('tasks', nargs=-1, required=False)
+@click.option('--group', '-g', 'groups', multiple=True,
+ help='Submit task groups as defined in tests.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+ help='Additional task parameters for rendering the CI templates')
+@click.option('--arrow-version', '-v', default=None,
+ help='Set target version explicitly.')
+@click.pass_obj
+def submit(obj, tasks, groups, params, arrow_version):
+ """
+ Submit crossbow testing tasks.
+
+ See groups defined in arrow/dev/tasks/tasks.yml
+ """
+ crossbow_repo = obj['crossbow_repo']
+ pull_request = obj['pull_request']
+ with tempfile.TemporaryDirectory() as tmpdir:
+ tmpdir = Path(tmpdir)
+ arrow, queue = _clone_arrow_and_crossbow(
+ dest=Path(tmpdir),
+ crossbow_repo=crossbow_repo,
+ pull_request=pull_request,
+ )
+ # load available tasks configuration and groups from yaml
+ config = Config.load_yaml(arrow.path / "dev" / "tasks" / "tasks.yml")
+ config.validate()
+
+ # initialize the crossbow build's target repository
+ target = Target.from_repo(arrow, version=arrow_version,
+ remote=pull_request.head.repo.clone_url,
+ branch=pull_request.head.ref)
+
+ # parse additional job parameters
+ params = dict([p.split("=") for p in params])
+
+ # instantiate the job object
+ job = Job.from_config(config=config, target=target, tasks=tasks,
+ groups=groups, params=params)
+
+ # add the job to the crossbow queue and push to the remote repository
+ queue.put(job, prefix="actions")
+ queue.push()
+
+ # render the response comment's content
+ report = CommentReport(job, crossbow_repo=crossbow_repo)
+
+ # send the response
+ pull_request.create_issue_comment(report.show())
diff --git a/src/arrow/dev/archery/archery/cli.py b/src/arrow/dev/archery/archery/cli.py
new file mode 100644
index 000000000..d408be3cc
--- /dev/null
+++ b/src/arrow/dev/archery/archery/cli.py
@@ -0,0 +1,943 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import namedtuple
+from io import StringIO
+import click
+import errno
+import json
+import logging
+import os
+import pathlib
+import sys
+
+from .benchmark.codec import JsonEncoder
+from .benchmark.compare import RunnerComparator, DEFAULT_THRESHOLD
+from .benchmark.runner import CppBenchmarkRunner, JavaBenchmarkRunner
+from .compat import _import_pandas
+from .lang.cpp import CppCMakeDefinition, CppConfiguration
+from .utils.cli import ArrowBool, validate_arrow_sources, add_optional_command
+from .utils.lint import linter, python_numpydoc, LintValidationException
+from .utils.logger import logger, ctx as log_ctx
+from .utils.source import ArrowSources
+from .utils.tmpdir import tmpdir
+
+# Set default logging to INFO in command line.
+logging.basicConfig(level=logging.INFO)
+
+
+BOOL = ArrowBool()
+
+
+@click.group()
+@click.option("--debug", type=BOOL, is_flag=True, default=False,
+ help="Increase logging with debugging output.")
+@click.option("--pdb", type=BOOL, is_flag=True, default=False,
+ help="Invoke pdb on uncaught exception.")
+@click.option("-q", "--quiet", type=BOOL, is_flag=True, default=False,
+ help="Silence executed commands.")
+@click.pass_context
+def archery(ctx, debug, pdb, quiet):
+ """ Apache Arrow developer utilities.
+
+ See sub-commands help with `archery <cmd> --help`.
+
+ """
+ # Ensure ctx.obj exists
+ ctx.ensure_object(dict)
+
+ log_ctx.quiet = quiet
+ if debug:
+ logger.setLevel(logging.DEBUG)
+
+ ctx.debug = debug
+
+ if pdb:
+ import pdb
+ sys.excepthook = lambda t, v, e: pdb.pm()
+
+
+build_dir_type = click.Path(dir_okay=True, file_okay=False, resolve_path=True)
+# Supported build types
+build_type = click.Choice(["debug", "relwithdebinfo", "release"],
+ case_sensitive=False)
+# Supported warn levels
+warn_level_type = click.Choice(["everything", "checkin", "production"],
+ case_sensitive=False)
+
+simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"],
+ case_sensitive=True)
+
+
+def cpp_toolchain_options(cmd):
+ options = [
+ click.option("--cc", metavar="<compiler>", help="C compiler."),
+ click.option("--cxx", metavar="<compiler>", help="C++ compiler."),
+ click.option("--cxx-flags", help="C++ compiler flags."),
+ click.option("--cpp-package-prefix",
+ help=("Value to pass for ARROW_PACKAGE_PREFIX and "
+ "use ARROW_DEPENDENCY_SOURCE=SYSTEM"))
+ ]
+ return _apply_options(cmd, options)
+
+
+def java_toolchain_options(cmd):
+ options = [
+ click.option("--java-home", metavar="<java_home>",
+ help="Path to Java Developers Kit."),
+ click.option("--java-options", help="java compiler options."),
+ ]
+ return _apply_options(cmd, options)
+
+
+def _apply_options(cmd, options):
+ for option in options:
+ cmd = option(cmd)
+ return cmd
+
+
+@archery.command(short_help="Initialize an Arrow C++ build")
+@click.option("--src", metavar="<arrow_src>", default=None,
+ callback=validate_arrow_sources,
+ help="Specify Arrow source directory")
+# toolchain
+@cpp_toolchain_options
+@click.option("--build-type", default=None, type=build_type,
+ help="CMake's CMAKE_BUILD_TYPE")
+@click.option("--warn-level", default="production", type=warn_level_type,
+ help="Controls compiler warnings -W(no-)error.")
+@click.option("--use-gold-linker", default=True, type=BOOL,
+ help="Toggles ARROW_USE_LD_GOLD option.")
+@click.option("--simd-level", default="SSE4_2", type=simd_level,
+ help="Toggles ARROW_SIMD_LEVEL option.")
+# Tests and benchmarks
+@click.option("--with-tests", default=True, type=BOOL,
+ help="Build with tests.")
+@click.option("--with-benchmarks", default=None, type=BOOL,
+ help="Build with benchmarks.")
+@click.option("--with-examples", default=None, type=BOOL,
+ help="Build with examples.")
+@click.option("--with-integration", default=None, type=BOOL,
+ help="Build with integration test executables.")
+# Static checks
+@click.option("--use-asan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_ASAN sanitizer.")
+@click.option("--use-tsan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_TSAN sanitizer.")
+@click.option("--use-ubsan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_UBSAN sanitizer.")
+@click.option("--with-fuzzing", default=None, type=BOOL,
+ help="Toggle ARROW_FUZZING.")
+# Components
+@click.option("--with-compute", default=None, type=BOOL,
+ help="Build the Arrow compute module.")
+@click.option("--with-csv", default=None, type=BOOL,
+ help="Build the Arrow CSV parser module.")
+@click.option("--with-cuda", default=None, type=BOOL,
+ help="Build the Arrow CUDA extensions.")
+@click.option("--with-dataset", default=None, type=BOOL,
+ help="Build the Arrow dataset module.")
+@click.option("--with-filesystem", default=None, type=BOOL,
+ help="Build the Arrow filesystem layer.")
+@click.option("--with-flight", default=None, type=BOOL,
+ help="Build with Flight rpc support.")
+@click.option("--with-gandiva", default=None, type=BOOL,
+ help="Build with Gandiva expression compiler support.")
+@click.option("--with-hdfs", default=None, type=BOOL,
+ help="Build the Arrow HDFS bridge.")
+@click.option("--with-hiveserver2", default=None, type=BOOL,
+ help="Build the HiveServer2 client and arrow adapater.")
+@click.option("--with-ipc", default=None, type=BOOL,
+ help="Build the Arrow IPC extensions.")
+@click.option("--with-json", default=None, type=BOOL,
+ help="Build the Arrow JSON parser module.")
+@click.option("--with-jni", default=None, type=BOOL,
+ help="Build the Arrow JNI lib.")
+@click.option("--with-mimalloc", default=None, type=BOOL,
+ help="Build the Arrow mimalloc based allocator.")
+@click.option("--with-parquet", default=None, type=BOOL,
+ help="Build with Parquet file support.")
+@click.option("--with-plasma", default=None, type=BOOL,
+ help="Build with Plasma object store support.")
+@click.option("--with-python", default=None, type=BOOL,
+ help="Build the Arrow CPython extesions.")
+@click.option("--with-r", default=None, type=BOOL,
+ help="Build the Arrow R extensions. This is not a CMake option, "
+ "it will toggle required options")
+@click.option("--with-s3", default=None, type=BOOL,
+ help="Build Arrow with S3 support.")
+# Compressions
+@click.option("--with-brotli", default=None, type=BOOL,
+ help="Build Arrow with brotli compression.")
+@click.option("--with-bz2", default=None, type=BOOL,
+ help="Build Arrow with bz2 compression.")
+@click.option("--with-lz4", default=None, type=BOOL,
+ help="Build Arrow with lz4 compression.")
+@click.option("--with-snappy", default=None, type=BOOL,
+ help="Build Arrow with snappy compression.")
+@click.option("--with-zlib", default=None, type=BOOL,
+ help="Build Arrow with zlib compression.")
+@click.option("--with-zstd", default=None, type=BOOL,
+ help="Build Arrow with zstd compression.")
+# CMake extra feature
+@click.option("--cmake-extras", type=str, multiple=True,
+ help="Extra flags/options to pass to cmake invocation. "
+ "Can be stacked")
+@click.option("--install-prefix", type=str,
+ help="Destination directory where files are installed. Expand to"
+ "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the"
+ "variable exists.")
+# misc
+@click.option("-f", "--force", type=BOOL, is_flag=True, default=False,
+ help="Delete existing build directory if found.")
+@click.option("--targets", type=str, multiple=True,
+ help="Generator targets to run. Can be stacked.")
+@click.argument("build_dir", type=build_dir_type)
+@click.pass_context
+def build(ctx, src, build_dir, force, targets, **kwargs):
+ """ Initialize a C++ build directory.
+
+ The build command creates a directory initialized with Arrow's cpp source
+ cmake and configuration. It can also optionally invoke the generator to
+ test the build (and used in scripts).
+
+ Note that archery will carry the caller environment. It will also not touch
+ an existing directory, one must use the `--force` option to remove the
+ existing directory.
+
+ Examples:
+
+ \b
+ # Initialize build with clang8 and avx2 support in directory `clang8-build`
+ \b
+ archery build --cc=clang-8 --cxx=clang++-8 --cxx-flags=-mavx2 clang8-build
+
+ \b
+ # Builds and run test
+ archery build --targets=all --targets=test build
+ """
+ # Arrow's cpp cmake configuration
+ conf = CppConfiguration(**kwargs)
+ # This is a closure around cmake invocation, e.g. calling `def.build()`
+ # yields a directory ready to be run with the generator
+ cmake_def = CppCMakeDefinition(src.cpp, conf)
+ # Create build directory
+ build = cmake_def.build(build_dir, force=force)
+
+ for target in targets:
+ build.run(target)
+
+
+LintCheck = namedtuple('LintCheck', ('option_name', 'help'))
+
+lint_checks = [
+ LintCheck('clang-format', "Format C++ files with clang-format."),
+ LintCheck('clang-tidy', "Lint C++ files with clang-tidy."),
+ LintCheck('cpplint', "Lint C++ files with cpplint."),
+ LintCheck('iwyu', "Lint changed C++ files with Include-What-You-Use."),
+ LintCheck('python',
+ "Format and lint Python files with autopep8 and flake8."),
+ LintCheck('numpydoc', "Lint Python files with numpydoc."),
+ LintCheck('cmake-format', "Format CMake files with cmake-format.py."),
+ LintCheck('rat',
+ "Check all sources files for license texts via Apache RAT."),
+ LintCheck('r', "Lint R files."),
+ LintCheck('docker', "Lint Dockerfiles with hadolint."),
+]
+
+
+def decorate_lint_command(cmd):
+ """
+ Decorate the lint() command function to add individual per-check options.
+ """
+ for check in lint_checks:
+ option = click.option("--{0}/--no-{0}".format(check.option_name),
+ default=None, help=check.help)
+ cmd = option(cmd)
+ return cmd
+
+
+@archery.command(short_help="Check Arrow source tree for errors")
+@click.option("--src", metavar="<arrow_src>", default=None,
+ callback=validate_arrow_sources,
+ help="Specify Arrow source directory")
+@click.option("--fix", is_flag=True, type=BOOL, default=False,
+ help="Toggle fixing the lint errors if the linter supports it.")
+@click.option("--iwyu_all", is_flag=True, type=BOOL, default=False,
+ help="Run IWYU on all C++ files if enabled")
+@click.option("-a", "--all", is_flag=True, default=False,
+ help="Enable all checks.")
+@decorate_lint_command
+@click.pass_context
+def lint(ctx, src, fix, iwyu_all, **checks):
+ if checks.pop('all'):
+ # "--all" is given => enable all non-selected checks
+ for k, v in checks.items():
+ if v is None:
+ checks[k] = True
+ if not any(checks.values()):
+ raise click.UsageError(
+ "Need to enable at least one lint check (try --help)")
+ try:
+ linter(src, fix, iwyu_all=iwyu_all, **checks)
+ except LintValidationException:
+ sys.exit(1)
+
+
+@archery.command(short_help="Lint python docstring with NumpyDoc")
+@click.argument('symbols', nargs=-1)
+@click.option("--src", metavar="<arrow_src>", default=None,
+ callback=validate_arrow_sources,
+ help="Specify Arrow source directory")
+@click.option("--allow-rule", "-a", multiple=True,
+ help="Allow only these rules")
+@click.option("--disallow-rule", "-d", multiple=True,
+ help="Disallow these rules")
+def numpydoc(src, symbols, allow_rule, disallow_rule):
+ """
+ Pass list of modules or symbols as arguments to restrict the validation.
+
+ By default all modules of pyarrow are tried to be validated.
+
+ Examples
+ --------
+ archery numpydoc pyarrow.dataset
+ archery numpydoc pyarrow.csv pyarrow.json pyarrow.parquet
+ archery numpydoc pyarrow.array
+ """
+ disallow_rule = disallow_rule or {'GL01', 'SA01', 'EX01', 'ES01'}
+ try:
+ results = python_numpydoc(symbols, allow_rules=allow_rule,
+ disallow_rules=disallow_rule)
+ for result in results:
+ result.ok()
+ except LintValidationException:
+ sys.exit(1)
+
+
+@archery.group()
+@click.pass_context
+def benchmark(ctx):
+ """ Arrow benchmarking.
+
+ Use the diff sub-command to benchmark revisions, and/or build directories.
+ """
+ pass
+
+
+def benchmark_common_options(cmd):
+ def check_language(ctx, param, value):
+ if value not in {"cpp", "java"}:
+ raise click.BadParameter("cpp or java is supported now")
+ return value
+
+ options = [
+ click.option("--src", metavar="<arrow_src>", show_default=True,
+ default=None, callback=validate_arrow_sources,
+ help="Specify Arrow source directory"),
+ click.option("--preserve", type=BOOL, default=False, show_default=True,
+ is_flag=True,
+ help="Preserve workspace for investigation."),
+ click.option("--output", metavar="<output>",
+ type=click.File("w", encoding="utf8"), default="-",
+ help="Capture output result into file."),
+ click.option("--language", metavar="<lang>", type=str, default="cpp",
+ show_default=True, callback=check_language,
+ help="Specify target language for the benchmark"),
+ click.option("--build-extras", type=str, multiple=True,
+ help="Extra flags/options to pass to mvn build. "
+ "Can be stacked. For language=java"),
+ click.option("--benchmark-extras", type=str, multiple=True,
+ help="Extra flags/options to pass to mvn benchmark. "
+ "Can be stacked. For language=java"),
+ click.option("--cmake-extras", type=str, multiple=True,
+ help="Extra flags/options to pass to cmake invocation. "
+ "Can be stacked. For language=cpp")
+ ]
+
+ cmd = java_toolchain_options(cmd)
+ cmd = cpp_toolchain_options(cmd)
+ return _apply_options(cmd, options)
+
+
+def benchmark_filter_options(cmd):
+ options = [
+ click.option("--suite-filter", metavar="<regex>", show_default=True,
+ type=str, default=None,
+ help="Regex filtering benchmark suites."),
+ click.option("--benchmark-filter", metavar="<regex>",
+ show_default=True, type=str, default=None,
+ help="Regex filtering benchmarks.")
+ ]
+ return _apply_options(cmd, options)
+
+
+@benchmark.command(name="list", short_help="List benchmark suite")
+@click.argument("rev_or_path", metavar="[<rev_or_path>]",
+ default="WORKSPACE", required=False)
+@benchmark_common_options
+@click.pass_context
+def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras,
+ java_home, java_options, build_extras, benchmark_extras,
+ language, **kwargs):
+ """ List benchmark suite.
+ """
+ with tmpdir(preserve=preserve) as root:
+ logger.debug("Running benchmark {}".format(rev_or_path))
+
+ if language == "cpp":
+ conf = CppBenchmarkRunner.default_configuration(
+ cmake_extras=cmake_extras, **kwargs)
+
+ runner_base = CppBenchmarkRunner.from_rev_or_path(
+ src, root, rev_or_path, conf)
+
+ elif language == "java":
+ for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}:
+ del kwargs[key]
+ conf = JavaBenchmarkRunner.default_configuration(
+ java_home=java_home, java_options=java_options,
+ build_extras=build_extras, benchmark_extras=benchmark_extras,
+ **kwargs)
+
+ runner_base = JavaBenchmarkRunner.from_rev_or_path(
+ src, root, rev_or_path, conf)
+
+ for b in runner_base.list_benchmarks:
+ click.echo(b, file=output)
+
+
+@benchmark.command(name="run", short_help="Run benchmark suite")
+@click.argument("rev_or_path", metavar="[<rev_or_path>]",
+ default="WORKSPACE", required=False)
+@benchmark_common_options
+@benchmark_filter_options
+@click.option("--repetitions", type=int, default=-1,
+ help=("Number of repetitions of each benchmark. Increasing "
+ "may improve result precision. "
+ "[default: 1 for cpp, 5 for java"))
+@click.pass_context
+def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras,
+ java_home, java_options, build_extras, benchmark_extras,
+ language, suite_filter, benchmark_filter, repetitions,
+ **kwargs):
+ """ Run benchmark suite.
+
+ This command will run the benchmark suite for a single build. This is
+ used to capture (and/or publish) the results.
+
+ The caller can optionally specify a target which is either a git revision
+ (commit, tag, special values like HEAD) or a cmake build directory.
+
+ When a commit is referenced, a local clone of the arrow sources (specified
+ via --src) is performed and the proper branch is created. This is done in
+ a temporary directory which can be left intact with the `--preserve` flag.
+
+ The special token "WORKSPACE" is reserved to specify the current git
+ workspace. This imply that no clone will be performed.
+
+ Examples:
+
+ \b
+ # Run the benchmarks on current git workspace
+ \b
+ archery benchmark run
+
+ \b
+ # Run the benchmarks on current previous commit
+ \b
+ archery benchmark run HEAD~1
+
+ \b
+ # Run the benchmarks on current previous commit
+ \b
+ archery benchmark run --output=run.json
+ """
+ with tmpdir(preserve=preserve) as root:
+ logger.debug("Running benchmark {}".format(rev_or_path))
+
+ if language == "cpp":
+ conf = CppBenchmarkRunner.default_configuration(
+ cmake_extras=cmake_extras, **kwargs)
+
+ repetitions = repetitions if repetitions != -1 else 1
+ runner_base = CppBenchmarkRunner.from_rev_or_path(
+ src, root, rev_or_path, conf,
+ repetitions=repetitions,
+ suite_filter=suite_filter, benchmark_filter=benchmark_filter)
+
+ elif language == "java":
+ for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}:
+ del kwargs[key]
+ conf = JavaBenchmarkRunner.default_configuration(
+ java_home=java_home, java_options=java_options,
+ build_extras=build_extras, benchmark_extras=benchmark_extras,
+ **kwargs)
+
+ repetitions = repetitions if repetitions != -1 else 5
+ runner_base = JavaBenchmarkRunner.from_rev_or_path(
+ src, root, rev_or_path, conf,
+ repetitions=repetitions,
+ benchmark_filter=benchmark_filter)
+
+ json.dump(runner_base, output, cls=JsonEncoder)
+
+
+@benchmark.command(name="diff", short_help="Compare benchmark suites")
+@benchmark_common_options
+@benchmark_filter_options
+@click.option("--threshold", type=float, default=DEFAULT_THRESHOLD,
+ show_default=True,
+ help="Regression failure threshold in percentage.")
+@click.option("--repetitions", type=int, default=1, show_default=True,
+ help=("Number of repetitions of each benchmark. Increasing "
+ "may improve result precision. "
+ "[default: 1 for cpp, 5 for java"))
+@click.option("--no-counters", type=BOOL, default=False, is_flag=True,
+ help="Hide counters field in diff report.")
+@click.argument("contender", metavar="[<contender>",
+ default=ArrowSources.WORKSPACE, required=False)
+@click.argument("baseline", metavar="[<baseline>]]", default="origin/master",
+ required=False)
+@click.pass_context
+def benchmark_diff(ctx, src, preserve, output, language, cmake_extras,
+ suite_filter, benchmark_filter, repetitions, no_counters,
+ java_home, java_options, build_extras, benchmark_extras,
+ threshold, contender, baseline, **kwargs):
+ """Compare (diff) benchmark runs.
+
+ This command acts like git-diff but for benchmark results.
+
+ The caller can optionally specify both the contender and the baseline. If
+ unspecified, the contender will default to the current workspace (like git)
+ and the baseline will default to master.
+
+ Each target (contender or baseline) can either be a git revision
+ (commit, tag, special values like HEAD) or a cmake build directory. This
+ allow comparing git commits, and/or different compilers and/or compiler
+ flags.
+
+ When a commit is referenced, a local clone of the arrow sources (specified
+ via --src) is performed and the proper branch is created. This is done in
+ a temporary directory which can be left intact with the `--preserve` flag.
+
+ The special token "WORKSPACE" is reserved to specify the current git
+ workspace. This imply that no clone will be performed.
+
+ Examples:
+
+ \b
+ # Compare workspace (contender) with master (baseline)
+ \b
+ archery benchmark diff
+
+ \b
+ # Compare master (contender) with latest version (baseline)
+ \b
+ export LAST=$(git tag -l "apache-arrow-[0-9]*" | sort -rV | head -1)
+ \b
+ archery benchmark diff master "$LAST"
+
+ \b
+ # Compare g++7 (contender) with clang++-8 (baseline) builds
+ \b
+ archery build --with-benchmarks=true \\
+ --cxx-flags=-ftree-vectorize \\
+ --cc=gcc-7 --cxx=g++-7 gcc7-build
+ \b
+ archery build --with-benchmarks=true \\
+ --cxx-flags=-flax-vector-conversions \\
+ --cc=clang-8 --cxx=clang++-8 clang8-build
+ \b
+ archery benchmark diff gcc7-build clang8-build
+
+ \b
+ # Compare default targets but scoped to the suites matching
+ # `^arrow-compute-aggregate` and benchmarks matching `(Sum|Mean)Kernel`.
+ \b
+ archery benchmark diff --suite-filter="^arrow-compute-aggregate" \\
+ --benchmark-filter="(Sum|Mean)Kernel"
+
+ \b
+ # Capture result in file `result.json`
+ \b
+ archery benchmark diff --output=result.json
+ \b
+ # Equivalently with no stdout clutter.
+ archery --quiet benchmark diff > result.json
+
+ \b
+ # Comparing with a cached results from `archery benchmark run`
+ \b
+ archery benchmark run --output=run.json HEAD~1
+ \b
+ # This should not recompute the benchmark from run.json
+ archery --quiet benchmark diff WORKSPACE run.json > result.json
+ """
+ with tmpdir(preserve=preserve) as root:
+ logger.debug("Comparing {} (contender) with {} (baseline)"
+ .format(contender, baseline))
+
+ if language == "cpp":
+ conf = CppBenchmarkRunner.default_configuration(
+ cmake_extras=cmake_extras, **kwargs)
+
+ repetitions = repetitions if repetitions != -1 else 1
+ runner_cont = CppBenchmarkRunner.from_rev_or_path(
+ src, root, contender, conf,
+ repetitions=repetitions,
+ suite_filter=suite_filter,
+ benchmark_filter=benchmark_filter)
+ runner_base = CppBenchmarkRunner.from_rev_or_path(
+ src, root, baseline, conf,
+ repetitions=repetitions,
+ suite_filter=suite_filter,
+ benchmark_filter=benchmark_filter)
+
+ elif language == "java":
+ for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}:
+ del kwargs[key]
+ conf = JavaBenchmarkRunner.default_configuration(
+ java_home=java_home, java_options=java_options,
+ build_extras=build_extras, benchmark_extras=benchmark_extras,
+ **kwargs)
+
+ repetitions = repetitions if repetitions != -1 else 5
+ runner_cont = JavaBenchmarkRunner.from_rev_or_path(
+ src, root, contender, conf,
+ repetitions=repetitions,
+ benchmark_filter=benchmark_filter)
+ runner_base = JavaBenchmarkRunner.from_rev_or_path(
+ src, root, baseline, conf,
+ repetitions=repetitions,
+ benchmark_filter=benchmark_filter)
+
+ runner_comp = RunnerComparator(runner_cont, runner_base, threshold)
+
+ # TODO(kszucs): test that the output is properly formatted jsonlines
+ comparisons_json = _get_comparisons_as_json(runner_comp.comparisons)
+ ren_counters = language == "java"
+ formatted = _format_comparisons_with_pandas(comparisons_json,
+ no_counters, ren_counters)
+ output.write(formatted)
+ output.write('\n')
+
+
+def _get_comparisons_as_json(comparisons):
+ buf = StringIO()
+ for comparator in comparisons:
+ json.dump(comparator, buf, cls=JsonEncoder)
+ buf.write("\n")
+
+ return buf.getvalue()
+
+
+def _format_comparisons_with_pandas(comparisons_json, no_counters,
+ ren_counters):
+ pd = _import_pandas()
+ df = pd.read_json(StringIO(comparisons_json), lines=True)
+ # parse change % so we can sort by it
+ df['change %'] = df.pop('change').str[:-1].map(float)
+ first_regression = len(df) - df['regression'].sum()
+
+ fields = ['benchmark', 'baseline', 'contender', 'change %']
+ if not no_counters:
+ fields += ['counters']
+
+ df = df[fields]
+ if ren_counters:
+ df = df.rename(columns={'counters': 'configurations'})
+ df = df.sort_values(by='change %', ascending=False)
+
+ def labelled(title, df):
+ if len(df) == 0:
+ return ''
+ title += ': ({})'.format(len(df))
+ df_str = df.to_string(index=False)
+ bar = '-' * df_str.index('\n')
+ return '\n'.join([bar, title, bar, df_str])
+
+ return '\n\n'.join([labelled('Non-regressions', df[:first_regression]),
+ labelled('Regressions', df[first_regression:])])
+
+
+# ----------------------------------------------------------------------
+# Integration testing
+
+def _set_default(opt, default):
+ if opt is None:
+ return default
+ return opt
+
+
+@archery.command(short_help="Execute protocol and Flight integration tests")
+@click.option('--with-all', is_flag=True, default=False,
+ help=('Include all known languages by default '
+ 'in integration tests'))
+@click.option('--random-seed', type=int, default=12345,
+ help="Seed for PRNG when generating test data")
+@click.option('--with-cpp', type=bool, default=False,
+ help='Include C++ in integration tests')
+@click.option('--with-csharp', type=bool, default=False,
+ help='Include C# in integration tests')
+@click.option('--with-java', type=bool, default=False,
+ help='Include Java in integration tests')
+@click.option('--with-js', type=bool, default=False,
+ help='Include JavaScript in integration tests')
+@click.option('--with-go', type=bool, default=False,
+ help='Include Go in integration tests')
+@click.option('--with-rust', type=bool, default=False,
+ help='Include Rust in integration tests',
+ envvar="ARCHERY_INTEGRATION_WITH_RUST")
+@click.option('--write_generated_json', default=False,
+ help='Generate test JSON to indicated path')
+@click.option('--run-flight', is_flag=True, default=False,
+ help='Run Flight integration tests')
+@click.option('--debug', is_flag=True, default=False,
+ help='Run executables in debug mode as relevant')
+@click.option('--serial', is_flag=True, default=False,
+ help='Run tests serially, rather than in parallel')
+@click.option('--tempdir', default=None,
+ help=('Directory to use for writing '
+ 'integration test temporary files'))
+@click.option('stop_on_error', '-x', '--stop-on-error',
+ is_flag=True, default=False,
+ help='Stop on first error')
+@click.option('--gold-dirs', multiple=True,
+ help="gold integration test file paths")
+@click.option('-k', '--match',
+ help=("Substring for test names to include in run, "
+ "e.g. -k primitive"))
+def integration(with_all=False, random_seed=12345, **args):
+ from .integration.runner import write_js_test_json, run_all_tests
+ import numpy as np
+
+ # FIXME(bkietz) Include help strings for individual testers.
+ # For example, CPPTester's ARROW_CPP_EXE_PATH environment variable.
+
+ # Make runs involving data generation deterministic
+ np.random.seed(random_seed)
+
+ gen_path = args['write_generated_json']
+
+ languages = ['cpp', 'csharp', 'java', 'js', 'go', 'rust']
+
+ enabled_languages = 0
+ for lang in languages:
+ param = 'with_{}'.format(lang)
+ if with_all:
+ args[param] = with_all
+
+ if args[param]:
+ enabled_languages += 1
+
+ if gen_path:
+ try:
+ os.makedirs(gen_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ write_js_test_json(gen_path)
+ else:
+ if enabled_languages == 0:
+ raise Exception("Must enable at least 1 language to test")
+ run_all_tests(**args)
+
+
+@archery.command()
+@click.option('--event-name', '-n', required=True)
+@click.option('--event-payload', '-p', type=click.File('r', encoding='utf8'),
+ default='-', required=True)
+@click.option('--arrow-token', envvar='ARROW_GITHUB_TOKEN',
+ help='OAuth token for responding comment in the arrow repo')
+def trigger_bot(event_name, event_payload, arrow_token):
+ from .bot import CommentBot, actions
+
+ event_payload = json.loads(event_payload.read())
+
+ bot = CommentBot(name='github-actions', handler=actions, token=arrow_token)
+ bot.handle(event_name, event_payload)
+
+
+@archery.group('release')
+@click.option("--src", metavar="<arrow_src>", default=None,
+ callback=validate_arrow_sources,
+ help="Specify Arrow source directory.")
+@click.option("--jira-cache", type=click.Path(), default=None,
+ help="File path to cache queried JIRA issues per version.")
+@click.pass_obj
+def release(obj, src, jira_cache):
+ """Release releated commands."""
+ from .release import Jira, CachedJira
+
+ jira = Jira()
+ if jira_cache is not None:
+ jira = CachedJira(jira_cache, jira=jira)
+
+ obj['jira'] = jira
+ obj['repo'] = src.path
+
+
+@release.command('curate')
+@click.argument('version')
+@click.pass_obj
+def release_curate(obj, version):
+ """Release curation."""
+ from .release import Release
+
+ release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo'])
+ curation = release.curate()
+
+ click.echo(curation.render('console'))
+
+
+@release.group('changelog')
+def release_changelog():
+ """Release changelog."""
+ pass
+
+
+@release_changelog.command('add')
+@click.argument('version')
+@click.pass_obj
+def release_changelog_add(obj, version):
+ """Prepend the changelog with the current release"""
+ from .release import Release
+
+ jira, repo = obj['jira'], obj['repo']
+
+ # just handle the current version
+ release = Release.from_jira(version, jira=jira, repo=repo)
+ if release.is_released:
+ raise ValueError('This version has been already released!')
+
+ changelog = release.changelog()
+ changelog_path = pathlib.Path(repo) / 'CHANGELOG.md'
+
+ current_content = changelog_path.read_text()
+ new_content = changelog.render('markdown') + current_content
+
+ changelog_path.write_text(new_content)
+ click.echo("CHANGELOG.md is updated!")
+
+
+@release_changelog.command('generate')
+@click.argument('version')
+@click.argument('output', type=click.File('w', encoding='utf8'), default='-')
+@click.pass_obj
+def release_changelog_generate(obj, version, output):
+ """Generate the changelog of a specific release."""
+ from .release import Release
+
+ jira, repo = obj['jira'], obj['repo']
+
+ # just handle the current version
+ release = Release.from_jira(version, jira=jira, repo=repo)
+
+ changelog = release.changelog()
+ output.write(changelog.render('markdown'))
+
+
+@release_changelog.command('regenerate')
+@click.pass_obj
+def release_changelog_regenerate(obj):
+ """Regeneretate the whole CHANGELOG.md file"""
+ from .release import Release
+
+ jira, repo = obj['jira'], obj['repo']
+ changelogs = []
+
+ for version in jira.project_versions('ARROW'):
+ if not version.released:
+ continue
+ release = Release.from_jira(version, jira=jira, repo=repo)
+ click.echo('Querying changelog for version: {}'.format(version))
+ changelogs.append(release.changelog())
+
+ click.echo('Rendering new CHANGELOG.md file...')
+ changelog_path = pathlib.Path(repo) / 'CHANGELOG.md'
+ with changelog_path.open('w') as fp:
+ for cl in changelogs:
+ fp.write(cl.render('markdown'))
+
+
+@release.command('cherry-pick')
+@click.argument('version')
+@click.option('--dry-run/--execute', default=True,
+ help="Display the git commands instead of executing them.")
+@click.option('--recreate/--continue', default=True,
+ help="Recreate the maintenance branch or only apply unapplied "
+ "patches.")
+@click.pass_obj
+def release_cherry_pick(obj, version, dry_run, recreate):
+ """
+ Cherry pick commits.
+ """
+ from .release import Release, MinorRelease, PatchRelease
+
+ release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo'])
+ if not isinstance(release, (MinorRelease, PatchRelease)):
+ raise click.UsageError('Cherry-pick command only supported for minor '
+ 'and patch releases')
+
+ if not dry_run:
+ release.cherry_pick_commits(recreate_branch=recreate)
+ click.echo('Executed the following commands:\n')
+
+ click.echo(
+ 'git checkout {} -b {}'.format(release.previous.tag, release.branch)
+ )
+ for commit in release.commits_to_pick():
+ click.echo('git cherry-pick {}'.format(commit.hexsha))
+
+
+@archery.group("linking")
+@click.pass_obj
+def linking(obj):
+ """
+ Quick and dirty utilities for checking library linkage.
+ """
+ pass
+
+
+@linking.command("check-dependencies")
+@click.argument("paths", nargs=-1)
+@click.option("--allow", "-a", "allowed", multiple=True,
+ help="Name of the allowed libraries")
+@click.option("--disallow", "-d", "disallowed", multiple=True,
+ help="Name of the disallowed libraries")
+@click.pass_obj
+def linking_check_dependencies(obj, allowed, disallowed, paths):
+ from .linking import check_dynamic_library_dependencies, DependencyError
+
+ allowed, disallowed = set(allowed), set(disallowed)
+ try:
+ for path in map(pathlib.Path, paths):
+ check_dynamic_library_dependencies(path, allowed=allowed,
+ disallowed=disallowed)
+ except DependencyError as e:
+ raise click.ClickException(str(e))
+
+
+add_optional_command("docker", module=".docker.cli", function="docker",
+ parent=archery)
+add_optional_command("crossbow", module=".crossbow.cli", function="crossbow",
+ parent=archery)
+
+
+if __name__ == "__main__":
+ archery(obj={})
diff --git a/src/arrow/dev/archery/archery/compat.py b/src/arrow/dev/archery/archery/compat.py
new file mode 100644
index 000000000..bb0b15428
--- /dev/null
+++ b/src/arrow/dev/archery/archery/compat.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pathlib
+import sys
+
+
+def _is_path_like(path):
+ # PEP519 filesystem path protocol is available from python 3.6, so pathlib
+ # doesn't implement __fspath__ for earlier versions
+ return (isinstance(path, str) or
+ hasattr(path, '__fspath__') or
+ isinstance(path, pathlib.Path))
+
+
+def _ensure_path(path):
+ if isinstance(path, pathlib.Path):
+ return path
+ else:
+ return pathlib.Path(_stringify_path(path))
+
+
+def _stringify_path(path):
+ """
+ Convert *path* to a string or unicode path if possible.
+ """
+ if isinstance(path, str):
+ return path
+
+ # checking whether path implements the filesystem protocol
+ try:
+ return path.__fspath__() # new in python 3.6
+ except AttributeError:
+ # fallback pathlib ckeck for earlier python versions than 3.6
+ if isinstance(path, pathlib.Path):
+ return str(path)
+
+ raise TypeError("not a path-like object")
+
+
+def _import_pandas():
+ # ARROW-13425: avoid importing PyArrow from Pandas
+ sys.modules['pyarrow'] = None
+ import pandas as pd
+ return pd
diff --git a/src/arrow/dev/archery/archery/crossbow/__init__.py b/src/arrow/dev/archery/archery/crossbow/__init__.py
new file mode 100644
index 000000000..bc72e81f0
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/__init__.py
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .core import Config, Repo, Queue, Target, Job # noqa
+from .reports import CommentReport, ConsoleReport, EmailReport # noqa
diff --git a/src/arrow/dev/archery/archery/crossbow/cli.py b/src/arrow/dev/archery/archery/crossbow/cli.py
new file mode 100644
index 000000000..1d0610343
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/cli.py
@@ -0,0 +1,365 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+
+import click
+
+from .core import Config, Repo, Queue, Target, Job, CrossbowError
+from .reports import EmailReport, ConsoleReport
+from ..utils.source import ArrowSources
+
+
+_default_arrow_path = ArrowSources.find().path
+_default_queue_path = _default_arrow_path.parent / "crossbow"
+_default_config_path = _default_arrow_path / "dev" / "tasks" / "tasks.yml"
+
+
+@click.group()
+@click.option('--github-token', '-t', default=None,
+ envvar="CROSSBOW_GITHUB_TOKEN",
+ help='OAuth token for GitHub authentication')
+@click.option('--arrow-path', '-a',
+ type=click.Path(), default=_default_arrow_path,
+ help='Arrow\'s repository path. Defaults to the repository of '
+ 'this script')
+@click.option('--queue-path', '-q',
+ type=click.Path(), default=_default_queue_path,
+ help='The repository path used for scheduling the tasks. '
+ 'Defaults to crossbow directory placed next to arrow')
+@click.option('--queue-remote', '-qr', default=None,
+ help='Force to use this remote URL for the Queue repository')
+@click.option('--output-file', metavar='<output>',
+ type=click.File('w', encoding='utf8'), default='-',
+ help='Capture output result into file.')
+@click.pass_context
+def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote,
+ output_file):
+ """
+ Schedule packaging tasks or nightly builds on CI services.
+ """
+ ctx.ensure_object(dict)
+ ctx.obj['output'] = output_file
+ ctx.obj['arrow'] = Repo(arrow_path)
+ ctx.obj['queue'] = Queue(queue_path, remote_url=queue_remote,
+ github_token=github_token, require_https=True)
+
+
+@crossbow.command()
+@click.option('--config-path', '-c',
+ type=click.Path(exists=True), default=_default_config_path,
+ help='Task configuration yml. Defaults to tasks.yml')
+@click.pass_obj
+def check_config(obj, config_path):
+ # load available tasks configuration and groups from yaml
+ config = Config.load_yaml(config_path)
+ config.validate()
+
+ output = obj['output']
+ config.show(output)
+
+
+@crossbow.command()
+@click.argument('tasks', nargs=-1, required=False)
+@click.option('--group', '-g', 'groups', multiple=True,
+ help='Submit task groups as defined in task.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+ help='Additional task parameters for rendering the CI templates')
+@click.option('--job-prefix', default='build',
+ help='Arbitrary prefix for branch names, e.g. nightly')
+@click.option('--config-path', '-c',
+ type=click.Path(exists=True), default=_default_config_path,
+ help='Task configuration yml. Defaults to tasks.yml')
+@click.option('--arrow-version', '-v', default=None,
+ help='Set target version explicitly.')
+@click.option('--arrow-remote', '-r', default=None,
+ help='Set GitHub remote explicitly, which is going to be cloned '
+ 'on the CI services. Note, that no validation happens '
+ 'locally. Examples: https://github.com/apache/arrow or '
+ 'https://github.com/kszucs/arrow.')
+@click.option('--arrow-branch', '-b', default=None,
+ help='Give the branch name explicitly, e.g. master, ARROW-1949.')
+@click.option('--arrow-sha', '-t', default=None,
+ help='Set commit SHA or Tag name explicitly, e.g. f67a515, '
+ 'apache-arrow-0.11.1.')
+@click.option('--fetch/--no-fetch', default=True,
+ help='Fetch references (branches and tags) from the remote')
+@click.option('--dry-run/--commit', default=False,
+ help='Just display the rendered CI configurations without '
+ 'committing them')
+@click.option('--no-push/--push', default=False,
+ help='Don\'t push the changes')
+@click.pass_obj
+def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version,
+ arrow_remote, arrow_branch, arrow_sha, fetch, dry_run, no_push):
+ output = obj['output']
+ queue, arrow = obj['queue'], obj['arrow']
+
+ # load available tasks configuration and groups from yaml
+ config = Config.load_yaml(config_path)
+ try:
+ config.validate()
+ except CrossbowError as e:
+ raise click.ClickException(str(e))
+
+ # Override the detected repo url / remote, branch and sha - this aims to
+ # make release procedure a bit simpler.
+ # Note, that the target resivion's crossbow templates must be
+ # compatible with the locally checked out version of crossbow (which is
+ # in case of the release procedure), because the templates still
+ # contain some business logic (dependency installation, deployments)
+ # which will be reduced to a single command in the future.
+ target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch,
+ head=arrow_sha, version=arrow_version)
+
+ # parse additional job parameters
+ params = dict([p.split("=") for p in params])
+
+ # instantiate the job object
+ try:
+ job = Job.from_config(config=config, target=target, tasks=tasks,
+ groups=groups, params=params)
+ except CrossbowError as e:
+ raise click.ClickException(str(e))
+
+ job.show(output)
+ if dry_run:
+ return
+
+ if fetch:
+ queue.fetch()
+ queue.put(job, prefix=job_prefix)
+
+ if no_push:
+ click.echo('Branches and commits created but not pushed: `{}`'
+ .format(job.branch))
+ else:
+ queue.push()
+ click.echo('Pushed job identifier is: `{}`'.format(job.branch))
+
+
+@crossbow.command()
+@click.argument('task', required=True)
+@click.option('--config-path', '-c',
+ type=click.Path(exists=True), default=_default_config_path,
+ help='Task configuration yml. Defaults to tasks.yml')
+@click.option('--arrow-version', '-v', default=None,
+ help='Set target version explicitly.')
+@click.option('--arrow-remote', '-r', default=None,
+ help='Set GitHub remote explicitly, which is going to be cloned '
+ 'on the CI services. Note, that no validation happens '
+ 'locally. Examples: https://github.com/apache/arrow or '
+ 'https://github.com/kszucs/arrow.')
+@click.option('--arrow-branch', '-b', default=None,
+ help='Give the branch name explicitly, e.g. master, ARROW-1949.')
+@click.option('--arrow-sha', '-t', default=None,
+ help='Set commit SHA or Tag name explicitly, e.g. f67a515, '
+ 'apache-arrow-0.11.1.')
+@click.option('--param', '-p', 'params', multiple=True,
+ help='Additional task parameters for rendering the CI templates')
+@click.pass_obj
+def render(obj, task, config_path, arrow_version, arrow_remote, arrow_branch,
+ arrow_sha, params):
+ """
+ Utility command to check the rendered CI templates.
+ """
+ from .core import _flatten
+
+ def highlight(code):
+ try:
+ from pygments import highlight
+ from pygments.lexers import YamlLexer
+ from pygments.formatters import TerminalFormatter
+ return highlight(code, YamlLexer(), TerminalFormatter())
+ except ImportError:
+ return code
+
+ arrow = obj['arrow']
+
+ target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch,
+ head=arrow_sha, version=arrow_version)
+ config = Config.load_yaml(config_path)
+ params = dict([p.split("=") for p in params])
+ params["queue_remote_url"] = "https://github.com/org/crossbow"
+ job = Job.from_config(config=config, target=target, tasks=[task],
+ params=params)
+
+ for task_name, rendered_files in job.render_tasks().items():
+ for path, content in _flatten(rendered_files).items():
+ click.echo('#' * 80)
+ click.echo('### {:^72} ###'.format("/".join(path)))
+ click.echo('#' * 80)
+ click.echo(highlight(content))
+
+
+@crossbow.command()
+@click.argument('job-name', required=True)
+@click.option('--fetch/--no-fetch', default=True,
+ help='Fetch references (branches and tags) from the remote')
+@click.option('--task-filter', '-f', 'task_filters', multiple=True,
+ help='Glob pattern for filtering relevant tasks')
+@click.pass_obj
+def status(obj, job_name, fetch, task_filters):
+ output = obj['output']
+ queue = obj['queue']
+ if fetch:
+ queue.fetch()
+ job = queue.get(job_name)
+
+ report = ConsoleReport(job, task_filters=task_filters)
+ report.show(output)
+
+
+@crossbow.command()
+@click.argument('prefix', required=True)
+@click.option('--fetch/--no-fetch', default=True,
+ help='Fetch references (branches and tags) from the remote')
+@click.pass_obj
+def latest_prefix(obj, prefix, fetch):
+ queue = obj['queue']
+ if fetch:
+ queue.fetch()
+ latest = queue.latest_for_prefix(prefix)
+ click.echo(latest.branch)
+
+
+@crossbow.command()
+@click.argument('job-name', required=True)
+@click.option('--sender-name', '-n',
+ help='Name to use for report e-mail.')
+@click.option('--sender-email', '-e',
+ help='E-mail to use for report e-mail.')
+@click.option('--recipient-email', '-r',
+ help='Where to send the e-mail report')
+@click.option('--smtp-user', '-u',
+ help='E-mail address to use for SMTP login')
+@click.option('--smtp-password', '-P',
+ help='SMTP password to use for report e-mail.')
+@click.option('--smtp-server', '-s', default='smtp.gmail.com',
+ help='SMTP server to use for report e-mail.')
+@click.option('--smtp-port', '-p', default=465,
+ help='SMTP port to use for report e-mail.')
+@click.option('--poll/--no-poll', default=False,
+ help='Wait for completion if there are tasks pending')
+@click.option('--poll-max-minutes', default=180,
+ help='Maximum amount of time waiting for job completion')
+@click.option('--poll-interval-minutes', default=10,
+ help='Number of minutes to wait to check job status again')
+@click.option('--send/--dry-run', default=False,
+ help='Just display the report, don\'t send it')
+@click.option('--fetch/--no-fetch', default=True,
+ help='Fetch references (branches and tags) from the remote')
+@click.pass_obj
+def report(obj, job_name, sender_name, sender_email, recipient_email,
+ smtp_user, smtp_password, smtp_server, smtp_port, poll,
+ poll_max_minutes, poll_interval_minutes, send, fetch):
+ """
+ Send an e-mail report showing success/failure of tasks in a Crossbow run
+ """
+ output = obj['output']
+ queue = obj['queue']
+ if fetch:
+ queue.fetch()
+
+ job = queue.get(job_name)
+ report = EmailReport(
+ job=job,
+ sender_name=sender_name,
+ sender_email=sender_email,
+ recipient_email=recipient_email
+ )
+
+ if poll:
+ job.wait_until_finished(
+ poll_max_minutes=poll_max_minutes,
+ poll_interval_minutes=poll_interval_minutes
+ )
+
+ if send:
+ report.send(
+ smtp_user=smtp_user,
+ smtp_password=smtp_password,
+ smtp_server=smtp_server,
+ smtp_port=smtp_port
+ )
+ else:
+ report.show(output)
+
+
+@crossbow.command()
+@click.argument('job-name', required=True)
+@click.option('-t', '--target-dir',
+ default=_default_arrow_path / 'packages',
+ type=click.Path(file_okay=False, dir_okay=True),
+ help='Directory to download the build artifacts')
+@click.option('--dry-run/--execute', default=False,
+ help='Just display process, don\'t download anything')
+@click.option('--fetch/--no-fetch', default=True,
+ help='Fetch references (branches and tags) from the remote')
+@click.option('--task-filter', '-f', 'task_filters', multiple=True,
+ help='Glob pattern for filtering relevant tasks')
+@click.option('--validate-patterns/--skip-pattern-validation', default=True,
+ help='Whether to validate artifact name patterns or not')
+@click.pass_obj
+def download_artifacts(obj, job_name, target_dir, dry_run, fetch,
+ validate_patterns, task_filters):
+ """Download build artifacts from GitHub releases"""
+ output = obj['output']
+
+ # fetch the queue repository
+ queue = obj['queue']
+ if fetch:
+ queue.fetch()
+
+ # query the job's artifacts
+ job = queue.get(job_name)
+
+ # create directory to download the assets to
+ target_dir = Path(target_dir).absolute() / job_name
+ target_dir.mkdir(parents=True, exist_ok=True)
+
+ # download the assets while showing the job status
+ def asset_callback(task_name, task, asset):
+ if asset is not None:
+ path = target_dir / task_name / asset.name
+ path.parent.mkdir(exist_ok=True)
+ if not dry_run:
+ asset.download(path)
+
+ click.echo('Downloading {}\'s artifacts.'.format(job_name))
+ click.echo('Destination directory is {}'.format(target_dir))
+ click.echo()
+
+ report = ConsoleReport(job, task_filters=task_filters)
+ report.show(
+ output,
+ asset_callback=asset_callback,
+ validate_patterns=validate_patterns
+ )
+
+
+@crossbow.command()
+@click.argument('patterns', nargs=-1, required=True)
+@click.option('--sha', required=True, help='Target committish')
+@click.option('--tag', required=True, help='Target tag')
+@click.option('--method', default='curl', help='Use cURL to upload')
+@click.pass_obj
+def upload_artifacts(obj, tag, sha, patterns, method):
+ queue = obj['queue']
+ queue.github_overwrite_release_assets(
+ tag_name=tag, target_commitish=sha, method=method, patterns=patterns
+ )
diff --git a/src/arrow/dev/archery/archery/crossbow/core.py b/src/arrow/dev/archery/archery/crossbow/core.py
new file mode 100644
index 000000000..0f2309e47
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/core.py
@@ -0,0 +1,1172 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import fnmatch
+import glob
+import time
+import logging
+import mimetypes
+import subprocess
+import textwrap
+from io import StringIO
+from pathlib import Path
+from datetime import date
+
+import jinja2
+from ruamel.yaml import YAML
+
+try:
+ import github3
+ _have_github3 = True
+except ImportError:
+ github3 = object
+ _have_github3 = False
+
+try:
+ import pygit2
+except ImportError:
+ PygitRemoteCallbacks = object
+else:
+ PygitRemoteCallbacks = pygit2.RemoteCallbacks
+
+from ..utils.source import ArrowSources
+
+
+for pkg in ["requests", "urllib3", "github3"]:
+ logging.getLogger(pkg).setLevel(logging.WARNING)
+
+logger = logging.getLogger("crossbow")
+
+
+class CrossbowError(Exception):
+ pass
+
+
+def _flatten(mapping):
+ """Converts a hierarchical mapping to a flat dictionary"""
+ result = {}
+ for k, v in mapping.items():
+ if isinstance(v, dict):
+ for ik, iv in _flatten(v).items():
+ ik = ik if isinstance(ik, tuple) else (ik,)
+ result[(k,) + ik] = iv
+ elif isinstance(v, list):
+ for ik, iv in enumerate(_flatten(v)):
+ ik = ik if isinstance(ik, tuple) else (ik,)
+ result[(k,) + ik] = iv
+ else:
+ result[(k,)] = v
+ return result
+
+
+def _unflatten(mapping):
+ """Converts a flat tuple => object mapping to hierarchical one"""
+ result = {}
+ for path, value in mapping.items():
+ parents, leaf = path[:-1], path[-1]
+ # create the hierarchy until we reach the leaf value
+ temp = result
+ for parent in parents:
+ temp.setdefault(parent, {})
+ temp = temp[parent]
+ # set the leaf value
+ temp[leaf] = value
+
+ return result
+
+
+def _unflatten_tree(files):
+ """Converts a flat path => object mapping to a hierarchical directories
+
+ Input:
+ {
+ 'path/to/file.a': a_content,
+ 'path/to/file.b': b_content,
+ 'path/file.c': c_content
+ }
+ Output:
+ {
+ 'path': {
+ 'to': {
+ 'file.a': a_content,
+ 'file.b': b_content
+ },
+ 'file.c': c_content
+ }
+ }
+ """
+ files = {tuple(k.split('/')): v for k, v in files.items()}
+ return _unflatten(files)
+
+
+def _render_jinja_template(searchpath, template, params):
+ def format_all(items, pattern):
+ return [pattern.format(item) for item in items]
+
+ loader = jinja2.FileSystemLoader(searchpath)
+ env = jinja2.Environment(loader=loader, trim_blocks=True,
+ lstrip_blocks=True,
+ undefined=jinja2.StrictUndefined)
+ env.filters['format_all'] = format_all
+ template = env.get_template(template)
+ return template.render(**params)
+
+
+# configurations for setting up branch skipping
+# - appveyor has a feature to skip builds without an appveyor.yml
+# - travis reads from the master branch and applies the rules
+# - circle requires the configuration to be present on all branch, even ones
+# that are configured to be skipped
+# - azure skips branches without azure-pipelines.yml by default
+# - github skips branches without .github/workflows/ by default
+
+_default_travis_yml = """
+branches:
+ only:
+ - master
+ - /.*-travis-.*/
+
+os: linux
+dist: trusty
+language: generic
+"""
+
+_default_circle_yml = """
+version: 2
+
+jobs:
+ build:
+ machine: true
+
+workflows:
+ version: 2
+ build:
+ jobs:
+ - build:
+ filters:
+ branches:
+ only:
+ - /.*-circle-.*/
+"""
+
+_default_tree = {
+ '.travis.yml': _default_travis_yml,
+ '.circleci/config.yml': _default_circle_yml
+}
+
+
+class GitRemoteCallbacks(PygitRemoteCallbacks):
+
+ def __init__(self, token):
+ self.token = token
+ self.attempts = 0
+ super().__init__()
+
+ def push_update_reference(self, refname, message):
+ pass
+
+ def update_tips(self, refname, old, new):
+ pass
+
+ def credentials(self, url, username_from_url, allowed_types):
+ # its a libgit2 bug, that it infinitely retries the authentication
+ self.attempts += 1
+
+ if self.attempts >= 5:
+ # pygit2 doesn't propagate the exception properly
+ msg = 'Wrong oauth personal access token'
+ print(msg)
+ raise CrossbowError(msg)
+
+ if (allowed_types &
+ pygit2.credentials.GIT_CREDENTIAL_USERPASS_PLAINTEXT):
+ return pygit2.UserPass(self.token, 'x-oauth-basic')
+ else:
+ return None
+
+
+def _git_ssh_to_https(url):
+ return url.replace('git@github.com:', 'https://github.com/')
+
+
+class Repo:
+ """
+ Base class for interaction with local git repositories
+
+ A high level wrapper used for both reading revision information from
+ arrow's repository and pushing continuous integration tasks to the queue
+ repository.
+
+ Parameters
+ ----------
+ require_https : boolean, default False
+ Raise exception for SSH origin URLs
+ """
+
+ def __init__(self, path, github_token=None, remote_url=None,
+ require_https=False):
+ self.path = Path(path)
+ self.github_token = github_token
+ self.require_https = require_https
+ self._remote_url = remote_url
+ self._pygit_repo = None
+ self._github_repo = None # set by as_github_repo()
+ self._updated_refs = []
+
+ def __str__(self):
+ tpl = textwrap.dedent('''
+ Repo: {remote}@{branch}
+ Commit: {head}
+ ''')
+ return tpl.format(
+ remote=self.remote_url,
+ branch=self.branch.branch_name,
+ head=self.head
+ )
+
+ @property
+ def repo(self):
+ if self._pygit_repo is None:
+ self._pygit_repo = pygit2.Repository(str(self.path))
+ return self._pygit_repo
+
+ @property
+ def origin(self):
+ remote = self.repo.remotes['origin']
+ if self.require_https and remote.url.startswith('git@github.com'):
+ raise CrossbowError("Change SSH origin URL to HTTPS to use "
+ "Crossbow: {}".format(remote.url))
+ return remote
+
+ def fetch(self):
+ refspec = '+refs/heads/*:refs/remotes/origin/*'
+ self.origin.fetch([refspec])
+
+ def push(self, refs=None, github_token=None):
+ github_token = github_token or self.github_token
+ if github_token is None:
+ raise RuntimeError(
+ 'Could not determine GitHub token. Please set the '
+ 'CROSSBOW_GITHUB_TOKEN environment variable to a '
+ 'valid GitHub access token or pass one to --github-token.'
+ )
+ callbacks = GitRemoteCallbacks(github_token)
+ refs = refs or []
+ try:
+ self.origin.push(refs + self._updated_refs, callbacks=callbacks)
+ except pygit2.GitError:
+ raise RuntimeError('Failed to push updated references, '
+ 'potentially because of credential issues: {}'
+ .format(self._updated_refs))
+ else:
+ self.updated_refs = []
+
+ @property
+ def head(self):
+ """Currently checked out commit's sha"""
+ return self.repo.head
+
+ @property
+ def branch(self):
+ """Currently checked out branch"""
+ try:
+ return self.repo.branches[self.repo.head.shorthand]
+ except KeyError:
+ return None # detached
+
+ @property
+ def remote(self):
+ """Currently checked out branch's remote counterpart"""
+ try:
+ return self.repo.remotes[self.branch.upstream.remote_name]
+ except (AttributeError, KeyError):
+ return None # cannot detect
+
+ @property
+ def remote_url(self):
+ """Currently checked out branch's remote counterpart URL
+
+ If an SSH github url is set, it will be replaced by the https
+ equivalent usable with GitHub OAuth token.
+ """
+ try:
+ return self._remote_url or _git_ssh_to_https(self.remote.url)
+ except AttributeError:
+ return None
+
+ @property
+ def user_name(self):
+ try:
+ return next(self.repo.config.get_multivar('user.name'))
+ except StopIteration:
+ return os.environ.get('GIT_COMMITTER_NAME', 'unknown')
+
+ @property
+ def user_email(self):
+ try:
+ return next(self.repo.config.get_multivar('user.email'))
+ except StopIteration:
+ return os.environ.get('GIT_COMMITTER_EMAIL', 'unknown')
+
+ @property
+ def signature(self):
+ return pygit2.Signature(self.user_name, self.user_email,
+ int(time.time()))
+
+ def create_tree(self, files):
+ builder = self.repo.TreeBuilder()
+
+ for filename, content in files.items():
+ if isinstance(content, dict):
+ # create a subtree
+ tree_id = self.create_tree(content)
+ builder.insert(filename, tree_id, pygit2.GIT_FILEMODE_TREE)
+ else:
+ # create a file
+ blob_id = self.repo.create_blob(content)
+ builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB)
+
+ tree_id = builder.write()
+ return tree_id
+
+ def create_commit(self, files, parents=None, message='',
+ reference_name=None):
+ if parents is None:
+ # by default use the main branch as the base of the new branch
+ # required to reuse github actions cache across crossbow tasks
+ commit, _ = self.repo.resolve_refish("master")
+ parents = [commit.id]
+ tree_id = self.create_tree(files)
+
+ author = committer = self.signature
+ commit_id = self.repo.create_commit(reference_name, author, committer,
+ message, tree_id, parents)
+ return self.repo[commit_id]
+
+ def create_branch(self, branch_name, files, parents=None, message='',
+ signature=None):
+ # create commit with the passed tree
+ commit = self.create_commit(files, parents=parents, message=message)
+
+ # create branch pointing to the previously created commit
+ branch = self.repo.create_branch(branch_name, commit)
+
+ # append to the pushable references
+ self._updated_refs.append('refs/heads/{}'.format(branch_name))
+
+ return branch
+
+ def create_tag(self, tag_name, commit_id, message=''):
+ tag_id = self.repo.create_tag(tag_name, commit_id,
+ pygit2.GIT_OBJ_COMMIT, self.signature,
+ message)
+
+ # append to the pushable references
+ self._updated_refs.append('refs/tags/{}'.format(tag_name))
+
+ return self.repo[tag_id]
+
+ def file_contents(self, commit_id, file):
+ commit = self.repo[commit_id]
+ entry = commit.tree[file]
+ blob = self.repo[entry.id]
+ return blob.data
+
+ def _parse_github_user_repo(self):
+ m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url)
+ if m is None:
+ raise CrossbowError(
+ "Unable to parse the github owner and repository from the "
+ "repository's remote url '{}'".format(self.remote_url)
+ )
+ user, repo = m.group(1), m.group(2)
+ return user, repo
+
+ def as_github_repo(self, github_token=None):
+ """Converts it to a repository object which wraps the GitHub API"""
+ if self._github_repo is None:
+ if not _have_github3:
+ raise ImportError('Must install github3.py')
+ github_token = github_token or self.github_token
+ username, reponame = self._parse_github_user_repo()
+ session = github3.session.GitHubSession(
+ default_connect_timeout=10,
+ default_read_timeout=30
+ )
+ github = github3.GitHub(session=session)
+ github.login(token=github_token)
+ self._github_repo = github.repository(username, reponame)
+ return self._github_repo
+
+ def github_commit(self, sha):
+ repo = self.as_github_repo()
+ return repo.commit(sha)
+
+ def github_release(self, tag):
+ repo = self.as_github_repo()
+ try:
+ return repo.release_from_tag(tag)
+ except github3.exceptions.NotFoundError:
+ return None
+
+ def github_upload_asset_requests(self, release, path, name, mime,
+ max_retries=None, retry_backoff=None):
+ if max_retries is None:
+ max_retries = int(os.environ.get('CROSSBOW_MAX_RETRIES', 8))
+ if retry_backoff is None:
+ retry_backoff = int(os.environ.get('CROSSBOW_RETRY_BACKOFF', 5))
+
+ for i in range(max_retries):
+ try:
+ with open(path, 'rb') as fp:
+ result = release.upload_asset(name=name, asset=fp,
+ content_type=mime)
+ except github3.exceptions.ResponseError as e:
+ logger.error('Attempt {} has failed with message: {}.'
+ .format(i + 1, str(e)))
+ logger.error('Error message {}'.format(e.msg))
+ logger.error('List of errors provided by Github:')
+ for err in e.errors:
+ logger.error(' - {}'.format(err))
+
+ if e.code == 422:
+ # 422 Validation Failed, probably raised because
+ # ReleaseAsset already exists, so try to remove it before
+ # reattempting the asset upload
+ for asset in release.assets():
+ if asset.name == name:
+ logger.info('Release asset {} already exists, '
+ 'removing it...'.format(name))
+ asset.delete()
+ logger.info('Asset {} removed.'.format(name))
+ break
+ except github3.exceptions.ConnectionError as e:
+ logger.error('Attempt {} has failed with message: {}.'
+ .format(i + 1, str(e)))
+ else:
+ logger.info('Attempt {} has finished.'.format(i + 1))
+ return result
+
+ time.sleep(retry_backoff)
+
+ raise RuntimeError('Github asset uploading has failed!')
+
+ def github_upload_asset_curl(self, release, path, name, mime):
+ upload_url, _ = release.upload_url.split('{?')
+ upload_url += '?name={}'.format(name)
+
+ command = [
+ 'curl',
+ '--fail',
+ '-H', "Authorization: token {}".format(self.github_token),
+ '-H', "Content-Type: {}".format(mime),
+ '--data-binary', '@{}'.format(path),
+ upload_url
+ ]
+ return subprocess.run(command, shell=False, check=True)
+
+ def github_overwrite_release_assets(self, tag_name, target_commitish,
+ patterns, method='requests'):
+ # Since github has changed something the asset uploading via requests
+ # got instable, so prefer the cURL alternative.
+ # Potential cause:
+ # sigmavirus24/github3.py/issues/779#issuecomment-379470626
+ repo = self.as_github_repo()
+ if not tag_name:
+ raise CrossbowError('Empty tag name')
+ if not target_commitish:
+ raise CrossbowError('Empty target commit for the release tag')
+
+ # remove the whole release if it already exists
+ try:
+ release = repo.release_from_tag(tag_name)
+ except github3.exceptions.NotFoundError:
+ pass
+ else:
+ release.delete()
+
+ release = repo.create_release(tag_name, target_commitish)
+ for pattern in patterns:
+ for path in glob.glob(pattern, recursive=True):
+ name = os.path.basename(path)
+ size = os.path.getsize(path)
+ mime = mimetypes.guess_type(name)[0] or 'application/zip'
+
+ logger.info(
+ 'Uploading asset `{}` with mimetype {} and size {}...'
+ .format(name, mime, size)
+ )
+
+ if method == 'requests':
+ self.github_upload_asset_requests(release, path, name=name,
+ mime=mime)
+ elif method == 'curl':
+ self.github_upload_asset_curl(release, path, name=name,
+ mime=mime)
+ else:
+ raise CrossbowError(
+ 'Unsupported upload method {}'.format(method)
+ )
+
+
+class Queue(Repo):
+
+ def _latest_prefix_id(self, prefix):
+ pattern = re.compile(r'[\w\/-]*{}-(\d+)'.format(prefix))
+ matches = list(filter(None, map(pattern.match, self.repo.branches)))
+ if matches:
+ latest = max(int(m.group(1)) for m in matches)
+ else:
+ latest = -1
+ return latest
+
+ def _next_job_id(self, prefix):
+ """Auto increments the branch's identifier based on the prefix"""
+ latest_id = self._latest_prefix_id(prefix)
+ return '{}-{}'.format(prefix, latest_id + 1)
+
+ def latest_for_prefix(self, prefix):
+ latest_id = self._latest_prefix_id(prefix)
+ if latest_id < 0:
+ raise RuntimeError(
+ 'No job has been submitted with prefix {} yet'.format(prefix)
+ )
+ job_name = '{}-{}'.format(prefix, latest_id)
+ return self.get(job_name)
+
+ def date_of(self, job):
+ # it'd be better to bound to the queue repository on deserialization
+ # and reorganize these methods to Job
+ branch_name = 'origin/{}'.format(job.branch)
+ branch = self.repo.branches[branch_name]
+ commit = self.repo[branch.target]
+ return date.fromtimestamp(commit.commit_time)
+
+ def jobs(self, pattern):
+ """Return jobs sorted by its identifier in reverse order"""
+ job_names = []
+ for name in self.repo.branches.remote:
+ origin, name = name.split('/', 1)
+ result = re.match(pattern, name)
+ if result:
+ job_names.append(name)
+
+ for name in sorted(job_names, reverse=True):
+ yield self.get(name)
+
+ def get(self, job_name):
+ branch_name = 'origin/{}'.format(job_name)
+ branch = self.repo.branches[branch_name]
+ try:
+ content = self.file_contents(branch.target, 'job.yml')
+ except KeyError:
+ raise CrossbowError(
+ 'No job is found with name: {}'.format(job_name)
+ )
+
+ buffer = StringIO(content.decode('utf-8'))
+ job = yaml.load(buffer)
+ job.queue = self
+ return job
+
+ def put(self, job, prefix='build'):
+ if not isinstance(job, Job):
+ raise CrossbowError('`job` must be an instance of Job')
+ if job.branch is not None:
+ raise CrossbowError('`job.branch` is automatically generated, '
+ 'thus it must be blank')
+
+ if job.target.remote is None:
+ raise CrossbowError(
+ 'Cannot determine git remote for the Arrow repository to '
+ 'clone or push to, try to push the `{}` branch first to have '
+ 'a remote tracking counterpart.'.format(job.target.branch)
+ )
+ if job.target.branch is None:
+ raise CrossbowError(
+ 'Cannot determine the current branch of the Arrow repository '
+ 'to clone or push to, perhaps it is in detached HEAD state. '
+ 'Please checkout a branch.'
+ )
+
+ # auto increment and set next job id, e.g. build-85
+ job._queue = self
+ job.branch = self._next_job_id(prefix)
+
+ # create tasks' branches
+ for task_name, task in job.tasks.items():
+ # adding CI's name to the end of the branch in order to use skip
+ # patterns on travis and circleci
+ task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
+ params = {
+ **job.params,
+ "arrow": job.target,
+ "queue_remote_url": self.remote_url
+ }
+ files = task.render_files(job.template_searchpath, params=params)
+ branch = self.create_branch(task.branch, files=files)
+ self.create_tag(task.tag, branch.target)
+ task.commit = str(branch.target)
+
+ # create job's branch with its description
+ return self.create_branch(job.branch, files=job.render_files())
+
+
+def get_version(root, **kwargs):
+ """
+ Parse function for setuptools_scm that ignores tags for non-C++
+ subprojects, e.g. apache-arrow-js-XXX tags.
+ """
+ from setuptools_scm.git import parse as parse_git_version
+
+ # query the calculated version based on the git tags
+ kwargs['describe_command'] = (
+ 'git describe --dirty --tags --long --match "apache-arrow-[0-9].*"'
+ )
+ version = parse_git_version(root, **kwargs)
+ tag = str(version.tag)
+
+ # We may get a development tag for the next version, such as "5.0.0.dev0",
+ # or the tag of an already released version, such as "4.0.0".
+ # In the latter case, we need to increment the version so that the computed
+ # version comes after any patch release (the next feature version after
+ # 4.0.0 is 5.0.0).
+ pattern = r"^(\d+)\.(\d+)\.(\d+)"
+ match = re.match(pattern, tag)
+ major, minor, patch = map(int, match.groups())
+ if 'dev' not in tag:
+ major += 1
+
+ return "{}.{}.{}.dev{}".format(major, minor, patch, version.distance)
+
+
+class Serializable:
+
+ @classmethod
+ def to_yaml(cls, representer, data):
+ tag = '!{}'.format(cls.__name__)
+ dct = {k: v for k, v in data.__dict__.items() if not k.startswith('_')}
+ return representer.represent_mapping(tag, dct)
+
+
+class Target(Serializable):
+ """
+ Describes target repository and revision the builds run against
+
+ This serializable data container holding information about arrow's
+ git remote, branch, sha and version number as well as some metadata
+ (currently only an email address where the notification should be sent).
+ """
+
+ def __init__(self, head, branch, remote, version, email=None):
+ self.head = head
+ self.email = email
+ self.branch = branch
+ self.remote = remote
+ self.version = version
+ self.no_rc_version = re.sub(r'-rc\d+\Z', '', version)
+ # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html
+ #
+ # > A pre-release version number MAY be denoted by appending an
+ # > arbitrary string immediately following the patch version and a
+ # > dash. The string MUST be comprised of only alphanumerics plus
+ # > dash [0-9A-Za-z-].
+ #
+ # Example:
+ #
+ # '0.16.1.dev10' ->
+ # '0.16.1-dev10'
+ self.no_rc_semver_version = \
+ re.sub(r'\.(dev\d+)\Z', r'-\1', self.no_rc_version)
+
+ @classmethod
+ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None,
+ email=None):
+ """Initialize from a repository
+
+ Optionally override detected remote, branch, head, and/or version.
+ """
+ assert isinstance(repo, Repo)
+
+ if head is None:
+ head = str(repo.head.target)
+ if branch is None:
+ branch = repo.branch.branch_name
+ if remote is None:
+ remote = repo.remote_url
+ if version is None:
+ version = get_version(repo.path)
+ if email is None:
+ email = repo.user_email
+
+ return cls(head=head, email=email, branch=branch, remote=remote,
+ version=version)
+
+
+class Task(Serializable):
+ """
+ Describes a build task and metadata required to render CI templates
+
+ A task is represented as a single git commit and branch containing jinja2
+ rendered files (currently appveyor.yml or .travis.yml configurations).
+
+ A task can't be directly submitted to a queue, must belong to a job.
+ Each task's unique identifier is its branch name, which is generated after
+ submitting the job to a queue.
+ """
+
+ def __init__(self, ci, template, artifacts=None, params=None):
+ assert ci in {
+ 'circle',
+ 'travis',
+ 'appveyor',
+ 'azure',
+ 'github',
+ 'drone',
+ }
+ self.ci = ci
+ self.template = template
+ self.artifacts = artifacts or []
+ self.params = params or {}
+ self.branch = None # filled after adding to a queue
+ self.commit = None # filled after adding to a queue
+ self._queue = None # set by the queue object after put or get
+ self._status = None # status cache
+ self._assets = None # assets cache
+
+ def render_files(self, searchpath, params=None):
+ params = {**self.params, **(params or {}), "task": self}
+ try:
+ rendered = _render_jinja_template(searchpath, self.template,
+ params=params)
+ except jinja2.TemplateError as e:
+ raise RuntimeError(
+ 'Failed to render template `{}` with {}: {}'.format(
+ self.template, e.__class__.__name__, str(e)
+ )
+ )
+
+ tree = {**_default_tree, self.filename: rendered}
+ return _unflatten_tree(tree)
+
+ @property
+ def tag(self):
+ return self.branch
+
+ @property
+ def filename(self):
+ config_files = {
+ 'circle': '.circleci/config.yml',
+ 'travis': '.travis.yml',
+ 'appveyor': 'appveyor.yml',
+ 'azure': 'azure-pipelines.yml',
+ 'github': '.github/workflows/crossbow.yml',
+ 'drone': '.drone.yml',
+ }
+ return config_files[self.ci]
+
+ def status(self, force_query=False):
+ _status = getattr(self, '_status', None)
+ if force_query or _status is None:
+ github_commit = self._queue.github_commit(self.commit)
+ self._status = TaskStatus(github_commit)
+ return self._status
+
+ def assets(self, force_query=False, validate_patterns=True):
+ _assets = getattr(self, '_assets', None)
+ if force_query or _assets is None:
+ github_release = self._queue.github_release(self.tag)
+ self._assets = TaskAssets(github_release,
+ artifact_patterns=self.artifacts,
+ validate_patterns=validate_patterns)
+ return self._assets
+
+
+class TaskStatus:
+ """
+ Combine the results from status and checks API to a single state.
+
+ Azure pipelines uses checks API which doesn't provide a combined
+ interface like status API does, so we need to manually combine
+ both the commit statuses and the commit checks coming from
+ different API endpoint
+
+ Status.state: error, failure, pending or success, default pending
+ CheckRun.status: queued, in_progress or completed, default: queued
+ CheckRun.conclusion: success, failure, neutral, cancelled, timed_out
+ or action_required, only set if
+ CheckRun.status == 'completed'
+
+ 1. Convert CheckRun's status and conclusion to one of Status.state
+ 2. Merge the states based on the following rules:
+ - failure if any of the contexts report as error or failure
+ - pending if there are no statuses or a context is pending
+ - success if the latest status for all contexts is success
+ error otherwise.
+
+ Parameters
+ ----------
+ commit : github3.Commit
+ Commit to query the combined status for.
+
+ Returns
+ -------
+ TaskStatus(
+ combined_state='error|failure|pending|success',
+ github_status='original github status object',
+ github_check_runs='github checks associated with the commit',
+ total_count='number of statuses and checks'
+ )
+ """
+
+ def __init__(self, commit):
+ status = commit.status()
+ check_runs = list(commit.check_runs())
+ states = [s.state for s in status.statuses]
+
+ for check in check_runs:
+ if check.status == 'completed':
+ if check.conclusion in {'success', 'failure'}:
+ states.append(check.conclusion)
+ elif check.conclusion in {'cancelled', 'timed_out',
+ 'action_required'}:
+ states.append('error')
+ # omit `neutral` conclusion
+ else:
+ states.append('pending')
+
+ # it could be more effective, but the following is more descriptive
+ combined_state = 'error'
+ if len(states):
+ if any(state in {'error', 'failure'} for state in states):
+ combined_state = 'failure'
+ elif any(state == 'pending' for state in states):
+ combined_state = 'pending'
+ elif all(state == 'success' for state in states):
+ combined_state = 'success'
+
+ # show link to the actual build, some of the CI providers implement
+ # the statuses API others implement the checks API, so display both
+ build_links = [s.target_url for s in status.statuses]
+ build_links += [c.html_url for c in check_runs]
+
+ self.combined_state = combined_state
+ self.github_status = status
+ self.github_check_runs = check_runs
+ self.total_count = len(states)
+ self.build_links = build_links
+
+
+class TaskAssets(dict):
+
+ def __init__(self, github_release, artifact_patterns,
+ validate_patterns=True):
+ # HACK(kszucs): don't expect uploaded assets of no atifacts were
+ # defiened for the tasks in order to spare a bit of github rate limit
+ if not artifact_patterns:
+ return
+
+ if github_release is None:
+ github_assets = {} # no assets have been uploaded for the task
+ else:
+ github_assets = {a.name: a for a in github_release.assets()}
+
+ if not validate_patterns:
+ # shortcut to avoid pattern validation and just set all artifacts
+ return self.update(github_assets)
+
+ for pattern in artifact_patterns:
+ # artifact can be a regex pattern
+ compiled = re.compile(f"^{pattern}$")
+ matches = list(
+ filter(None, map(compiled.match, github_assets.keys()))
+ )
+ num_matches = len(matches)
+
+ # validate artifact pattern matches single asset
+ if num_matches == 0:
+ self[pattern] = None
+ elif num_matches == 1:
+ self[pattern] = github_assets[matches[0].group(0)]
+ else:
+ raise CrossbowError(
+ 'Only a single asset should match pattern `{}`, there are '
+ 'multiple ones: {}'.format(pattern, ', '.join(matches))
+ )
+
+ def missing_patterns(self):
+ return [pattern for pattern, asset in self.items() if asset is None]
+
+ def uploaded_assets(self):
+ return [asset for asset in self.values() if asset is not None]
+
+
+class Job(Serializable):
+ """Describes multiple tasks against a single target repository"""
+
+ def __init__(self, target, tasks, params=None, template_searchpath=None):
+ if not tasks:
+ raise ValueError('no tasks were provided for the job')
+ if not all(isinstance(task, Task) for task in tasks.values()):
+ raise ValueError('each `tasks` mus be an instance of Task')
+ if not isinstance(target, Target):
+ raise ValueError('`target` must be an instance of Target')
+ if not isinstance(target, Target):
+ raise ValueError('`target` must be an instance of Target')
+ if not isinstance(params, dict):
+ raise ValueError('`params` must be an instance of dict')
+
+ self.target = target
+ self.tasks = tasks
+ self.params = params or {} # additional parameters for the tasks
+ self.branch = None # filled after adding to a queue
+ self._queue = None # set by the queue object after put or get
+ if template_searchpath is None:
+ self._template_searchpath = ArrowSources.find().path
+ else:
+ self._template_searchpath = template_searchpath
+
+ def render_files(self):
+ with StringIO() as buf:
+ yaml.dump(self, buf)
+ content = buf.getvalue()
+ tree = {**_default_tree, "job.yml": content}
+ return _unflatten_tree(tree)
+
+ def render_tasks(self, params=None):
+ result = {}
+ params = {
+ **self.params,
+ "arrow": self.target,
+ **(params or {})
+ }
+ for task_name, task in self.tasks.items():
+ files = task.render_files(self._template_searchpath, params)
+ result[task_name] = files
+ return result
+
+ @property
+ def template_searchpath(self):
+ return self._template_searchpath
+
+ @property
+ def queue(self):
+ assert isinstance(self._queue, Queue)
+ return self._queue
+
+ @queue.setter
+ def queue(self, queue):
+ assert isinstance(queue, Queue)
+ self._queue = queue
+ for task in self.tasks.values():
+ task._queue = queue
+
+ @property
+ def email(self):
+ return os.environ.get('CROSSBOW_EMAIL', self.target.email)
+
+ @property
+ def date(self):
+ return self.queue.date_of(self)
+
+ def show(self, stream=None):
+ return yaml.dump(self, stream=stream)
+
+ @classmethod
+ def from_config(cls, config, target, tasks=None, groups=None, params=None):
+ """
+ Intantiate a job from based on a config.
+
+ Parameters
+ ----------
+ config : dict
+ Deserialized content of tasks.yml
+ target : Target
+ Describes target repository and revision the builds run against.
+ tasks : Optional[List[str]], default None
+ List of glob patterns for matching task names.
+ groups : Optional[List[str]], default None
+ List of exact group names matching predefined task sets in the
+ config.
+ params : Optional[Dict[str, str]], default None
+ Additional rendering parameters for the task templates.
+
+ Returns
+ -------
+ Job
+
+ Raises
+ ------
+ Exception:
+ If invalid groups or tasks has been passed.
+ """
+ task_definitions = config.select(tasks, groups=groups)
+
+ # instantiate the tasks
+ tasks = {}
+ versions = {'version': target.version,
+ 'no_rc_version': target.no_rc_version,
+ 'no_rc_semver_version': target.no_rc_semver_version}
+ for task_name, task in task_definitions.items():
+ artifacts = task.pop('artifacts', None) or [] # because of yaml
+ artifacts = [fn.format(**versions) for fn in artifacts]
+ tasks[task_name] = Task(artifacts=artifacts, **task)
+
+ return cls(target=target, tasks=tasks, params=params,
+ template_searchpath=config.template_searchpath)
+
+ def is_finished(self):
+ for task in self.tasks.values():
+ status = task.status(force_query=True)
+ if status.combined_state == 'pending':
+ return False
+ return True
+
+ def wait_until_finished(self, poll_max_minutes=120,
+ poll_interval_minutes=10):
+ started_at = time.time()
+ while True:
+ if self.is_finished():
+ break
+
+ waited_for_minutes = (time.time() - started_at) / 60
+ if waited_for_minutes > poll_max_minutes:
+ msg = ('Exceeded the maximum amount of time waiting for job '
+ 'to finish, waited for {} minutes.')
+ raise RuntimeError(msg.format(waited_for_minutes))
+
+ logger.info('Waiting {} minutes and then checking again'
+ .format(poll_interval_minutes))
+ time.sleep(poll_interval_minutes * 60)
+
+
+class Config(dict):
+
+ def __init__(self, tasks, template_searchpath):
+ super().__init__(tasks)
+ self.template_searchpath = template_searchpath
+
+ @classmethod
+ def load_yaml(cls, path):
+ path = Path(path)
+ searchpath = path.parent
+ rendered = _render_jinja_template(searchpath, template=path.name,
+ params={})
+ config = yaml.load(rendered)
+ return cls(config, template_searchpath=searchpath)
+
+ def show(self, stream=None):
+ return yaml.dump(dict(self), stream=stream)
+
+ def select(self, tasks=None, groups=None):
+ config_groups = dict(self['groups'])
+ config_tasks = dict(self['tasks'])
+ valid_groups = set(config_groups.keys())
+ valid_tasks = set(config_tasks.keys())
+ group_whitelist = list(groups or [])
+ task_whitelist = list(tasks or [])
+
+ # validate that the passed groups are defined in the config
+ requested_groups = set(group_whitelist)
+ invalid_groups = requested_groups - valid_groups
+ if invalid_groups:
+ msg = 'Invalid group(s) {!r}. Must be one of {!r}'.format(
+ invalid_groups, valid_groups
+ )
+ raise CrossbowError(msg)
+
+ # merge the tasks defined in the selected groups
+ task_patterns = [list(config_groups[name]) for name in group_whitelist]
+ task_patterns = set(sum(task_patterns, task_whitelist))
+
+ # treat the task names as glob patterns to select tasks more easily
+ requested_tasks = set()
+ for pattern in task_patterns:
+ matches = fnmatch.filter(valid_tasks, pattern)
+ if len(matches):
+ requested_tasks.update(matches)
+ else:
+ raise CrossbowError(
+ "Unable to match any tasks for `{}`".format(pattern)
+ )
+
+ # validate that the passed and matched tasks are defined in the config
+ invalid_tasks = requested_tasks - valid_tasks
+ if invalid_tasks:
+ msg = 'Invalid task(s) {!r}. Must be one of {!r}'.format(
+ invalid_tasks, valid_tasks
+ )
+ raise CrossbowError(msg)
+
+ return {
+ task_name: config_tasks[task_name] for task_name in requested_tasks
+ }
+
+ def validate(self):
+ # validate that the task groups are properly referening the tasks
+ for group_name, group in self['groups'].items():
+ for pattern in group:
+ tasks = self.select(tasks=[pattern])
+ if not tasks:
+ raise CrossbowError(
+ "The pattern `{}` defined for task group `{}` is not "
+ "matching any of the tasks defined in the "
+ "configuration file.".format(pattern, group_name)
+ )
+
+ # validate that the tasks are constructible
+ for task_name, task in self['tasks'].items():
+ try:
+ Task(**task)
+ except Exception as e:
+ raise CrossbowError(
+ 'Unable to construct a task object from the '
+ 'definition of task `{}`. The original error message '
+ 'is: `{}`'.format(task_name, str(e))
+ )
+
+ # validate that the defined tasks are renderable, in order to to that
+ # define the required object with dummy data
+ target = Target(
+ head='e279a7e06e61c14868ca7d71dea795420aea6539',
+ branch='master',
+ remote='https://github.com/apache/arrow',
+ version='1.0.0dev123',
+ email='dummy@example.ltd'
+ )
+
+ for task_name, task in self['tasks'].items():
+ task = Task(**task)
+ files = task.render_files(
+ self.template_searchpath,
+ params=dict(
+ arrow=target,
+ queue_remote_url='https://github.com/org/crossbow'
+ )
+ )
+ if not files:
+ raise CrossbowError('No files have been rendered for task `{}`'
+ .format(task_name))
+
+
+# configure yaml serializer
+yaml = YAML()
+yaml.register_class(Job)
+yaml.register_class(Task)
+yaml.register_class(Target)
diff --git a/src/arrow/dev/archery/archery/crossbow/reports.py b/src/arrow/dev/archery/archery/crossbow/reports.py
new file mode 100644
index 000000000..f86a67a74
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/reports.py
@@ -0,0 +1,315 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import click
+import collections
+import operator
+import fnmatch
+import functools
+from io import StringIO
+import textwrap
+
+
+# TODO(kszucs): use archery.report.JinjaReport instead
+class Report:
+
+ def __init__(self, job, task_filters=None):
+ self.job = job
+
+ tasks = sorted(job.tasks.items())
+ if task_filters:
+ filtered = set()
+ for pattern in task_filters:
+ filtered |= set(fnmatch.filter(job.tasks.keys(), pattern))
+
+ tasks = [(name, task) for name, task in tasks if name in filtered]
+
+ self._tasks = dict(tasks)
+
+ @property
+ def tasks(self):
+ return self._tasks
+
+ def show(self):
+ raise NotImplementedError()
+
+
+class ConsoleReport(Report):
+ """Report the status of a Job to the console using click"""
+
+ # output table's header template
+ HEADER = '[{state:>7}] {branch:<52} {content:>16}'
+ DETAILS = ' └ {url}'
+
+ # output table's row template for assets
+ ARTIFACT_NAME = '{artifact:>69} '
+ ARTIFACT_STATE = '[{state:>7}]'
+
+ # state color mapping to highlight console output
+ COLORS = {
+ # from CombinedStatus
+ 'error': 'red',
+ 'failure': 'red',
+ 'pending': 'yellow',
+ 'success': 'green',
+ # custom state messages
+ 'ok': 'green',
+ 'missing': 'red'
+ }
+
+ def lead(self, state, branch, n_uploaded, n_expected):
+ line = self.HEADER.format(
+ state=state.upper(),
+ branch=branch,
+ content='uploaded {} / {}'.format(n_uploaded, n_expected)
+ )
+ return click.style(line, fg=self.COLORS[state.lower()])
+
+ def header(self):
+ header = self.HEADER.format(
+ state='state',
+ branch='Task / Branch',
+ content='Artifacts'
+ )
+ delimiter = '-' * len(header)
+ return '{}\n{}'.format(header, delimiter)
+
+ def artifact(self, state, pattern, asset):
+ if asset is None:
+ artifact = pattern
+ state = 'pending' if state == 'pending' else 'missing'
+ else:
+ artifact = asset.name
+ state = 'ok'
+
+ name_ = self.ARTIFACT_NAME.format(artifact=artifact)
+ state_ = click.style(
+ self.ARTIFACT_STATE.format(state=state.upper()),
+ self.COLORS[state]
+ )
+ return name_ + state_
+
+ def show(self, outstream, asset_callback=None, validate_patterns=True):
+ echo = functools.partial(click.echo, file=outstream)
+
+ # write table's header
+ echo(self.header())
+
+ # write table's body
+ for task_name, task in self.tasks.items():
+ # write summary of the uploaded vs total assets
+ status = task.status()
+ assets = task.assets(validate_patterns=validate_patterns)
+
+ # mapping of artifact pattern to asset or None of not uploaded
+ n_expected = len(task.artifacts)
+ n_uploaded = len(assets.uploaded_assets())
+ echo(self.lead(status.combined_state, task_name, n_uploaded,
+ n_expected))
+
+ # show link to the actual build, some of the CI providers implement
+ # the statuses API others implement the checks API, so display both
+ for link in status.build_links:
+ echo(self.DETAILS.format(url=link))
+
+ # write per asset status
+ for artifact_pattern, asset in assets.items():
+ if asset_callback is not None:
+ asset_callback(task_name, task, asset)
+ echo(self.artifact(status.combined_state, artifact_pattern,
+ asset))
+
+
+class EmailReport(Report):
+
+ HEADER = textwrap.dedent("""
+ Arrow Build Report for Job {job_name}
+
+ All tasks: {all_tasks_url}
+ """)
+
+ TASK = textwrap.dedent("""
+ - {name}:
+ URL: {url}
+ """).strip()
+
+ EMAIL = textwrap.dedent("""
+ From: {sender_name} <{sender_email}>
+ To: {recipient_email}
+ Subject: {subject}
+
+ {body}
+ """).strip()
+
+ STATUS_HEADERS = {
+ # from CombinedStatus
+ 'error': 'Errored Tasks:',
+ 'failure': 'Failed Tasks:',
+ 'pending': 'Pending Tasks:',
+ 'success': 'Succeeded Tasks:',
+ }
+
+ def __init__(self, job, sender_name, sender_email, recipient_email):
+ self.sender_name = sender_name
+ self.sender_email = sender_email
+ self.recipient_email = recipient_email
+ super().__init__(job)
+
+ def url(self, query):
+ repo_url = self.job.queue.remote_url.strip('.git')
+ return '{}/branches/all?query={}'.format(repo_url, query)
+
+ def listing(self, tasks):
+ return '\n'.join(
+ sorted(
+ self.TASK.format(name=task_name, url=self.url(task.branch))
+ for task_name, task in tasks.items()
+ )
+ )
+
+ def header(self):
+ url = self.url(self.job.branch)
+ return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url)
+
+ def subject(self):
+ return (
+ "[NIGHTLY] Arrow Build Report for Job {}".format(self.job.branch)
+ )
+
+ def body(self):
+ buffer = StringIO()
+ buffer.write(self.header())
+
+ tasks_by_state = collections.defaultdict(dict)
+ for task_name, task in self.job.tasks.items():
+ state = task.status().combined_state
+ tasks_by_state[state][task_name] = task
+
+ for state in ('failure', 'error', 'pending', 'success'):
+ if state in tasks_by_state:
+ tasks = tasks_by_state[state]
+ buffer.write('\n')
+ buffer.write(self.STATUS_HEADERS[state])
+ buffer.write('\n')
+ buffer.write(self.listing(tasks))
+ buffer.write('\n')
+
+ return buffer.getvalue()
+
+ def email(self):
+ return self.EMAIL.format(
+ sender_name=self.sender_name,
+ sender_email=self.sender_email,
+ recipient_email=self.recipient_email,
+ subject=self.subject(),
+ body=self.body()
+ )
+
+ def show(self, outstream):
+ outstream.write(self.email())
+
+ def send(self, smtp_user, smtp_password, smtp_server, smtp_port):
+ import smtplib
+
+ email = self.email()
+
+ server = smtplib.SMTP_SSL(smtp_server, smtp_port)
+ server.ehlo()
+ server.login(smtp_user, smtp_password)
+ server.sendmail(smtp_user, self.recipient_email, email)
+ server.close()
+
+
+class CommentReport(Report):
+
+ _markdown_badge = '[![{title}]({badge})]({url})'
+
+ badges = {
+ 'github': _markdown_badge.format(
+ title='Github Actions',
+ url='https://github.com/{repo}/actions?query=branch:{branch}',
+ badge=(
+ 'https://github.com/{repo}/workflows/Crossbow/'
+ 'badge.svg?branch={branch}'
+ ),
+ ),
+ 'azure': _markdown_badge.format(
+ title='Azure',
+ url=(
+ 'https://dev.azure.com/{repo}/_build/latest'
+ '?definitionId=1&branchName={branch}'
+ ),
+ badge=(
+ 'https://dev.azure.com/{repo}/_apis/build/status/'
+ '{repo_dotted}?branchName={branch}'
+ )
+ ),
+ 'travis': _markdown_badge.format(
+ title='TravisCI',
+ url='https://travis-ci.com/{repo}/branches',
+ badge='https://img.shields.io/travis/{repo}/{branch}.svg'
+ ),
+ 'circle': _markdown_badge.format(
+ title='CircleCI',
+ url='https://circleci.com/gh/{repo}/tree/{branch}',
+ badge=(
+ 'https://img.shields.io/circleci/build/github'
+ '/{repo}/{branch}.svg'
+ )
+ ),
+ 'appveyor': _markdown_badge.format(
+ title='Appveyor',
+ url='https://ci.appveyor.com/project/{repo}/history',
+ badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg'
+ ),
+ 'drone': _markdown_badge.format(
+ title='Drone',
+ url='https://cloud.drone.io/{repo}',
+ badge='https://img.shields.io/drone/build/{repo}/{branch}.svg'
+ ),
+ }
+
+ def __init__(self, job, crossbow_repo):
+ self.crossbow_repo = crossbow_repo
+ super().__init__(job)
+
+ def show(self):
+ url = 'https://github.com/{repo}/branches/all?query={branch}'
+ sha = self.job.target.head
+
+ msg = 'Revision: {}\n\n'.format(sha)
+ msg += 'Submitted crossbow builds: [{repo} @ {branch}]'
+ msg += '({})\n'.format(url)
+ msg += '\n|Task|Status|\n|----|------|'
+
+ tasks = sorted(self.job.tasks.items(), key=operator.itemgetter(0))
+ for key, task in tasks:
+ branch = task.branch
+
+ try:
+ template = self.badges[task.ci]
+ badge = template.format(
+ repo=self.crossbow_repo,
+ repo_dotted=self.crossbow_repo.replace('/', '.'),
+ branch=branch
+ )
+ except KeyError:
+ badge = 'unsupported CI service `{}`'.format(task.ci)
+
+ msg += '\n|{}|{}|'.format(key, badge)
+
+ return msg.format(repo=self.crossbow_repo, branch=self.job.branch)
diff --git a/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
new file mode 100644
index 000000000..c37c7b553
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
@@ -0,0 +1,51 @@
+!Job
+target: !Target
+ head: f766a1d615dd1b7ee706d05102e579195951a61c
+ email: unkown
+ branch: refs/pull/4435/merge
+ remote: https://github.com/apache/arrow
+ version: 0.13.0.dev306
+ no_rc_version: 0.13.0.dev306
+tasks:
+ docker-cpp-cmake32: !Task
+ ci: circle
+ platform: linux
+ template: docker-tests/circle.linux.yml
+ artifacts: []
+ params:
+ commands:
+ - docker-compose build cpp-cmake32
+ - docker-compose run cpp-cmake32
+ branch: ursabot-1-circle-docker-cpp-cmake32
+ commit: a56b077c8d1b891a7935048e5672bf6fc07599ec
+ wheel-osx-cp37m: !Task
+ ci: travis
+ platform: osx
+ template: python-wheels/travis.osx.yml
+ artifacts:
+ - pyarrow-0.13.0.dev306-cp37-cp37m-macosx_10_6_intel.whl
+ params:
+ python_version: 3.7
+ branch: ursabot-1-travis-wheel-osx-cp37m
+ commit: a56b077c8d1b891a7935048e5672bf6fc07599ec
+ wheel-osx-cp36m: !Task
+ ci: travis
+ platform: osx
+ template: python-wheels/travis.osx.yml
+ artifacts:
+ - pyarrow-0.13.0.dev306-cp36-cp36m-macosx_10_6_intel.whl
+ params:
+ python_version: 3.6
+ branch: ursabot-1-travis-wheel-osx-cp36m
+ commit: a56b077c8d1b891a7935048e5672bf6fc07599ec
+ wheel-win-cp36m: !Task
+ ci: appveyor
+ platform: win
+ template: python-wheels/appveyor.yml
+ artifacts:
+ - pyarrow-0.13.0.dev306-cp36-cp36m-win_amd64.whl
+ params:
+ python_version: 3.6
+ branch: ursabot-1-appveyor-wheel-win-cp36m
+ commit: a56b077c8d1b891a7935048e5672bf6fc07599ec
+branch: ursabot-1
diff --git a/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md
new file mode 100644
index 000000000..15825218c
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md
@@ -0,0 +1,10 @@
+Revision: {revision}
+
+Submitted crossbow builds: [{repo} @ {branch}](https://github.com/{repo}/branches/all?query={branch})
+
+|Task|Status|
+|----|------|
+|docker-cpp-cmake32|[![CircleCI](https://img.shields.io/circleci/build/github/{repo}/{branch}-circle-docker-cpp-cmake32.svg)](https://circleci.com/gh/{repo}/tree/{branch}-circle-docker-cpp-cmake32)|
+|wheel-osx-cp36m|[![TravisCI](https://img.shields.io/travis/{repo}/{branch}-travis-wheel-osx-cp36m.svg)](https://travis-ci.com/{repo}/branches)|
+|wheel-osx-cp37m|[![TravisCI](https://img.shields.io/travis/{repo}/{branch}-travis-wheel-osx-cp37m.svg)](https://travis-ci.com/{repo}/branches)|
+|wheel-win-cp36m|[![Appveyor](https://img.shields.io/appveyor/ci/{repo}/{branch}-appveyor-wheel-win-cp36m.svg)](https://ci.appveyor.com/project/{repo}/history)|
diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_core.py b/src/arrow/dev/archery/archery/crossbow/tests/test_core.py
new file mode 100644
index 000000000..518474236
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/tests/test_core.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from archery.utils.source import ArrowSources
+from archery.crossbow import Config
+
+
+def test_config():
+ src = ArrowSources.find()
+ conf = Config.load_yaml(src.dev / "tasks" / "tasks.yml")
+ conf.validate()
diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py b/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py
new file mode 100644
index 000000000..ee9ba1ee2
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from click.testing import CliRunner
+import pytest
+
+from archery.crossbow.cli import crossbow
+from archery.utils.git import git
+
+
+@pytest.mark.integration
+def test_crossbow_submit(tmp_path):
+ runner = CliRunner()
+
+ def invoke(*args):
+ return runner.invoke(crossbow, ['--queue-path', str(tmp_path), *args])
+
+ # initialize an empty crossbow repository
+ git.run_cmd("init", str(tmp_path))
+ git.run_cmd("-C", str(tmp_path), "remote", "add", "origin",
+ "https://github.com/dummy/repo")
+ git.run_cmd("-C", str(tmp_path), "commit", "-m", "initial",
+ "--allow-empty")
+
+ result = invoke('check-config')
+ assert result.exit_code == 0
+
+ result = invoke('submit', '--no-fetch', '--no-push', '-g', 'wheel')
+ assert result.exit_code == 0
diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py b/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py
new file mode 100644
index 000000000..0df292bb5
--- /dev/null
+++ b/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import textwrap
+
+from archery.crossbow.core import yaml
+from archery.crossbow.reports import CommentReport
+
+
+def test_crossbow_comment_formatter(load_fixture):
+ msg = load_fixture('crossbow-success-message.md')
+ job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
+
+ report = CommentReport(job, crossbow_repo='ursa-labs/crossbow')
+ expected = msg.format(
+ repo='ursa-labs/crossbow',
+ branch='ursabot-1',
+ revision='f766a1d615dd1b7ee706d05102e579195951a61c',
+ status='has been succeeded.'
+ )
+ assert report.show() == textwrap.dedent(expected).strip()
diff --git a/src/arrow/dev/archery/archery/docker.py b/src/arrow/dev/archery/archery/docker.py
new file mode 100644
index 000000000..17d4c713a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker.py
@@ -0,0 +1,402 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import subprocess
+from io import StringIO
+
+from dotenv import dotenv_values
+from ruamel.yaml import YAML
+
+from .utils.command import Command, default_bin
+from .compat import _ensure_path
+
+
+def flatten(node, parents=None):
+ parents = list(parents or [])
+ if isinstance(node, str):
+ yield (node, parents)
+ elif isinstance(node, list):
+ for value in node:
+ yield from flatten(value, parents=parents)
+ elif isinstance(node, dict):
+ for key, value in node.items():
+ yield (key, parents)
+ yield from flatten(value, parents=parents + [key])
+ else:
+ raise TypeError(node)
+
+
+def _sanitize_command(cmd):
+ if isinstance(cmd, list):
+ cmd = " ".join(cmd)
+ return re.sub(r"\s+", " ", cmd)
+
+
+class UndefinedImage(Exception):
+ pass
+
+
+class ComposeConfig:
+
+ def __init__(self, config_path, dotenv_path, compose_bin, params=None):
+ config_path = _ensure_path(config_path)
+ if dotenv_path:
+ dotenv_path = _ensure_path(dotenv_path)
+ else:
+ dotenv_path = config_path.parent / '.env'
+ self._read_env(dotenv_path, params)
+ self._read_config(config_path, compose_bin)
+
+ def _read_env(self, dotenv_path, params):
+ """
+ Read .env and merge it with explicitly passed parameters.
+ """
+ self.dotenv = dotenv_values(str(dotenv_path))
+ if params is None:
+ self.params = {}
+ else:
+ self.params = {k: v for k, v in params.items() if k in self.dotenv}
+
+ # forward the process' environment variables
+ self.env = os.environ.copy()
+ # set the defaults from the dotenv files
+ self.env.update(self.dotenv)
+ # override the defaults passed as parameters
+ self.env.update(self.params)
+
+ # translate docker's architecture notation to a more widely used one
+ arch = self.env.get('ARCH', 'amd64')
+ arch_aliases = {
+ 'amd64': 'x86_64',
+ 'arm64v8': 'aarch64',
+ 's390x': 's390x'
+ }
+ arch_short_aliases = {
+ 'amd64': 'x64',
+ 'arm64v8': 'arm64',
+ 's390x': 's390x'
+ }
+ self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch)
+ self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch)
+
+ def _read_config(self, config_path, compose_bin):
+ """
+ Validate and read the docker-compose.yml
+ """
+ yaml = YAML()
+ with config_path.open() as fp:
+ config = yaml.load(fp)
+
+ services = config['services'].keys()
+ self.hierarchy = dict(flatten(config.get('x-hierarchy', {})))
+ self.with_gpus = config.get('x-with-gpus', [])
+ nodes = self.hierarchy.keys()
+ errors = []
+
+ for name in self.with_gpus:
+ if name not in services:
+ errors.append(
+ 'Service `{}` defined in `x-with-gpus` bot not in '
+ '`services`'.format(name)
+ )
+ for name in nodes - services:
+ errors.append(
+ 'Service `{}` is defined in `x-hierarchy` bot not in '
+ '`services`'.format(name)
+ )
+ for name in services - nodes:
+ errors.append(
+ 'Service `{}` is defined in `services` but not in '
+ '`x-hierarchy`'.format(name)
+ )
+
+ # trigger docker-compose's own validation
+ compose = Command('docker-compose')
+ args = ['--file', str(config_path), 'config']
+ result = compose.run(*args, env=self.env, check=False,
+ stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+
+ if result.returncode != 0:
+ # strip the intro line of docker-compose errors
+ errors += result.stderr.decode().splitlines()
+
+ if errors:
+ msg = '\n'.join([' - {}'.format(msg) for msg in errors])
+ raise ValueError(
+ 'Found errors with docker-compose:\n{}'.format(msg)
+ )
+
+ rendered_config = StringIO(result.stdout.decode())
+ self.path = config_path
+ self.config = yaml.load(rendered_config)
+
+ def get(self, service_name):
+ try:
+ service = self.config['services'][service_name]
+ except KeyError:
+ raise UndefinedImage(service_name)
+ service['name'] = service_name
+ service['need_gpu'] = service_name in self.with_gpus
+ service['ancestors'] = self.hierarchy[service_name]
+ return service
+
+ def __getitem__(self, service_name):
+ return self.get(service_name)
+
+
+class Docker(Command):
+
+ def __init__(self, docker_bin=None):
+ self.bin = default_bin(docker_bin, "docker")
+
+
+class DockerCompose(Command):
+
+ def __init__(self, config_path, dotenv_path=None, compose_bin=None,
+ params=None):
+ compose_bin = default_bin(compose_bin, 'docker-compose')
+ self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
+ params)
+ self.bin = compose_bin
+ self.pull_memory = set()
+
+ def clear_pull_memory(self):
+ self.pull_memory = set()
+
+ def _execute_compose(self, *args, **kwargs):
+ # execute as a docker compose command
+ try:
+ result = super().run('--file', str(self.config.path), *args,
+ env=self.config.env, **kwargs)
+ result.check_returncode()
+ except subprocess.CalledProcessError as e:
+ def formatdict(d, template):
+ return '\n'.join(
+ template.format(k, v) for k, v in sorted(d.items())
+ )
+ msg = (
+ "`{cmd}` exited with a non-zero exit code {code}, see the "
+ "process log above.\n\nThe docker-compose command was "
+ "invoked with the following parameters:\n\nDefaults defined "
+ "in .env:\n{dotenv}\n\nArchery was called with:\n{params}"
+ )
+ raise RuntimeError(
+ msg.format(
+ cmd=' '.join(e.cmd),
+ code=e.returncode,
+ dotenv=formatdict(self.config.dotenv, template=' {}: {}'),
+ params=formatdict(
+ self.config.params, template=' export {}={}'
+ )
+ )
+ )
+
+ def _execute_docker(self, *args, **kwargs):
+ # execute as a plain docker cli command
+ try:
+ result = Docker().run(*args, **kwargs)
+ result.check_returncode()
+ except subprocess.CalledProcessError as e:
+ raise RuntimeError(
+ "{} exited with non-zero exit code {}".format(
+ ' '.join(e.cmd), e.returncode
+ )
+ )
+
+ def pull(self, service_name, pull_leaf=True, using_docker=False):
+ def _pull(service):
+ args = ['pull']
+ if service['image'] in self.pull_memory:
+ return
+
+ if using_docker:
+ try:
+ self._execute_docker(*args, service['image'])
+ except Exception as e:
+ # better --ignore-pull-failures handling
+ print(e)
+ else:
+ args.append('--ignore-pull-failures')
+ self._execute_compose(*args, service['name'])
+
+ self.pull_memory.add(service['image'])
+
+ service = self.config.get(service_name)
+ for ancestor in service['ancestors']:
+ _pull(self.config.get(ancestor))
+ if pull_leaf:
+ _pull(service)
+
+ def build(self, service_name, use_cache=True, use_leaf_cache=True,
+ using_docker=False, using_buildx=False):
+ def _build(service, use_cache):
+ if 'build' not in service:
+ # nothing to do
+ return
+
+ args = []
+ cache_from = list(service.get('build', {}).get('cache_from', []))
+ if use_cache:
+ for image in cache_from:
+ if image not in self.pull_memory:
+ try:
+ self._execute_docker('pull', image)
+ except Exception as e:
+ print(e)
+ finally:
+ self.pull_memory.add(image)
+ else:
+ args.append('--no-cache')
+
+ # turn on inline build cache, this is a docker buildx feature
+ # used to bundle the image build cache to the pushed image manifest
+ # so the build cache can be reused across hosts, documented at
+ # https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+ if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1':
+ args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1'])
+
+ if using_buildx:
+ for k, v in service['build'].get('args', {}).items():
+ args.extend(['--build-arg', '{}={}'.format(k, v)])
+
+ if use_cache:
+ cache_ref = '{}-cache'.format(service['image'])
+ cache_from = 'type=registry,ref={}'.format(cache_ref)
+ cache_to = (
+ 'type=registry,ref={},mode=max'.format(cache_ref)
+ )
+ args.extend([
+ '--cache-from', cache_from,
+ '--cache-to', cache_to,
+ ])
+
+ args.extend([
+ '--output', 'type=docker',
+ '-f', service['build']['dockerfile'],
+ '-t', service['image'],
+ service['build'].get('context', '.')
+ ])
+ self._execute_docker("buildx", "build", *args)
+ elif using_docker:
+ # better for caching
+ for k, v in service['build'].get('args', {}).items():
+ args.extend(['--build-arg', '{}={}'.format(k, v)])
+ for img in cache_from:
+ args.append('--cache-from="{}"'.format(img))
+ args.extend([
+ '-f', service['build']['dockerfile'],
+ '-t', service['image'],
+ service['build'].get('context', '.')
+ ])
+ self._execute_docker("build", *args)
+ else:
+ self._execute_compose("build", *args, service['name'])
+
+ service = self.config.get(service_name)
+ # build ancestor services
+ for ancestor in service['ancestors']:
+ _build(self.config.get(ancestor), use_cache=use_cache)
+ # build the leaf/target service
+ _build(service, use_cache=use_cache and use_leaf_cache)
+
+ def run(self, service_name, command=None, *, env=None, volumes=None,
+ user=None, using_docker=False):
+ service = self.config.get(service_name)
+
+ args = []
+ if user is not None:
+ args.extend(['-u', user])
+
+ if env is not None:
+ for k, v in env.items():
+ args.extend(['-e', '{}={}'.format(k, v)])
+
+ if volumes is not None:
+ for volume in volumes:
+ args.extend(['--volume', volume])
+
+ if using_docker or service['need_gpu']:
+ # use gpus, requires docker>=19.03
+ if service['need_gpu']:
+ args.extend(['--gpus', 'all'])
+
+ if service.get('shm_size'):
+ args.extend(['--shm-size', service['shm_size']])
+
+ # append env variables from the compose conf
+ for k, v in service.get('environment', {}).items():
+ args.extend(['-e', '{}={}'.format(k, v)])
+
+ # append volumes from the compose conf
+ for v in service.get('volumes', []):
+ if not isinstance(v, str):
+ # if not the compact string volume definition
+ v = "{}:{}".format(v['source'], v['target'])
+ args.extend(['-v', v])
+
+ # infer whether an interactive shell is desired or not
+ if command in ['cmd.exe', 'bash', 'sh', 'powershell']:
+ args.append('-it')
+
+ # get the actual docker image name instead of the compose service
+ # name which we refer as image in general
+ args.append(service['image'])
+
+ # add command from compose if it wasn't overridden
+ if command is not None:
+ args.append(command)
+ else:
+ # replace whitespaces from the preformatted compose command
+ cmd = _sanitize_command(service.get('command', ''))
+ if cmd:
+ args.append(cmd)
+
+ # execute as a plain docker cli command
+ self._execute_docker('run', '--rm', *args)
+ else:
+ # execute as a docker-compose command
+ args.append(service_name)
+ if command is not None:
+ args.append(command)
+ self._execute_compose('run', '--rm', *args)
+
+ def push(self, service_name, user=None, password=None, using_docker=False):
+ def _push(service):
+ if using_docker:
+ return self._execute_docker('push', service['image'])
+ else:
+ return self._execute_compose('push', service['name'])
+
+ if user is not None:
+ try:
+ # TODO(kszucs): have an option for a prompt
+ self._execute_docker('login', '-u', user, '-p', password)
+ except subprocess.CalledProcessError:
+ # hide credentials
+ msg = ('Failed to push `{}`, check the passed credentials'
+ .format(service_name))
+ raise RuntimeError(msg) from None
+
+ service = self.config.get(service_name)
+ for ancestor in service['ancestors']:
+ _push(self.config.get(ancestor))
+ _push(service)
+
+ def images(self):
+ return sorted(self.config.hierarchy.keys())
diff --git a/src/arrow/dev/archery/archery/docker/__init__.py b/src/arrow/dev/archery/archery/docker/__init__.py
new file mode 100644
index 000000000..6be29c916
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker/__init__.py
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .core import DockerCompose, UndefinedImage # noqa
diff --git a/src/arrow/dev/archery/archery/docker/cli.py b/src/arrow/dev/archery/archery/docker/cli.py
new file mode 100644
index 000000000..c6b4a6473
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker/cli.py
@@ -0,0 +1,261 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import click
+
+from ..utils.cli import validate_arrow_sources
+from .core import DockerCompose, UndefinedImage
+
+
+def _mock_compose_calls(compose):
+ from types import MethodType
+ from subprocess import CompletedProcess
+
+ def _mock(compose, executable):
+ def _execute(self, *args, **kwargs):
+ params = ['{}={}'.format(k, v)
+ for k, v in self.config.params.items()]
+ command = ' '.join(params + [executable] + list(args))
+ click.echo(command)
+ return CompletedProcess([], 0)
+ return MethodType(_execute, compose)
+
+ compose._execute_docker = _mock(compose, executable='docker')
+ compose._execute_compose = _mock(compose, executable='docker-compose')
+
+
+@click.group()
+@click.option("--src", metavar="<arrow_src>", default=None,
+ callback=validate_arrow_sources,
+ help="Specify Arrow source directory.")
+@click.option('--dry-run/--execute', default=False,
+ help="Display the docker-compose commands instead of executing "
+ "them.")
+@click.pass_context
+def docker(ctx, src, dry_run):
+ """
+ Interact with docker-compose based builds.
+ """
+ ctx.ensure_object(dict)
+
+ config_path = src.path / 'docker-compose.yml'
+ if not config_path.exists():
+ raise click.ClickException(
+ "Docker compose configuration cannot be found in directory {}, "
+ "try to pass the arrow source directory explicitly.".format(src)
+ )
+
+ # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
+ # environment variables to keep the usage similar to docker-compose
+ compose = DockerCompose(config_path, params=os.environ)
+ if dry_run:
+ _mock_compose_calls(compose)
+ ctx.obj['compose'] = compose
+
+
+@docker.command("check-config")
+@click.pass_obj
+def check_config(obj):
+ """
+ Validate docker-compose configuration.
+ """
+ # executes the body of the docker function above which does the validation
+ # during the configuration loading
+
+
+@docker.command('build')
+@click.argument('image')
+@click.option('--force-pull/--no-pull', default=True,
+ help="Whether to force pull the image and its ancestor images")
+@click.option('--using-docker-cli', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_CLI',
+ help="Use docker CLI directly for building instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
+@click.option('--using-docker-buildx', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_BUILDX',
+ help="Use buildx with docker CLI directly for building instead "
+ "of calling docker-compose or the plain docker build "
+ "command. This option makes the build cache reusable "
+ "across hosts.")
+@click.option('--use-cache/--no-cache', default=True,
+ help="Whether to use cache when building the image and its "
+ "ancestor images")
+@click.option('--use-leaf-cache/--no-leaf-cache', default=True,
+ help="Whether to use cache when building only the (leaf) image "
+ "passed as the argument. To disable caching for both the "
+ "image and its ancestors use --no-cache option.")
+@click.pass_obj
+def docker_build(obj, image, *, force_pull, using_docker_cli,
+ using_docker_buildx, use_cache, use_leaf_cache):
+ """
+ Execute docker-compose builds.
+ """
+ compose = obj['compose']
+
+ using_docker_cli |= using_docker_buildx
+ try:
+ if force_pull:
+ compose.pull(image, pull_leaf=use_leaf_cache,
+ using_docker=using_docker_cli)
+ compose.build(image, use_cache=use_cache,
+ use_leaf_cache=use_leaf_cache,
+ using_docker=using_docker_cli,
+ using_buildx=using_docker_buildx,
+ pull_parents=force_pull)
+ except UndefinedImage as e:
+ raise click.ClickException(
+ "There is no service/image defined in docker-compose.yml with "
+ "name: {}".format(str(e))
+ )
+ except RuntimeError as e:
+ raise click.ClickException(str(e))
+
+
+@docker.command('run')
+@click.argument('image')
+@click.argument('command', required=False, default=None)
+@click.option('--env', '-e', multiple=True,
+ help="Set environment variable within the container")
+@click.option('--user', '-u', default=None,
+ help="Username or UID to run the container with")
+@click.option('--force-pull/--no-pull', default=True,
+ help="Whether to force pull the image and its ancestor images")
+@click.option('--force-build/--no-build', default=True,
+ help="Whether to force build the image and its ancestor images")
+@click.option('--build-only', default=False, is_flag=True,
+ help="Pull and/or build the image, but do not run it")
+@click.option('--using-docker-cli', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_CLI',
+ help="Use docker CLI directly for building instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
+@click.option('--using-docker-buildx', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_BUILDX',
+ help="Use buildx with docker CLI directly for building instead "
+ "of calling docker-compose or the plain docker build "
+ "command. This option makes the build cache reusable "
+ "across hosts.")
+@click.option('--use-cache/--no-cache', default=True,
+ help="Whether to use cache when building the image and its "
+ "ancestor images")
+@click.option('--use-leaf-cache/--no-leaf-cache', default=True,
+ help="Whether to use cache when building only the (leaf) image "
+ "passed as the argument. To disable caching for both the "
+ "image and its ancestors use --no-cache option.")
+@click.option('--resource-limit', default=None,
+ help="A CPU/memory limit preset to mimic CI environments like "
+ "GitHub Actions. Implies --using-docker-cli. Note that "
+ "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
+ "required, unless Docker is configured with cgroups v2 "
+ "(else Docker will silently ignore the limits).")
+@click.option('--volume', '-v', multiple=True,
+ help="Set volume within the container")
+@click.pass_obj
+def docker_run(obj, image, command, *, env, user, force_pull, force_build,
+ build_only, using_docker_cli, using_docker_buildx, use_cache,
+ use_leaf_cache, resource_limit, volume):
+ """
+ Execute docker-compose builds.
+
+ To see the available builds run `archery docker images`.
+
+ Examples:
+
+ # execute a single build
+ archery docker run conda-python
+
+ # execute the builds but disable the image pulling
+ archery docker run --no-cache conda-python
+
+ # pass a docker-compose parameter, like the python version
+ PYTHON=3.8 archery docker run conda-python
+
+ # disable the cache only for the leaf image
+ PANDAS=master archery docker run --no-leaf-cache conda-python-pandas
+
+ # entirely skip building the image
+ archery docker run --no-pull --no-build conda-python
+
+ # pass runtime parameters via docker environment variables
+ archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp
+
+ # set a volume
+ archery docker run -v $PWD/build:/build ubuntu-cpp
+
+ # starting an interactive bash session for debugging
+ archery docker run ubuntu-cpp bash
+ """
+ compose = obj['compose']
+ using_docker_cli |= using_docker_buildx
+
+ env = dict(kv.split('=', 1) for kv in env)
+ try:
+ if force_pull:
+ compose.pull(image, pull_leaf=use_leaf_cache,
+ using_docker=using_docker_cli)
+ if force_build:
+ compose.build(image, use_cache=use_cache,
+ use_leaf_cache=use_leaf_cache,
+ using_docker=using_docker_cli,
+ using_buildx=using_docker_buildx)
+ if build_only:
+ return
+ compose.run(
+ image,
+ command=command,
+ env=env,
+ user=user,
+ using_docker=using_docker_cli,
+ resource_limit=resource_limit,
+ volumes=volume
+ )
+ except UndefinedImage as e:
+ raise click.ClickException(
+ "There is no service/image defined in docker-compose.yml with "
+ "name: {}".format(str(e))
+ )
+ except RuntimeError as e:
+ raise click.ClickException(str(e))
+
+
+@docker.command('push')
+@click.argument('image')
+@click.option('--user', '-u', required=False, envvar='ARCHERY_DOCKER_USER',
+ help='Docker repository username')
+@click.option('--password', '-p', required=False,
+ envvar='ARCHERY_DOCKER_PASSWORD',
+ help='Docker repository password')
+@click.option('--using-docker-cli', default=False, is_flag=True,
+ help="Use docker CLI directly for building instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
+@click.pass_obj
+def docker_compose_push(obj, image, user, password, using_docker_cli):
+ """Push the generated docker-compose image."""
+ compose = obj['compose']
+ compose.push(image, user=user, password=password,
+ using_docker=using_docker_cli)
+
+
+@docker.command('images')
+@click.pass_obj
+def docker_compose_images(obj):
+ """List the available docker-compose images."""
+ compose = obj['compose']
+ click.echo('Available images:')
+ for image in compose.images():
+ click.echo(f' - {image}')
diff --git a/src/arrow/dev/archery/archery/docker/core.py b/src/arrow/dev/archery/archery/docker/core.py
new file mode 100644
index 000000000..aaf16bdfa
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker/core.py
@@ -0,0 +1,417 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import subprocess
+from io import StringIO
+
+from dotenv import dotenv_values
+from ruamel.yaml import YAML
+
+from ..utils.command import Command, default_bin
+from ..compat import _ensure_path
+
+
+def flatten(node, parents=None):
+ parents = list(parents or [])
+ if isinstance(node, str):
+ yield (node, parents)
+ elif isinstance(node, list):
+ for value in node:
+ yield from flatten(value, parents=parents)
+ elif isinstance(node, dict):
+ for key, value in node.items():
+ yield (key, parents)
+ yield from flatten(value, parents=parents + [key])
+ else:
+ raise TypeError(node)
+
+
+def _sanitize_command(cmd):
+ if isinstance(cmd, list):
+ cmd = " ".join(cmd)
+ return re.sub(r"\s+", " ", cmd)
+
+
+class UndefinedImage(Exception):
+ pass
+
+
+class ComposeConfig:
+
+ def __init__(self, config_path, dotenv_path, compose_bin, params=None):
+ config_path = _ensure_path(config_path)
+ if dotenv_path:
+ dotenv_path = _ensure_path(dotenv_path)
+ else:
+ dotenv_path = config_path.parent / '.env'
+ self._read_env(dotenv_path, params)
+ self._read_config(config_path, compose_bin)
+
+ def _read_env(self, dotenv_path, params):
+ """
+ Read .env and merge it with explicitly passed parameters.
+ """
+ self.dotenv = dotenv_values(str(dotenv_path))
+ if params is None:
+ self.params = {}
+ else:
+ self.params = {k: v for k, v in params.items() if k in self.dotenv}
+
+ # forward the process' environment variables
+ self.env = os.environ.copy()
+ # set the defaults from the dotenv files
+ self.env.update(self.dotenv)
+ # override the defaults passed as parameters
+ self.env.update(self.params)
+
+ # translate docker's architecture notation to a more widely used one
+ arch = self.env.get('ARCH', 'amd64')
+ arch_aliases = {
+ 'amd64': 'x86_64',
+ 'arm64v8': 'aarch64',
+ 's390x': 's390x'
+ }
+ arch_short_aliases = {
+ 'amd64': 'x64',
+ 'arm64v8': 'arm64',
+ 's390x': 's390x'
+ }
+ self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch)
+ self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch)
+
+ def _read_config(self, config_path, compose_bin):
+ """
+ Validate and read the docker-compose.yml
+ """
+ yaml = YAML()
+ with config_path.open() as fp:
+ config = yaml.load(fp)
+
+ services = config['services'].keys()
+ self.hierarchy = dict(flatten(config.get('x-hierarchy', {})))
+ self.limit_presets = config.get('x-limit-presets', {})
+ self.with_gpus = config.get('x-with-gpus', [])
+ nodes = self.hierarchy.keys()
+ errors = []
+
+ for name in self.with_gpus:
+ if name not in services:
+ errors.append(
+ 'Service `{}` defined in `x-with-gpus` bot not in '
+ '`services`'.format(name)
+ )
+ for name in nodes - services:
+ errors.append(
+ 'Service `{}` is defined in `x-hierarchy` bot not in '
+ '`services`'.format(name)
+ )
+ for name in services - nodes:
+ errors.append(
+ 'Service `{}` is defined in `services` but not in '
+ '`x-hierarchy`'.format(name)
+ )
+
+ # trigger docker-compose's own validation
+ compose = Command('docker-compose')
+ args = ['--file', str(config_path), 'config']
+ result = compose.run(*args, env=self.env, check=False,
+ stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+
+ if result.returncode != 0:
+ # strip the intro line of docker-compose errors
+ errors += result.stderr.decode().splitlines()
+
+ if errors:
+ msg = '\n'.join([' - {}'.format(msg) for msg in errors])
+ raise ValueError(
+ 'Found errors with docker-compose:\n{}'.format(msg)
+ )
+
+ rendered_config = StringIO(result.stdout.decode())
+ self.path = config_path
+ self.config = yaml.load(rendered_config)
+
+ def get(self, service_name):
+ try:
+ service = self.config['services'][service_name]
+ except KeyError:
+ raise UndefinedImage(service_name)
+ service['name'] = service_name
+ service['need_gpu'] = service_name in self.with_gpus
+ service['ancestors'] = self.hierarchy[service_name]
+ return service
+
+ def __getitem__(self, service_name):
+ return self.get(service_name)
+
+
+class Docker(Command):
+
+ def __init__(self, docker_bin=None):
+ self.bin = default_bin(docker_bin, "docker")
+
+
+class DockerCompose(Command):
+
+ def __init__(self, config_path, dotenv_path=None, compose_bin=None,
+ params=None):
+ compose_bin = default_bin(compose_bin, 'docker-compose')
+ self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
+ params)
+ self.bin = compose_bin
+ self.pull_memory = set()
+
+ def clear_pull_memory(self):
+ self.pull_memory = set()
+
+ def _execute_compose(self, *args, **kwargs):
+ # execute as a docker compose command
+ try:
+ result = super().run('--file', str(self.config.path), *args,
+ env=self.config.env, **kwargs)
+ result.check_returncode()
+ except subprocess.CalledProcessError as e:
+ def formatdict(d, template):
+ return '\n'.join(
+ template.format(k, v) for k, v in sorted(d.items())
+ )
+ msg = (
+ "`{cmd}` exited with a non-zero exit code {code}, see the "
+ "process log above.\n\nThe docker-compose command was "
+ "invoked with the following parameters:\n\nDefaults defined "
+ "in .env:\n{dotenv}\n\nArchery was called with:\n{params}"
+ )
+ raise RuntimeError(
+ msg.format(
+ cmd=' '.join(e.cmd),
+ code=e.returncode,
+ dotenv=formatdict(self.config.dotenv, template=' {}: {}'),
+ params=formatdict(
+ self.config.params, template=' export {}={}'
+ )
+ )
+ )
+
+ def _execute_docker(self, *args, **kwargs):
+ # execute as a plain docker cli command
+ try:
+ result = Docker().run(*args, **kwargs)
+ result.check_returncode()
+ except subprocess.CalledProcessError as e:
+ raise RuntimeError(
+ "{} exited with non-zero exit code {}".format(
+ ' '.join(e.cmd), e.returncode
+ )
+ )
+
+ def pull(self, service_name, pull_leaf=True, using_docker=False):
+ def _pull(service):
+ args = ['pull']
+ if service['image'] in self.pull_memory:
+ return
+
+ if using_docker:
+ try:
+ self._execute_docker(*args, service['image'])
+ except Exception as e:
+ # better --ignore-pull-failures handling
+ print(e)
+ else:
+ args.append('--ignore-pull-failures')
+ self._execute_compose(*args, service['name'])
+
+ self.pull_memory.add(service['image'])
+
+ service = self.config.get(service_name)
+ for ancestor in service['ancestors']:
+ _pull(self.config.get(ancestor))
+ if pull_leaf:
+ _pull(service)
+
+ def build(self, service_name, use_cache=True, use_leaf_cache=True,
+ using_docker=False, using_buildx=False, pull_parents=True):
+ def _build(service, use_cache):
+ if 'build' not in service:
+ # nothing to do
+ return
+
+ args = []
+ cache_from = list(service.get('build', {}).get('cache_from', []))
+ if pull_parents:
+ for image in cache_from:
+ if image not in self.pull_memory:
+ try:
+ self._execute_docker('pull', image)
+ except Exception as e:
+ print(e)
+ finally:
+ self.pull_memory.add(image)
+
+ if not use_cache:
+ args.append('--no-cache')
+
+ # turn on inline build cache, this is a docker buildx feature
+ # used to bundle the image build cache to the pushed image manifest
+ # so the build cache can be reused across hosts, documented at
+ # https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+ if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1':
+ args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1'])
+
+ if using_buildx:
+ for k, v in service['build'].get('args', {}).items():
+ args.extend(['--build-arg', '{}={}'.format(k, v)])
+
+ if use_cache:
+ cache_ref = '{}-cache'.format(service['image'])
+ cache_from = 'type=registry,ref={}'.format(cache_ref)
+ cache_to = (
+ 'type=registry,ref={},mode=max'.format(cache_ref)
+ )
+ args.extend([
+ '--cache-from', cache_from,
+ '--cache-to', cache_to,
+ ])
+
+ args.extend([
+ '--output', 'type=docker',
+ '-f', service['build']['dockerfile'],
+ '-t', service['image'],
+ service['build'].get('context', '.')
+ ])
+ self._execute_docker("buildx", "build", *args)
+ elif using_docker:
+ # better for caching
+ for k, v in service['build'].get('args', {}).items():
+ args.extend(['--build-arg', '{}={}'.format(k, v)])
+ for img in cache_from:
+ args.append('--cache-from="{}"'.format(img))
+ args.extend([
+ '-f', service['build']['dockerfile'],
+ '-t', service['image'],
+ service['build'].get('context', '.')
+ ])
+ self._execute_docker("build", *args)
+ else:
+ self._execute_compose("build", *args, service['name'])
+
+ service = self.config.get(service_name)
+ # build ancestor services
+ for ancestor in service['ancestors']:
+ _build(self.config.get(ancestor), use_cache=use_cache)
+ # build the leaf/target service
+ _build(service, use_cache=use_cache and use_leaf_cache)
+
+ def run(self, service_name, command=None, *, env=None, volumes=None,
+ user=None, using_docker=False, resource_limit=None):
+ service = self.config.get(service_name)
+
+ args = []
+ if user is not None:
+ args.extend(['-u', user])
+
+ if env is not None:
+ for k, v in env.items():
+ args.extend(['-e', '{}={}'.format(k, v)])
+
+ if volumes is not None:
+ for volume in volumes:
+ args.extend(['--volume', volume])
+
+ if using_docker or service['need_gpu'] or resource_limit:
+ # use gpus, requires docker>=19.03
+ if service['need_gpu']:
+ args.extend(['--gpus', 'all'])
+
+ if service.get('shm_size'):
+ args.extend(['--shm-size', service['shm_size']])
+
+ # append env variables from the compose conf
+ for k, v in service.get('environment', {}).items():
+ args.extend(['-e', '{}={}'.format(k, v)])
+
+ # append volumes from the compose conf
+ for v in service.get('volumes', []):
+ if not isinstance(v, str):
+ # if not the compact string volume definition
+ v = "{}:{}".format(v['source'], v['target'])
+ args.extend(['-v', v])
+
+ # infer whether an interactive shell is desired or not
+ if command in ['cmd.exe', 'bash', 'sh', 'powershell']:
+ args.append('-it')
+
+ if resource_limit:
+ limits = self.config.limit_presets.get(resource_limit)
+ if not limits:
+ raise ValueError(
+ f"Unknown resource limit preset '{resource_limit}'")
+ cpuset = limits.get('cpuset_cpus', [])
+ if cpuset:
+ args.append(f'--cpuset-cpus={",".join(map(str, cpuset))}')
+ memory = limits.get('memory')
+ if memory:
+ args.append(f'--memory={memory}')
+ args.append(f'--memory-swap={memory}')
+
+ # get the actual docker image name instead of the compose service
+ # name which we refer as image in general
+ args.append(service['image'])
+
+ # add command from compose if it wasn't overridden
+ if command is not None:
+ args.append(command)
+ else:
+ # replace whitespaces from the preformatted compose command
+ cmd = _sanitize_command(service.get('command', ''))
+ if cmd:
+ args.append(cmd)
+
+ # execute as a plain docker cli command
+ self._execute_docker('run', '--rm', *args)
+ else:
+ # execute as a docker-compose command
+ args.append(service_name)
+ if command is not None:
+ args.append(command)
+ self._execute_compose('run', '--rm', *args)
+
+ def push(self, service_name, user=None, password=None, using_docker=False):
+ def _push(service):
+ if using_docker:
+ return self._execute_docker('push', service['image'])
+ else:
+ return self._execute_compose('push', service['name'])
+
+ if user is not None:
+ try:
+ # TODO(kszucs): have an option for a prompt
+ self._execute_docker('login', '-u', user, '-p', password)
+ except subprocess.CalledProcessError:
+ # hide credentials
+ msg = ('Failed to push `{}`, check the passed credentials'
+ .format(service_name))
+ raise RuntimeError(msg) from None
+
+ service = self.config.get(service_name)
+ for ancestor in service['ancestors']:
+ _push(self.config.get(ancestor))
+ _push(service)
+
+ def images(self):
+ return sorted(self.config.hierarchy.keys())
diff --git a/src/arrow/dev/archery/archery/docker/tests/test_docker.py b/src/arrow/dev/archery/archery/docker/tests/test_docker.py
new file mode 100644
index 000000000..982f3bfc1
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker/tests/test_docker.py
@@ -0,0 +1,531 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import collections
+import os
+import re
+import subprocess
+from unittest import mock
+
+import pytest
+
+from archery.docker import DockerCompose
+from archery.testing import assert_subprocess_calls, override_env, PartialEnv
+
+
+missing_service_compose_yml = """
+version: '3.5'
+
+x-hierarchy:
+ - foo:
+ - sub-foo:
+ - sub-sub-foo
+ - another-sub-sub-foo
+ - bar:
+ - sub-bar
+ - baz
+
+services:
+ foo:
+ image: org/foo
+ sub-sub-foo:
+ image: org/sub-sub-foo
+ another-sub-sub-foo:
+ image: org/another-sub-sub-foo
+ bar:
+ image: org/bar
+ sub-bar:
+ image: org/sub-bar
+ baz:
+ image: org/baz
+"""
+
+missing_node_compose_yml = """
+version: '3.5'
+
+x-hierarchy:
+ - foo:
+ - sub-foo:
+ - sub-sub-foo
+ - another-sub-sub-foo
+ - bar
+ - baz
+
+services:
+ foo:
+ image: org/foo
+ sub-foo:
+ image: org/sub-foo
+ sub-sub-foo:
+ image: org/sub-foo-foo
+ another-sub-sub-foo:
+ image: org/another-sub-sub-foo
+ bar:
+ image: org/bar
+ sub-bar:
+ image: org/sub-bar
+ baz:
+ image: org/baz
+"""
+
+ok_compose_yml = """
+version: '3.5'
+
+x-hierarchy:
+ - foo:
+ - sub-foo:
+ - sub-sub-foo
+ - another-sub-sub-foo
+ - bar:
+ - sub-bar
+ - baz
+
+services:
+ foo:
+ image: org/foo
+ sub-foo:
+ image: org/sub-foo
+ sub-sub-foo:
+ image: org/sub-sub-foo
+ another-sub-sub-foo:
+ image: org/another-sub-sub-foo
+ bar:
+ image: org/bar
+ sub-bar:
+ image: org/sub-bar
+ baz:
+ image: org/baz
+"""
+
+arrow_compose_yml = """
+version: '3.5'
+
+x-with-gpus:
+ - ubuntu-cuda
+
+x-hierarchy:
+ - conda-cpp:
+ - conda-python:
+ - conda-python-pandas
+ - conda-python-dask
+ - ubuntu-cpp:
+ - ubuntu-cpp-cmake32
+ - ubuntu-c-glib:
+ - ubuntu-ruby
+ - ubuntu-cuda
+
+x-limit-presets:
+ github:
+ cpuset_cpus: [0, 1]
+ memory: 7g
+
+services:
+ conda-cpp:
+ image: org/conda-cpp
+ build:
+ context: .
+ dockerfile: ci/docker/conda-cpp.dockerfile
+ conda-python:
+ image: org/conda-python
+ build:
+ context: .
+ dockerfile: ci/docker/conda-cpp.dockerfile
+ args:
+ python: 3.6
+ conda-python-pandas:
+ image: org/conda-python-pandas
+ build:
+ context: .
+ dockerfile: ci/docker/conda-python-pandas.dockerfile
+ conda-python-dask:
+ image: org/conda-python-dask
+ ubuntu-cpp:
+ image: org/ubuntu-cpp
+ build:
+ context: .
+ dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
+ ubuntu-cpp-cmake32:
+ image: org/ubuntu-cpp-cmake32
+ ubuntu-c-glib:
+ image: org/ubuntu-c-glib
+ ubuntu-ruby:
+ image: org/ubuntu-ruby
+ ubuntu-cuda:
+ image: org/ubuntu-cuda
+ environment:
+ CUDA_ENV: 1
+ OTHER_ENV: 2
+ volumes:
+ - /host:/container
+ command: /bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"
+"""
+
+arrow_compose_env = {
+ 'UBUNTU': '20.04', # overridden below
+ 'PYTHON': '3.6',
+ 'PANDAS': 'latest',
+ 'DASK': 'latest', # overridden below
+}
+
+
+def create_config(directory, yml_content, env_content=None):
+ env_path = directory / '.env'
+ config_path = directory / 'docker-compose.yml'
+
+ with config_path.open('w') as fp:
+ fp.write(yml_content)
+
+ if env_content is not None:
+ with env_path.open('w') as fp:
+ for k, v in env_content.items():
+ fp.write("{}={}\n".format(k, v))
+
+ return config_path
+
+
+def format_run(args):
+ cmd = ["run", "--rm"]
+ if isinstance(args, str):
+ return " ".join(cmd + [args])
+ else:
+ return cmd + args
+
+
+@pytest.fixture
+def arrow_compose_path(tmpdir):
+ return create_config(tmpdir, arrow_compose_yml, arrow_compose_env)
+
+
+def test_config_validation(tmpdir):
+ config_path = create_config(tmpdir, missing_service_compose_yml)
+ msg = "`sub-foo` is defined in `x-hierarchy` bot not in `services`"
+ with pytest.raises(ValueError, match=msg):
+ DockerCompose(config_path)
+
+ config_path = create_config(tmpdir, missing_node_compose_yml)
+ msg = "`sub-bar` is defined in `services` but not in `x-hierarchy`"
+ with pytest.raises(ValueError, match=msg):
+ DockerCompose(config_path)
+
+ config_path = create_config(tmpdir, ok_compose_yml)
+ DockerCompose(config_path) # no issue
+
+
+def assert_docker_calls(compose, expected_args):
+ base_command = ['docker']
+ expected_commands = []
+ for args in expected_args:
+ if isinstance(args, str):
+ args = re.split(r"\s", args)
+ expected_commands.append(base_command + args)
+ return assert_subprocess_calls(expected_commands, check=True)
+
+
+def assert_compose_calls(compose, expected_args, env=mock.ANY):
+ base_command = ['docker-compose', '--file', str(compose.config.path)]
+ expected_commands = []
+ for args in expected_args:
+ if isinstance(args, str):
+ args = re.split(r"\s", args)
+ expected_commands.append(base_command + args)
+ return assert_subprocess_calls(expected_commands, check=True, env=env)
+
+
+def test_arrow_example_validation_passes(arrow_compose_path):
+ DockerCompose(arrow_compose_path)
+
+
+def test_compose_default_params_and_env(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path, params=dict(
+ UBUNTU='18.04',
+ DASK='master'
+ ))
+ assert compose.config.dotenv == arrow_compose_env
+ assert compose.config.params == {
+ 'UBUNTU': '18.04',
+ 'DASK': 'master',
+ }
+
+
+def test_forwarding_env_variables(arrow_compose_path):
+ expected_calls = [
+ "pull --ignore-pull-failures conda-cpp",
+ "build conda-cpp",
+ ]
+ expected_env = PartialEnv(
+ MY_CUSTOM_VAR_A='a',
+ MY_CUSTOM_VAR_B='b'
+ )
+ with override_env({'MY_CUSTOM_VAR_A': 'a', 'MY_CUSTOM_VAR_B': 'b'}):
+ compose = DockerCompose(arrow_compose_path)
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ assert os.environ['MY_CUSTOM_VAR_A'] == 'a'
+ assert os.environ['MY_CUSTOM_VAR_B'] == 'b'
+ compose.pull('conda-cpp')
+ compose.build('conda-cpp')
+
+
+def test_compose_pull(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path)
+
+ expected_calls = [
+ "pull --ignore-pull-failures conda-cpp",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.clear_pull_memory()
+ compose.pull('conda-cpp')
+
+ expected_calls = [
+ "pull --ignore-pull-failures conda-cpp",
+ "pull --ignore-pull-failures conda-python",
+ "pull --ignore-pull-failures conda-python-pandas"
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.clear_pull_memory()
+ compose.pull('conda-python-pandas')
+
+ expected_calls = [
+ "pull --ignore-pull-failures conda-cpp",
+ "pull --ignore-pull-failures conda-python",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.clear_pull_memory()
+ compose.pull('conda-python-pandas', pull_leaf=False)
+
+
+def test_compose_pull_params(arrow_compose_path):
+ expected_calls = [
+ "pull --ignore-pull-failures conda-cpp",
+ "pull --ignore-pull-failures conda-python",
+ ]
+ compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04'))
+ expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.clear_pull_memory()
+ compose.pull('conda-python-pandas', pull_leaf=False)
+
+
+def test_compose_build(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path)
+
+ expected_calls = [
+ "build conda-cpp",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-cpp')
+
+ expected_calls = [
+ "build --no-cache conda-cpp"
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-cpp', use_cache=False)
+
+ expected_calls = [
+ "build conda-cpp",
+ "build conda-python",
+ "build conda-python-pandas"
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-python-pandas')
+
+ expected_calls = [
+ "build --no-cache conda-cpp",
+ "build --no-cache conda-python",
+ "build --no-cache conda-python-pandas",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-python-pandas', use_cache=False)
+
+ expected_calls = [
+ "build conda-cpp",
+ "build conda-python",
+ "build --no-cache conda-python-pandas",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-python-pandas', use_cache=True,
+ use_leaf_cache=False)
+
+
+@mock.patch.dict(os.environ, {"BUILDKIT_INLINE_CACHE": "1"})
+def test_compose_buildkit_inline_cache(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path)
+
+ expected_calls = [
+ "build --build-arg BUILDKIT_INLINE_CACHE=1 conda-cpp",
+ ]
+ with assert_compose_calls(compose, expected_calls):
+ compose.build('conda-cpp')
+
+
+def test_compose_build_params(arrow_compose_path):
+ expected_calls = [
+ "build ubuntu-cpp",
+ ]
+
+ compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04'))
+ expected_env = PartialEnv(UBUNTU="18.04")
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.build('ubuntu-cpp')
+
+ compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='16.04'))
+ expected_env = PartialEnv(UBUNTU="16.04")
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.build('ubuntu-cpp')
+
+ expected_calls = [
+ "build --no-cache conda-cpp",
+ "build --no-cache conda-python",
+ "build --no-cache conda-python-pandas",
+ ]
+ compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04'))
+ expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.build('conda-python-pandas', use_cache=False)
+
+
+def test_compose_run(arrow_compose_path):
+ expected_calls = [
+ format_run("conda-cpp"),
+ ]
+ compose = DockerCompose(arrow_compose_path)
+ with assert_compose_calls(compose, expected_calls):
+ compose.run('conda-cpp')
+
+ expected_calls = [
+ format_run("conda-python")
+ ]
+ expected_env = PartialEnv(PYTHON='3.6')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.run('conda-python')
+
+ compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8'))
+ expected_env = PartialEnv(PYTHON='3.8')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.run('conda-python')
+
+ compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8'))
+ for command in ["bash", "echo 1"]:
+ expected_calls = [
+ format_run(["conda-python", command]),
+ ]
+ expected_env = PartialEnv(PYTHON='3.8')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ compose.run('conda-python', command)
+
+ expected_calls = [
+ (
+ format_run("-e CONTAINER_ENV_VAR_A=a -e CONTAINER_ENV_VAR_B=b "
+ "conda-python")
+ )
+ ]
+ compose = DockerCompose(arrow_compose_path)
+ expected_env = PartialEnv(PYTHON='3.6')
+ with assert_compose_calls(compose, expected_calls, env=expected_env):
+ env = collections.OrderedDict([
+ ("CONTAINER_ENV_VAR_A", "a"),
+ ("CONTAINER_ENV_VAR_B", "b")
+ ])
+ compose.run('conda-python', env=env)
+
+ expected_calls = [
+ (
+ format_run("--volume /host/build:/build --volume "
+ "/host/ccache:/ccache:delegated conda-python")
+ )
+ ]
+ compose = DockerCompose(arrow_compose_path)
+ with assert_compose_calls(compose, expected_calls):
+ volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated")
+ compose.run('conda-python', volumes=volumes)
+
+
+def test_compose_run_with_resource_limits(arrow_compose_path):
+ expected_calls = [
+ format_run([
+ "--cpuset-cpus=0,1",
+ "--memory=7g",
+ "--memory-swap=7g",
+ "org/conda-cpp"
+ ]),
+ ]
+ compose = DockerCompose(arrow_compose_path)
+ with assert_docker_calls(compose, expected_calls):
+ compose.run('conda-cpp', resource_limit="github")
+
+
+def test_compose_push(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8'))
+ expected_env = PartialEnv(PYTHON="3.8")
+ expected_calls = [
+ mock.call(["docker", "login", "-u", "user", "-p", "pass"], check=True),
+ ]
+ for image in ["conda-cpp", "conda-python", "conda-python-pandas"]:
+ expected_calls.append(
+ mock.call(["docker-compose", "--file", str(compose.config.path),
+ "push", image], check=True, env=expected_env)
+ )
+ with assert_subprocess_calls(expected_calls):
+ compose.push('conda-python-pandas', user='user', password='pass')
+
+
+def test_compose_error(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path, params=dict(
+ PYTHON='3.8',
+ PANDAS='master'
+ ))
+
+ error = subprocess.CalledProcessError(99, [])
+ with mock.patch('subprocess.run', side_effect=error):
+ with pytest.raises(RuntimeError) as exc:
+ compose.run('conda-cpp')
+
+ exception_message = str(exc.value)
+ assert "exited with a non-zero exit code 99" in exception_message
+ assert "PANDAS: latest" in exception_message
+ assert "export PANDAS=master" in exception_message
+
+
+def test_image_with_gpu(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path)
+
+ expected_calls = [
+ [
+ "run", "--rm", "--gpus", "all",
+ "-e", "CUDA_ENV=1",
+ "-e", "OTHER_ENV=2",
+ "-v", "/host:/container:rw",
+ "org/ubuntu-cuda",
+ '/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"'
+ ]
+ ]
+ with assert_docker_calls(compose, expected_calls):
+ compose.run('ubuntu-cuda')
+
+
+def test_listing_images(arrow_compose_path):
+ compose = DockerCompose(arrow_compose_path)
+ assert sorted(compose.images()) == [
+ 'conda-cpp',
+ 'conda-python',
+ 'conda-python-dask',
+ 'conda-python-pandas',
+ 'ubuntu-c-glib',
+ 'ubuntu-cpp',
+ 'ubuntu-cpp-cmake32',
+ 'ubuntu-cuda',
+ 'ubuntu-ruby',
+ ]
diff --git a/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py b/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py
new file mode 100644
index 000000000..ab39c7b9d
--- /dev/null
+++ b/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py
@@ -0,0 +1,201 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import patch
+
+from click.testing import CliRunner
+
+from archery.docker import DockerCompose
+from archery.docker.cli import docker
+
+
+@patch.object(DockerCompose, "pull")
+@patch.object(DockerCompose, "build")
+@patch.object(DockerCompose, "run")
+def test_docker_run_with_custom_command(run, build, pull):
+ # with custom command
+ args = ["run", "ubuntu-cpp", "bash"]
+ result = CliRunner().invoke(docker, args)
+
+ assert result.exit_code == 0
+ pull.assert_called_once_with(
+ "ubuntu-cpp", pull_leaf=True, using_docker=False
+ )
+ build.assert_called_once_with(
+ "ubuntu-cpp",
+ use_cache=True,
+ use_leaf_cache=True,
+ using_docker=False,
+ using_buildx=False
+ )
+ run.assert_called_once_with(
+ "ubuntu-cpp",
+ command="bash",
+ env={},
+ resource_limit=None,
+ user=None,
+ using_docker=False,
+ volumes=(),
+ )
+
+
+@patch.object(DockerCompose, "pull")
+@patch.object(DockerCompose, "build")
+@patch.object(DockerCompose, "run")
+def test_docker_run_options(run, build, pull):
+ # environment variables and volumes
+ args = [
+ "run",
+ "-e",
+ "ARROW_GANDIVA=OFF",
+ "-e",
+ "ARROW_FLIGHT=ON",
+ "--volume",
+ "./build:/build",
+ "-v",
+ "./ccache:/ccache:delegated",
+ "-u",
+ "root",
+ "ubuntu-cpp",
+ ]
+ result = CliRunner().invoke(docker, args)
+ assert result.exit_code == 0
+ pull.assert_called_once_with(
+ "ubuntu-cpp", pull_leaf=True, using_docker=False
+ )
+ build.assert_called_once_with(
+ "ubuntu-cpp",
+ use_cache=True,
+ use_leaf_cache=True,
+ using_docker=False,
+ using_buildx=False
+ )
+ run.assert_called_once_with(
+ "ubuntu-cpp",
+ command=None,
+ env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"},
+ resource_limit=None,
+ user="root",
+ using_docker=False,
+ volumes=(
+ "./build:/build",
+ "./ccache:/ccache:delegated",
+ ),
+ )
+
+
+@patch.object(DockerCompose, "run")
+def test_docker_limit_options(run):
+ # environment variables and volumes
+ args = [
+ "run",
+ "-e",
+ "ARROW_GANDIVA=OFF",
+ "-e",
+ "ARROW_FLIGHT=ON",
+ "--volume",
+ "./build:/build",
+ "-v",
+ "./ccache:/ccache:delegated",
+ "-u",
+ "root",
+ "--resource-limit=github",
+ "--no-build",
+ "--no-pull",
+ "ubuntu-cpp",
+ ]
+ result = CliRunner().invoke(docker, args)
+ assert result.exit_code == 0
+ run.assert_called_once_with(
+ "ubuntu-cpp",
+ command=None,
+ env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"},
+ resource_limit="github",
+ user="root",
+ using_docker=False,
+ volumes=(
+ "./build:/build",
+ "./ccache:/ccache:delegated",
+ ),
+ )
+
+
+@patch.object(DockerCompose, "run")
+def test_docker_run_without_pulling_or_building(run):
+ args = ["run", "--no-pull", "--no-build", "ubuntu-cpp"]
+ result = CliRunner().invoke(docker, args)
+ assert result.exit_code == 0
+ run.assert_called_once_with(
+ "ubuntu-cpp",
+ command=None,
+ env={},
+ resource_limit=None,
+ user=None,
+ using_docker=False,
+ volumes=(),
+ )
+
+
+@patch.object(DockerCompose, "pull")
+@patch.object(DockerCompose, "build")
+def test_docker_run_only_pulling_and_building(build, pull):
+ args = ["run", "ubuntu-cpp", "--build-only"]
+ result = CliRunner().invoke(docker, args)
+ assert result.exit_code == 0
+ pull.assert_called_once_with(
+ "ubuntu-cpp", pull_leaf=True, using_docker=False
+ )
+ build.assert_called_once_with(
+ "ubuntu-cpp",
+ use_cache=True,
+ use_leaf_cache=True,
+ using_docker=False,
+ using_buildx=False
+ )
+
+
+@patch.object(DockerCompose, "build")
+@patch.object(DockerCompose, "run")
+def test_docker_run_without_build_cache(run, build):
+ args = [
+ "run",
+ "--no-pull",
+ "--force-build",
+ "--user",
+ "me",
+ "--no-cache",
+ "--no-leaf-cache",
+ "ubuntu-cpp",
+ ]
+ result = CliRunner().invoke(docker, args)
+ assert result.exit_code == 0
+ build.assert_called_once_with(
+ "ubuntu-cpp",
+ use_cache=False,
+ use_leaf_cache=False,
+ using_docker=False,
+ using_buildx=False
+ )
+ run.assert_called_once_with(
+ "ubuntu-cpp",
+ command=None,
+ env={},
+ resource_limit=None,
+ user="me",
+ using_docker=False,
+ volumes=(),
+ )
diff --git a/src/arrow/dev/archery/archery/integration/__init__.py b/src/arrow/dev/archery/archery/integration/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/arrow/dev/archery/archery/integration/datagen.py b/src/arrow/dev/archery/archery/integration/datagen.py
new file mode 100644
index 000000000..b764982bd
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/datagen.py
@@ -0,0 +1,1662 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import namedtuple, OrderedDict
+import binascii
+import json
+import os
+import random
+import tempfile
+
+import numpy as np
+
+from .util import frombytes, tobytes, random_bytes, random_utf8
+
+
+def metadata_key_values(pairs):
+ return [{'key': k, 'value': v} for k, v in pairs]
+
+
+class Field(object):
+
+ def __init__(self, name, *, nullable=True, metadata=None):
+ self.name = name
+ self.nullable = nullable
+ self.metadata = metadata or []
+
+ def get_json(self):
+ entries = [
+ ('name', self.name),
+ ('type', self._get_type()),
+ ('nullable', self.nullable),
+ ('children', self._get_children()),
+ ]
+
+ dct = self._get_dictionary()
+ if dct:
+ entries.append(('dictionary', dct))
+
+ if self.metadata is not None and len(self.metadata) > 0:
+ entries.append(('metadata', metadata_key_values(self.metadata)))
+
+ return OrderedDict(entries)
+
+ def _get_dictionary(self):
+ return None
+
+ def _make_is_valid(self, size, null_probability=0.4):
+ if self.nullable:
+ return (np.random.random_sample(size) > null_probability
+ ).astype(np.int8)
+ else:
+ return np.ones(size, dtype=np.int8)
+
+
+class Column(object):
+
+ def __init__(self, name, count):
+ self.name = name
+ self.count = count
+
+ def __len__(self):
+ return self.count
+
+ def _get_children(self):
+ return []
+
+ def _get_buffers(self):
+ return []
+
+ def get_json(self):
+ entries = [
+ ('name', self.name),
+ ('count', self.count)
+ ]
+
+ buffers = self._get_buffers()
+ entries.extend(buffers)
+
+ children = self._get_children()
+ if len(children) > 0:
+ entries.append(('children', children))
+
+ return OrderedDict(entries)
+
+
+class PrimitiveField(Field):
+
+ def _get_children(self):
+ return []
+
+
+class PrimitiveColumn(Column):
+
+ def __init__(self, name, count, is_valid, values):
+ super().__init__(name, count)
+ self.is_valid = is_valid
+ self.values = values
+
+ def _encode_value(self, x):
+ return x
+
+ def _get_buffers(self):
+ return [
+ ('VALIDITY', [int(v) for v in self.is_valid]),
+ ('DATA', list([self._encode_value(x) for x in self.values]))
+ ]
+
+
+class NullColumn(Column):
+ # This subclass is for readability only
+ pass
+
+
+class NullField(PrimitiveField):
+
+ def __init__(self, name, metadata=None):
+ super().__init__(name, nullable=True,
+ metadata=metadata)
+
+ def _get_type(self):
+ return OrderedDict([('name', 'null')])
+
+ def generate_column(self, size, name=None):
+ return NullColumn(name or self.name, size)
+
+
+TEST_INT_MAX = 2 ** 31 - 1
+TEST_INT_MIN = ~TEST_INT_MAX
+
+
+class IntegerField(PrimitiveField):
+
+ def __init__(self, name, is_signed, bit_width, *, nullable=True,
+ metadata=None,
+ min_value=TEST_INT_MIN,
+ max_value=TEST_INT_MAX):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ self.is_signed = is_signed
+ self.bit_width = bit_width
+ self.min_value = min_value
+ self.max_value = max_value
+
+ def _get_generated_data_bounds(self):
+ if self.is_signed:
+ signed_iinfo = np.iinfo('int' + str(self.bit_width))
+ min_value, max_value = signed_iinfo.min, signed_iinfo.max
+ else:
+ unsigned_iinfo = np.iinfo('uint' + str(self.bit_width))
+ min_value, max_value = 0, unsigned_iinfo.max
+
+ lower_bound = max(min_value, self.min_value)
+ upper_bound = min(max_value, self.max_value)
+ return lower_bound, upper_bound
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'int'),
+ ('isSigned', self.is_signed),
+ ('bitWidth', self.bit_width)
+ ])
+
+ def generate_column(self, size, name=None):
+ lower_bound, upper_bound = self._get_generated_data_bounds()
+ return self.generate_range(size, lower_bound, upper_bound,
+ name=name, include_extremes=True)
+
+ def generate_range(self, size, lower, upper, name=None,
+ include_extremes=False):
+ values = np.random.randint(lower, upper, size=size, dtype=np.int64)
+ if include_extremes and size >= 2:
+ values[:2] = [lower, upper]
+ values = list(map(int if self.bit_width < 64 else str, values))
+
+ is_valid = self._make_is_valid(size)
+
+ if name is None:
+ name = self.name
+ return PrimitiveColumn(name, size, is_valid, values)
+
+
+class DateField(IntegerField):
+
+ DAY = 0
+ MILLISECOND = 1
+
+ # 1/1/1 to 12/31/9999
+ _ranges = {
+ DAY: [-719162, 2932896],
+ MILLISECOND: [-62135596800000, 253402214400000]
+ }
+
+ def __init__(self, name, unit, *, nullable=True, metadata=None):
+ bit_width = 32 if unit == self.DAY else 64
+
+ min_value, max_value = self._ranges[unit]
+ super().__init__(
+ name, True, bit_width,
+ nullable=nullable, metadata=metadata,
+ min_value=min_value, max_value=max_value
+ )
+ self.unit = unit
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'date'),
+ ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
+ ])
+
+
+TIMEUNIT_NAMES = {
+ 's': 'SECOND',
+ 'ms': 'MILLISECOND',
+ 'us': 'MICROSECOND',
+ 'ns': 'NANOSECOND'
+}
+
+
+class TimeField(IntegerField):
+
+ BIT_WIDTHS = {
+ 's': 32,
+ 'ms': 32,
+ 'us': 64,
+ 'ns': 64
+ }
+
+ _ranges = {
+ 's': [0, 86400],
+ 'ms': [0, 86400000],
+ 'us': [0, 86400000000],
+ 'ns': [0, 86400000000000]
+ }
+
+ def __init__(self, name, unit='s', *, nullable=True,
+ metadata=None):
+ min_val, max_val = self._ranges[unit]
+ super().__init__(name, True, self.BIT_WIDTHS[unit],
+ nullable=nullable, metadata=metadata,
+ min_value=min_val, max_value=max_val)
+ self.unit = unit
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'time'),
+ ('unit', TIMEUNIT_NAMES[self.unit]),
+ ('bitWidth', self.bit_width)
+ ])
+
+
+class TimestampField(IntegerField):
+
+ # 1/1/1 to 12/31/9999
+ _ranges = {
+ 's': [-62135596800, 253402214400],
+ 'ms': [-62135596800000, 253402214400000],
+ 'us': [-62135596800000000, 253402214400000000],
+
+ # Physical range for int64, ~584 years and change
+ 'ns': [np.iinfo('int64').min, np.iinfo('int64').max]
+ }
+
+ def __init__(self, name, unit='s', tz=None, *, nullable=True,
+ metadata=None):
+ min_val, max_val = self._ranges[unit]
+ super().__init__(name, True, 64,
+ nullable=nullable,
+ metadata=metadata,
+ min_value=min_val,
+ max_value=max_val)
+ self.unit = unit
+ self.tz = tz
+
+ def _get_type(self):
+ fields = [
+ ('name', 'timestamp'),
+ ('unit', TIMEUNIT_NAMES[self.unit])
+ ]
+
+ if self.tz is not None:
+ fields.append(('timezone', self.tz))
+
+ return OrderedDict(fields)
+
+
+class DurationIntervalField(IntegerField):
+
+ def __init__(self, name, unit='s', *, nullable=True,
+ metadata=None):
+ min_val, max_val = np.iinfo('int64').min, np.iinfo('int64').max,
+ super().__init__(
+ name, True, 64,
+ nullable=nullable, metadata=metadata,
+ min_value=min_val, max_value=max_val)
+ self.unit = unit
+
+ def _get_type(self):
+ fields = [
+ ('name', 'duration'),
+ ('unit', TIMEUNIT_NAMES[self.unit])
+ ]
+
+ return OrderedDict(fields)
+
+
+class YearMonthIntervalField(IntegerField):
+ def __init__(self, name, *, nullable=True, metadata=None):
+ min_val, max_val = [-10000*12, 10000*12] # +/- 10000 years.
+ super().__init__(
+ name, True, 32,
+ nullable=nullable, metadata=metadata,
+ min_value=min_val, max_value=max_val)
+
+ def _get_type(self):
+ fields = [
+ ('name', 'interval'),
+ ('unit', 'YEAR_MONTH'),
+ ]
+
+ return OrderedDict(fields)
+
+
+class DayTimeIntervalField(PrimitiveField):
+ def __init__(self, name, *, nullable=True, metadata=None):
+ super().__init__(name,
+ nullable=True,
+ metadata=metadata)
+
+ @property
+ def numpy_type(self):
+ return object
+
+ def _get_type(self):
+
+ return OrderedDict([
+ ('name', 'interval'),
+ ('unit', 'DAY_TIME'),
+ ])
+
+ def generate_column(self, size, name=None):
+ min_day_value, max_day_value = -10000*366, 10000*366
+ values = [{'days': random.randint(min_day_value, max_day_value),
+ 'milliseconds': random.randint(-86400000, +86400000)}
+ for _ in range(size)]
+
+ is_valid = self._make_is_valid(size)
+ if name is None:
+ name = self.name
+ return PrimitiveColumn(name, size, is_valid, values)
+
+
+class MonthDayNanoIntervalField(PrimitiveField):
+ def __init__(self, name, *, nullable=True, metadata=None):
+ super().__init__(name,
+ nullable=True,
+ metadata=metadata)
+
+ @property
+ def numpy_type(self):
+ return object
+
+ def _get_type(self):
+
+ return OrderedDict([
+ ('name', 'interval'),
+ ('unit', 'MONTH_DAY_NANO'),
+ ])
+
+ def generate_column(self, size, name=None):
+ I32 = 'int32'
+ min_int_value, max_int_value = np.iinfo(I32).min, np.iinfo(I32).max
+ I64 = 'int64'
+ min_nano_val, max_nano_val = np.iinfo(I64).min, np.iinfo(I64).max,
+ values = [{'months': random.randint(min_int_value, max_int_value),
+ 'days': random.randint(min_int_value, max_int_value),
+ 'nanoseconds': random.randint(min_nano_val, max_nano_val)}
+ for _ in range(size)]
+
+ is_valid = self._make_is_valid(size)
+ if name is None:
+ name = self.name
+ return PrimitiveColumn(name, size, is_valid, values)
+
+
+class FloatingPointField(PrimitiveField):
+
+ def __init__(self, name, bit_width, *, nullable=True,
+ metadata=None):
+ super().__init__(name,
+ nullable=nullable,
+ metadata=metadata)
+
+ self.bit_width = bit_width
+ self.precision = {
+ 16: 'HALF',
+ 32: 'SINGLE',
+ 64: 'DOUBLE'
+ }[self.bit_width]
+
+ @property
+ def numpy_type(self):
+ return 'float' + str(self.bit_width)
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'floatingpoint'),
+ ('precision', self.precision)
+ ])
+
+ def generate_column(self, size, name=None):
+ values = np.random.randn(size) * 1000
+ values = np.round(values, 3)
+
+ is_valid = self._make_is_valid(size)
+ if name is None:
+ name = self.name
+ return PrimitiveColumn(name, size, is_valid, values)
+
+
+DECIMAL_PRECISION_TO_VALUE = {
+ key: (1 << (8 * i - 1)) - 1 for i, key in enumerate(
+ [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36,
+ 40, 42, 44, 50, 60, 70],
+ start=1,
+ )
+}
+
+
+def decimal_range_from_precision(precision):
+ assert 1 <= precision <= 76
+ try:
+ max_value = DECIMAL_PRECISION_TO_VALUE[precision]
+ except KeyError:
+ return decimal_range_from_precision(precision - 1)
+ else:
+ return ~max_value, max_value
+
+
+class DecimalField(PrimitiveField):
+ def __init__(self, name, precision, scale, bit_width, *,
+ nullable=True, metadata=None):
+ super().__init__(name, nullable=True,
+ metadata=metadata)
+ self.precision = precision
+ self.scale = scale
+ self.bit_width = bit_width
+
+ @property
+ def numpy_type(self):
+ return object
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'decimal'),
+ ('precision', self.precision),
+ ('scale', self.scale),
+ ('bitWidth', self.bit_width),
+ ])
+
+ def generate_column(self, size, name=None):
+ min_value, max_value = decimal_range_from_precision(self.precision)
+ values = [random.randint(min_value, max_value) for _ in range(size)]
+
+ is_valid = self._make_is_valid(size)
+ if name is None:
+ name = self.name
+ return DecimalColumn(name, size, is_valid, values, self.bit_width)
+
+
+class DecimalColumn(PrimitiveColumn):
+
+ def __init__(self, name, count, is_valid, values, bit_width):
+ super().__init__(name, count, is_valid, values)
+ self.bit_width = bit_width
+
+ def _encode_value(self, x):
+ return str(x)
+
+
+class BooleanField(PrimitiveField):
+ bit_width = 1
+
+ def _get_type(self):
+ return OrderedDict([('name', 'bool')])
+
+ @property
+ def numpy_type(self):
+ return 'bool'
+
+ def generate_column(self, size, name=None):
+ values = list(map(bool, np.random.randint(0, 2, size=size)))
+ is_valid = self._make_is_valid(size)
+ if name is None:
+ name = self.name
+ return PrimitiveColumn(name, size, is_valid, values)
+
+
+class FixedSizeBinaryField(PrimitiveField):
+
+ def __init__(self, name, byte_width, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ self.byte_width = byte_width
+
+ @property
+ def numpy_type(self):
+ return object
+
+ @property
+ def column_class(self):
+ return FixedSizeBinaryColumn
+
+ def _get_type(self):
+ return OrderedDict([('name', 'fixedsizebinary'),
+ ('byteWidth', self.byte_width)])
+
+ def generate_column(self, size, name=None):
+ is_valid = self._make_is_valid(size)
+ values = []
+
+ for i in range(size):
+ values.append(random_bytes(self.byte_width))
+
+ if name is None:
+ name = self.name
+ return self.column_class(name, size, is_valid, values)
+
+
+class BinaryField(PrimitiveField):
+
+ @property
+ def numpy_type(self):
+ return object
+
+ @property
+ def column_class(self):
+ return BinaryColumn
+
+ def _get_type(self):
+ return OrderedDict([('name', 'binary')])
+
+ def _random_sizes(self, size):
+ return np.random.exponential(scale=4, size=size).astype(np.int32)
+
+ def generate_column(self, size, name=None):
+ is_valid = self._make_is_valid(size)
+ values = []
+
+ sizes = self._random_sizes(size)
+
+ for i, nbytes in enumerate(sizes):
+ if is_valid[i]:
+ values.append(random_bytes(nbytes))
+ else:
+ values.append(b"")
+
+ if name is None:
+ name = self.name
+ return self.column_class(name, size, is_valid, values)
+
+
+class StringField(BinaryField):
+
+ @property
+ def column_class(self):
+ return StringColumn
+
+ def _get_type(self):
+ return OrderedDict([('name', 'utf8')])
+
+ def generate_column(self, size, name=None):
+ K = 7
+ is_valid = self._make_is_valid(size)
+ values = []
+
+ for i in range(size):
+ if is_valid[i]:
+ values.append(tobytes(random_utf8(K)))
+ else:
+ values.append(b"")
+
+ if name is None:
+ name = self.name
+ return self.column_class(name, size, is_valid, values)
+
+
+class LargeBinaryField(BinaryField):
+
+ @property
+ def column_class(self):
+ return LargeBinaryColumn
+
+ def _get_type(self):
+ return OrderedDict([('name', 'largebinary')])
+
+
+class LargeStringField(StringField):
+
+ @property
+ def column_class(self):
+ return LargeStringColumn
+
+ def _get_type(self):
+ return OrderedDict([('name', 'largeutf8')])
+
+
+class Schema(object):
+
+ def __init__(self, fields, metadata=None):
+ self.fields = fields
+ self.metadata = metadata
+
+ def get_json(self):
+ entries = [
+ ('fields', [field.get_json() for field in self.fields])
+ ]
+
+ if self.metadata is not None and len(self.metadata) > 0:
+ entries.append(('metadata', metadata_key_values(self.metadata)))
+
+ return OrderedDict(entries)
+
+
+class _NarrowOffsetsMixin:
+
+ def _encode_offsets(self, offsets):
+ return list(map(int, offsets))
+
+
+class _LargeOffsetsMixin:
+
+ def _encode_offsets(self, offsets):
+ # 64-bit offsets have to be represented as strings to roundtrip
+ # through JSON.
+ return list(map(str, offsets))
+
+
+class _BaseBinaryColumn(PrimitiveColumn):
+
+ def _encode_value(self, x):
+ return frombytes(binascii.hexlify(x).upper())
+
+ def _get_buffers(self):
+ offset = 0
+ offsets = [0]
+
+ data = []
+ for i, v in enumerate(self.values):
+ if self.is_valid[i]:
+ offset += len(v)
+ else:
+ v = b""
+
+ offsets.append(offset)
+ data.append(self._encode_value(v))
+
+ return [
+ ('VALIDITY', [int(x) for x in self.is_valid]),
+ ('OFFSET', self._encode_offsets(offsets)),
+ ('DATA', data)
+ ]
+
+
+class _BaseStringColumn(_BaseBinaryColumn):
+
+ def _encode_value(self, x):
+ return frombytes(x)
+
+
+class BinaryColumn(_BaseBinaryColumn, _NarrowOffsetsMixin):
+ pass
+
+
+class StringColumn(_BaseStringColumn, _NarrowOffsetsMixin):
+ pass
+
+
+class LargeBinaryColumn(_BaseBinaryColumn, _LargeOffsetsMixin):
+ pass
+
+
+class LargeStringColumn(_BaseStringColumn, _LargeOffsetsMixin):
+ pass
+
+
+class FixedSizeBinaryColumn(PrimitiveColumn):
+
+ def _encode_value(self, x):
+ return frombytes(binascii.hexlify(x).upper())
+
+ def _get_buffers(self):
+ data = []
+ for i, v in enumerate(self.values):
+ data.append(self._encode_value(v))
+
+ return [
+ ('VALIDITY', [int(x) for x in self.is_valid]),
+ ('DATA', data)
+ ]
+
+
+class ListField(Field):
+
+ def __init__(self, name, value_field, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ self.value_field = value_field
+
+ @property
+ def column_class(self):
+ return ListColumn
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'list')
+ ])
+
+ def _get_children(self):
+ return [self.value_field.get_json()]
+
+ def generate_column(self, size, name=None):
+ MAX_LIST_SIZE = 4
+
+ is_valid = self._make_is_valid(size)
+ list_sizes = np.random.randint(0, MAX_LIST_SIZE + 1, size=size)
+ offsets = [0]
+
+ offset = 0
+ for i in range(size):
+ if is_valid[i]:
+ offset += int(list_sizes[i])
+ offsets.append(offset)
+
+ # The offset now is the total number of elements in the child array
+ values = self.value_field.generate_column(offset)
+
+ if name is None:
+ name = self.name
+ return self.column_class(name, size, is_valid, offsets, values)
+
+
+class LargeListField(ListField):
+
+ @property
+ def column_class(self):
+ return LargeListColumn
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'largelist')
+ ])
+
+
+class _BaseListColumn(Column):
+
+ def __init__(self, name, count, is_valid, offsets, values):
+ super().__init__(name, count)
+ self.is_valid = is_valid
+ self.offsets = offsets
+ self.values = values
+
+ def _get_buffers(self):
+ return [
+ ('VALIDITY', [int(v) for v in self.is_valid]),
+ ('OFFSET', self._encode_offsets(self.offsets))
+ ]
+
+ def _get_children(self):
+ return [self.values.get_json()]
+
+
+class ListColumn(_BaseListColumn, _NarrowOffsetsMixin):
+ pass
+
+
+class LargeListColumn(_BaseListColumn, _LargeOffsetsMixin):
+ pass
+
+
+class MapField(Field):
+
+ def __init__(self, name, key_field, item_field, *, nullable=True,
+ metadata=None, keys_sorted=False, entries_name='entries'):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+
+ assert not key_field.nullable
+ self.key_field = key_field
+ self.item_field = item_field
+ self.pair_field = StructField(entries_name, [key_field, item_field],
+ nullable=False)
+ self.keys_sorted = keys_sorted
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'map'),
+ ('keysSorted', self.keys_sorted)
+ ])
+
+ def _get_children(self):
+ return [self.pair_field.get_json()]
+
+ def generate_column(self, size, name=None):
+ MAX_MAP_SIZE = 4
+
+ is_valid = self._make_is_valid(size)
+ map_sizes = np.random.randint(0, MAX_MAP_SIZE + 1, size=size)
+ offsets = [0]
+
+ offset = 0
+ for i in range(size):
+ if is_valid[i]:
+ offset += int(map_sizes[i])
+ offsets.append(offset)
+
+ # The offset now is the total number of elements in the child array
+ pairs = self.pair_field.generate_column(offset)
+ if name is None:
+ name = self.name
+
+ return MapColumn(name, size, is_valid, offsets, pairs)
+
+
+class MapColumn(Column):
+
+ def __init__(self, name, count, is_valid, offsets, pairs):
+ super().__init__(name, count)
+ self.is_valid = is_valid
+ self.offsets = offsets
+ self.pairs = pairs
+
+ def _get_buffers(self):
+ return [
+ ('VALIDITY', [int(v) for v in self.is_valid]),
+ ('OFFSET', list(self.offsets))
+ ]
+
+ def _get_children(self):
+ return [self.pairs.get_json()]
+
+
+class FixedSizeListField(Field):
+
+ def __init__(self, name, value_field, list_size, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ self.value_field = value_field
+ self.list_size = list_size
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'fixedsizelist'),
+ ('listSize', self.list_size)
+ ])
+
+ def _get_children(self):
+ return [self.value_field.get_json()]
+
+ def generate_column(self, size, name=None):
+ is_valid = self._make_is_valid(size)
+ values = self.value_field.generate_column(size * self.list_size)
+
+ if name is None:
+ name = self.name
+ return FixedSizeListColumn(name, size, is_valid, values)
+
+
+class FixedSizeListColumn(Column):
+
+ def __init__(self, name, count, is_valid, values):
+ super().__init__(name, count)
+ self.is_valid = is_valid
+ self.values = values
+
+ def _get_buffers(self):
+ return [
+ ('VALIDITY', [int(v) for v in self.is_valid])
+ ]
+
+ def _get_children(self):
+ return [self.values.get_json()]
+
+
+class StructField(Field):
+
+ def __init__(self, name, fields, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ self.fields = fields
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'struct')
+ ])
+
+ def _get_children(self):
+ return [field.get_json() for field in self.fields]
+
+ def generate_column(self, size, name=None):
+ is_valid = self._make_is_valid(size)
+
+ field_values = [field.generate_column(size) for field in self.fields]
+ if name is None:
+ name = self.name
+ return StructColumn(name, size, is_valid, field_values)
+
+
+class _BaseUnionField(Field):
+
+ def __init__(self, name, fields, type_ids=None, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable, metadata=metadata)
+ if type_ids is None:
+ type_ids = list(range(fields))
+ else:
+ assert len(fields) == len(type_ids)
+ self.fields = fields
+ self.type_ids = type_ids
+ assert all(x >= 0 for x in self.type_ids)
+
+ def _get_type(self):
+ return OrderedDict([
+ ('name', 'union'),
+ ('mode', self.mode),
+ ('typeIds', self.type_ids),
+ ])
+
+ def _get_children(self):
+ return [field.get_json() for field in self.fields]
+
+ def _make_type_ids(self, size):
+ return np.random.choice(self.type_ids, size)
+
+
+class SparseUnionField(_BaseUnionField):
+ mode = 'SPARSE'
+
+ def generate_column(self, size, name=None):
+ array_type_ids = self._make_type_ids(size)
+ field_values = [field.generate_column(size) for field in self.fields]
+
+ if name is None:
+ name = self.name
+ return SparseUnionColumn(name, size, array_type_ids, field_values)
+
+
+class DenseUnionField(_BaseUnionField):
+ mode = 'DENSE'
+
+ def generate_column(self, size, name=None):
+ # Reverse mapping {logical type id => physical child id}
+ child_ids = [None] * (max(self.type_ids) + 1)
+ for i, type_id in enumerate(self.type_ids):
+ child_ids[type_id] = i
+
+ array_type_ids = self._make_type_ids(size)
+ offsets = []
+ child_sizes = [0] * len(self.fields)
+
+ for i in range(size):
+ child_id = child_ids[array_type_ids[i]]
+ offset = child_sizes[child_id]
+ offsets.append(offset)
+ child_sizes[child_id] = offset + 1
+
+ field_values = [
+ field.generate_column(child_size)
+ for field, child_size in zip(self.fields, child_sizes)]
+
+ if name is None:
+ name = self.name
+ return DenseUnionColumn(name, size, array_type_ids, offsets,
+ field_values)
+
+
+class Dictionary(object):
+
+ def __init__(self, id_, field, size, name=None, ordered=False):
+ self.id_ = id_
+ self.field = field
+ self.values = field.generate_column(size=size, name=name)
+ self.ordered = ordered
+
+ def __len__(self):
+ return len(self.values)
+
+ def get_json(self):
+ dummy_batch = RecordBatch(len(self.values), [self.values])
+ return OrderedDict([
+ ('id', self.id_),
+ ('data', dummy_batch.get_json())
+ ])
+
+
+class DictionaryField(Field):
+
+ def __init__(self, name, index_field, dictionary, *, nullable=True,
+ metadata=None):
+ super().__init__(name, nullable=nullable,
+ metadata=metadata)
+ assert index_field.name == ''
+ assert isinstance(index_field, IntegerField)
+ assert isinstance(dictionary, Dictionary)
+
+ self.index_field = index_field
+ self.dictionary = dictionary
+
+ def _get_type(self):
+ return self.dictionary.field._get_type()
+
+ def _get_children(self):
+ return self.dictionary.field._get_children()
+
+ def _get_dictionary(self):
+ return OrderedDict([
+ ('id', self.dictionary.id_),
+ ('indexType', self.index_field._get_type()),
+ ('isOrdered', self.dictionary.ordered)
+ ])
+
+ def generate_column(self, size, name=None):
+ if name is None:
+ name = self.name
+ return self.index_field.generate_range(size, 0, len(self.dictionary),
+ name=name)
+
+
+ExtensionType = namedtuple(
+ 'ExtensionType', ['extension_name', 'serialized', 'storage_field'])
+
+
+class ExtensionField(Field):
+
+ def __init__(self, name, extension_type, *, nullable=True, metadata=None):
+ metadata = (metadata or []) + [
+ ('ARROW:extension:name', extension_type.extension_name),
+ ('ARROW:extension:metadata', extension_type.serialized),
+ ]
+ super().__init__(name, nullable=nullable, metadata=metadata)
+ self.extension_type = extension_type
+
+ def _get_type(self):
+ return self.extension_type.storage_field._get_type()
+
+ def _get_children(self):
+ return self.extension_type.storage_field._get_children()
+
+ def _get_dictionary(self):
+ return self.extension_type.storage_field._get_dictionary()
+
+ def generate_column(self, size, name=None):
+ if name is None:
+ name = self.name
+ return self.extension_type.storage_field.generate_column(size, name)
+
+
+class StructColumn(Column):
+
+ def __init__(self, name, count, is_valid, field_values):
+ super().__init__(name, count)
+ self.is_valid = is_valid
+ self.field_values = field_values
+
+ def _get_buffers(self):
+ return [
+ ('VALIDITY', [int(v) for v in self.is_valid])
+ ]
+
+ def _get_children(self):
+ return [field.get_json() for field in self.field_values]
+
+
+class SparseUnionColumn(Column):
+
+ def __init__(self, name, count, type_ids, field_values):
+ super().__init__(name, count)
+ self.type_ids = type_ids
+ self.field_values = field_values
+
+ def _get_buffers(self):
+ return [
+ ('TYPE_ID', [int(v) for v in self.type_ids])
+ ]
+
+ def _get_children(self):
+ return [field.get_json() for field in self.field_values]
+
+
+class DenseUnionColumn(Column):
+
+ def __init__(self, name, count, type_ids, offsets, field_values):
+ super().__init__(name, count)
+ self.type_ids = type_ids
+ self.offsets = offsets
+ self.field_values = field_values
+
+ def _get_buffers(self):
+ return [
+ ('TYPE_ID', [int(v) for v in self.type_ids]),
+ ('OFFSET', [int(v) for v in self.offsets]),
+ ]
+
+ def _get_children(self):
+ return [field.get_json() for field in self.field_values]
+
+
+class RecordBatch(object):
+
+ def __init__(self, count, columns):
+ self.count = count
+ self.columns = columns
+
+ def get_json(self):
+ return OrderedDict([
+ ('count', self.count),
+ ('columns', [col.get_json() for col in self.columns])
+ ])
+
+
+class File(object):
+
+ def __init__(self, name, schema, batches, dictionaries=None,
+ skip=None, path=None):
+ self.name = name
+ self.schema = schema
+ self.dictionaries = dictionaries or []
+ self.batches = batches
+ self.skip = set()
+ self.path = path
+ if skip:
+ self.skip.update(skip)
+
+ def get_json(self):
+ entries = [
+ ('schema', self.schema.get_json())
+ ]
+
+ if len(self.dictionaries) > 0:
+ entries.append(('dictionaries',
+ [dictionary.get_json()
+ for dictionary in self.dictionaries]))
+
+ entries.append(('batches', [batch.get_json()
+ for batch in self.batches]))
+ return OrderedDict(entries)
+
+ def write(self, path):
+ with open(path, 'wb') as f:
+ f.write(json.dumps(self.get_json(), indent=2).encode('utf-8'))
+ self.path = path
+
+ def skip_category(self, category):
+ """Skip this test for the given category.
+
+ Category should be SKIP_ARROW or SKIP_FLIGHT.
+ """
+ self.skip.add(category)
+ return self
+
+
+def get_field(name, type_, **kwargs):
+ if type_ == 'binary':
+ return BinaryField(name, **kwargs)
+ elif type_ == 'utf8':
+ return StringField(name, **kwargs)
+ elif type_ == 'largebinary':
+ return LargeBinaryField(name, **kwargs)
+ elif type_ == 'largeutf8':
+ return LargeStringField(name, **kwargs)
+ elif type_.startswith('fixedsizebinary_'):
+ byte_width = int(type_.split('_')[1])
+ return FixedSizeBinaryField(name, byte_width=byte_width, **kwargs)
+
+ dtype = np.dtype(type_)
+
+ if dtype.kind in ('i', 'u'):
+ signed = dtype.kind == 'i'
+ bit_width = dtype.itemsize * 8
+ return IntegerField(name, signed, bit_width, **kwargs)
+ elif dtype.kind == 'f':
+ bit_width = dtype.itemsize * 8
+ return FloatingPointField(name, bit_width, **kwargs)
+ elif dtype.kind == 'b':
+ return BooleanField(name, **kwargs)
+ else:
+ raise TypeError(dtype)
+
+
+def _generate_file(name, fields, batch_sizes, dictionaries=None, skip=None,
+ metadata=None):
+ schema = Schema(fields, metadata=metadata)
+ batches = []
+ for size in batch_sizes:
+ columns = []
+ for field in fields:
+ col = field.generate_column(size)
+ columns.append(col)
+
+ batches.append(RecordBatch(size, columns))
+
+ return File(name, schema, batches, dictionaries, skip=skip)
+
+
+def generate_custom_metadata_case():
+ def meta(items):
+ # Generate a simple block of metadata where each value is '{}'.
+ # Keys are delimited by whitespace in `items`.
+ return [(k, '{}') for k in items.split()]
+
+ fields = [
+ get_field('sort_of_pandas', 'int8', metadata=meta('pandas')),
+
+ get_field('lots_of_meta', 'int8', metadata=meta('a b c d .. w x y z')),
+
+ get_field(
+ 'unregistered_extension', 'int8',
+ metadata=[
+ ('ARROW:extension:name', '!nonexistent'),
+ ('ARROW:extension:metadata', ''),
+ ('ARROW:integration:allow_unregistered_extension', 'true'),
+ ]),
+
+ ListField('list_with_odd_values',
+ get_field('item', 'int32', metadata=meta('odd_values'))),
+ ]
+
+ batch_sizes = [1]
+ return _generate_file('custom_metadata', fields, batch_sizes,
+ metadata=meta('schema_custom_0 schema_custom_1'))
+
+
+def generate_duplicate_fieldnames_case():
+ fields = [
+ get_field('ints', 'int8'),
+ get_field('ints', 'int32'),
+
+ StructField('struct', [get_field('', 'int32'), get_field('', 'utf8')]),
+ ]
+
+ batch_sizes = [1]
+ return _generate_file('duplicate_fieldnames', fields, batch_sizes)
+
+
+def generate_primitive_case(batch_sizes, name='primitive'):
+ types = ['bool', 'int8', 'int16', 'int32', 'int64',
+ 'uint8', 'uint16', 'uint32', 'uint64',
+ 'float32', 'float64', 'binary', 'utf8',
+ 'fixedsizebinary_19', 'fixedsizebinary_120']
+
+ fields = []
+
+ for type_ in types:
+ fields.append(get_field(type_ + "_nullable", type_, nullable=True))
+ fields.append(get_field(type_ + "_nonnullable", type_, nullable=False))
+
+ return _generate_file(name, fields, batch_sizes)
+
+
+def generate_primitive_large_offsets_case(batch_sizes):
+ types = ['largebinary', 'largeutf8']
+
+ fields = []
+
+ for type_ in types:
+ fields.append(get_field(type_ + "_nullable", type_, nullable=True))
+ fields.append(get_field(type_ + "_nonnullable", type_, nullable=False))
+
+ return _generate_file('primitive_large_offsets', fields, batch_sizes)
+
+
+def generate_null_case(batch_sizes):
+ # Interleave null with non-null types to ensure the appropriate number of
+ # buffers (0) is read and written
+ fields = [
+ NullField(name='f0'),
+ get_field('f1', 'int32'),
+ NullField(name='f2'),
+ get_field('f3', 'float64'),
+ NullField(name='f4')
+ ]
+ return _generate_file('null', fields, batch_sizes)
+
+
+def generate_null_trivial_case(batch_sizes):
+ # Generate a case with no buffers
+ fields = [
+ NullField(name='f0'),
+ ]
+ return _generate_file('null_trivial', fields, batch_sizes)
+
+
+def generate_decimal128_case():
+ fields = [
+ DecimalField(name='f{}'.format(i), precision=precision, scale=2,
+ bit_width=128)
+ for i, precision in enumerate(range(3, 39))
+ ]
+
+ possible_batch_sizes = 7, 10
+ batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]
+ # 'decimal' is the original name for the test, and it must match
+ # provide "gold" files that test backwards compatibility, so they
+ # can be appropriately skipped.
+ return _generate_file('decimal', fields, batch_sizes)
+
+
+def generate_decimal256_case():
+ fields = [
+ DecimalField(name='f{}'.format(i), precision=precision, scale=5,
+ bit_width=256)
+ for i, precision in enumerate(range(37, 70))
+ ]
+
+ possible_batch_sizes = 7, 10
+ batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]
+ return _generate_file('decimal256', fields, batch_sizes)
+
+
+def generate_datetime_case():
+ fields = [
+ DateField('f0', DateField.DAY),
+ DateField('f1', DateField.MILLISECOND),
+ TimeField('f2', 's'),
+ TimeField('f3', 'ms'),
+ TimeField('f4', 'us'),
+ TimeField('f5', 'ns'),
+ TimestampField('f6', 's'),
+ TimestampField('f7', 'ms'),
+ TimestampField('f8', 'us'),
+ TimestampField('f9', 'ns'),
+ TimestampField('f10', 'ms', tz=None),
+ TimestampField('f11', 's', tz='UTC'),
+ TimestampField('f12', 'ms', tz='US/Eastern'),
+ TimestampField('f13', 'us', tz='Europe/Paris'),
+ TimestampField('f14', 'ns', tz='US/Pacific'),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("datetime", fields, batch_sizes)
+
+
+def generate_interval_case():
+ fields = [
+ DurationIntervalField('f1', 's'),
+ DurationIntervalField('f2', 'ms'),
+ DurationIntervalField('f3', 'us'),
+ DurationIntervalField('f4', 'ns'),
+ YearMonthIntervalField('f5'),
+ DayTimeIntervalField('f6'),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("interval", fields, batch_sizes)
+
+
+def generate_month_day_nano_interval_case():
+ fields = [
+ MonthDayNanoIntervalField('f1'),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("interval_mdn", fields, batch_sizes)
+
+
+def generate_map_case():
+ fields = [
+ MapField('map_nullable', get_field('key', 'utf8', nullable=False),
+ get_field('value', 'int32')),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("map", fields, batch_sizes)
+
+
+def generate_non_canonical_map_case():
+ fields = [
+ MapField('map_other_names',
+ get_field('some_key', 'utf8', nullable=False),
+ get_field('some_value', 'int32'),
+ entries_name='some_entries'),
+ ]
+
+ batch_sizes = [7]
+ return _generate_file("map_non_canonical", fields, batch_sizes)
+
+
+def generate_nested_case():
+ fields = [
+ ListField('list_nullable', get_field('item', 'int32')),
+ FixedSizeListField('fixedsizelist_nullable',
+ get_field('item', 'int32'), 4),
+ StructField('struct_nullable', [get_field('f1', 'int32'),
+ get_field('f2', 'utf8')]),
+ # Fails on Go (ARROW-8452)
+ # ListField('list_nonnullable', get_field('item', 'int32'),
+ # nullable=False),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("nested", fields, batch_sizes)
+
+
+def generate_recursive_nested_case():
+ fields = [
+ ListField('lists_list',
+ ListField('inner_list', get_field('item', 'int16'))),
+ ListField('structs_list',
+ StructField('inner_struct',
+ [get_field('f1', 'int32'),
+ get_field('f2', 'utf8')])),
+ ]
+
+ batch_sizes = [7, 10]
+ return _generate_file("recursive_nested", fields, batch_sizes)
+
+
+def generate_nested_large_offsets_case():
+ fields = [
+ LargeListField('large_list_nullable', get_field('item', 'int32')),
+ LargeListField('large_list_nonnullable',
+ get_field('item', 'int32'), nullable=False),
+ LargeListField('large_list_nested',
+ ListField('inner_list', get_field('item', 'int16'))),
+ ]
+
+ batch_sizes = [0, 13]
+ return _generate_file("nested_large_offsets", fields, batch_sizes)
+
+
+def generate_unions_case():
+ fields = [
+ SparseUnionField('sparse', [get_field('f1', 'int32'),
+ get_field('f2', 'utf8')],
+ type_ids=[5, 7]),
+ DenseUnionField('dense', [get_field('f1', 'int16'),
+ get_field('f2', 'binary')],
+ type_ids=[10, 20]),
+ SparseUnionField('sparse', [get_field('f1', 'float32', nullable=False),
+ get_field('f2', 'bool')],
+ type_ids=[5, 7], nullable=False),
+ DenseUnionField('dense', [get_field('f1', 'uint8', nullable=False),
+ get_field('f2', 'uint16'),
+ NullField('f3')],
+ type_ids=[42, 43, 44], nullable=False),
+ ]
+
+ batch_sizes = [0, 11]
+ return _generate_file("union", fields, batch_sizes)
+
+
+def generate_dictionary_case():
+ dict0 = Dictionary(0, StringField('dictionary1'), size=10, name='DICT0')
+ dict1 = Dictionary(1, StringField('dictionary1'), size=5, name='DICT1')
+ dict2 = Dictionary(2, get_field('dictionary2', 'int64'),
+ size=50, name='DICT2')
+
+ fields = [
+ DictionaryField('dict0', get_field('', 'int8'), dict0),
+ DictionaryField('dict1', get_field('', 'int32'), dict1),
+ DictionaryField('dict2', get_field('', 'int16'), dict2)
+ ]
+ batch_sizes = [7, 10]
+ return _generate_file("dictionary", fields, batch_sizes,
+ dictionaries=[dict0, dict1, dict2])
+
+
+def generate_dictionary_unsigned_case():
+ dict0 = Dictionary(0, StringField('dictionary0'), size=5, name='DICT0')
+ dict1 = Dictionary(1, StringField('dictionary1'), size=5, name='DICT1')
+ dict2 = Dictionary(2, StringField('dictionary2'), size=5, name='DICT2')
+
+ # TODO: JavaScript does not support uint64 dictionary indices, so disabled
+ # for now
+
+ # dict3 = Dictionary(3, StringField('dictionary3'), size=5, name='DICT3')
+ fields = [
+ DictionaryField('f0', get_field('', 'uint8'), dict0),
+ DictionaryField('f1', get_field('', 'uint16'), dict1),
+ DictionaryField('f2', get_field('', 'uint32'), dict2),
+ # DictionaryField('f3', get_field('', 'uint64'), dict3)
+ ]
+ batch_sizes = [7, 10]
+ return _generate_file("dictionary_unsigned", fields, batch_sizes,
+ dictionaries=[dict0, dict1, dict2])
+
+
+def generate_nested_dictionary_case():
+ dict0 = Dictionary(0, StringField('str'), size=10, name='DICT0')
+
+ list_of_dict = ListField(
+ 'list',
+ DictionaryField('str_dict', get_field('', 'int8'), dict0))
+ dict1 = Dictionary(1, list_of_dict, size=30, name='DICT1')
+
+ struct_of_dict = StructField('struct', [
+ DictionaryField('str_dict_a', get_field('', 'int8'), dict0),
+ DictionaryField('str_dict_b', get_field('', 'int8'), dict0)
+ ])
+ dict2 = Dictionary(2, struct_of_dict, size=30, name='DICT2')
+
+ fields = [
+ DictionaryField('list_dict', get_field('', 'int8'), dict1),
+ DictionaryField('struct_dict', get_field('', 'int8'), dict2)
+ ]
+
+ batch_sizes = [10, 13]
+ return _generate_file("nested_dictionary", fields, batch_sizes,
+ dictionaries=[dict0, dict1, dict2])
+
+
+def generate_extension_case():
+ dict0 = Dictionary(0, StringField('dictionary0'), size=5, name='DICT0')
+
+ uuid_type = ExtensionType('uuid', 'uuid-serialized',
+ FixedSizeBinaryField('', 16))
+ dict_ext_type = ExtensionType(
+ 'dict-extension', 'dict-extension-serialized',
+ DictionaryField('str_dict', get_field('', 'int8'), dict0))
+
+ fields = [
+ ExtensionField('uuids', uuid_type),
+ ExtensionField('dict_exts', dict_ext_type),
+ ]
+
+ batch_sizes = [0, 13]
+ return _generate_file("extension", fields, batch_sizes,
+ dictionaries=[dict0])
+
+
+def get_generated_json_files(tempdir=None):
+ tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
+
+ def _temp_path():
+ return
+
+ file_objs = [
+ generate_primitive_case([], name='primitive_no_batches'),
+ generate_primitive_case([17, 20], name='primitive'),
+ generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
+
+ generate_primitive_large_offsets_case([17, 20])
+ .skip_category('C#')
+ .skip_category('Go')
+ .skip_category('JS'),
+
+ generate_null_case([10, 0])
+ .skip_category('C#')
+ .skip_category('JS'), # TODO(ARROW-7900)
+
+ generate_null_trivial_case([0, 0])
+ .skip_category('C#')
+ .skip_category('JS'), # TODO(ARROW-7900)
+
+ generate_decimal128_case()
+ .skip_category('Rust'),
+
+ generate_decimal256_case()
+ .skip_category('Go') # TODO(ARROW-7948): Decimal + Go
+ .skip_category('JS')
+ .skip_category('Rust'),
+
+ generate_datetime_case()
+ .skip_category('C#'),
+
+ generate_interval_case()
+ .skip_category('C#')
+ .skip_category('JS') # TODO(ARROW-5239): Intervals + JS
+ .skip_category('Rust'),
+
+ generate_month_day_nano_interval_case()
+ .skip_category('C#')
+ .skip_category('JS')
+ .skip_category('Rust'),
+
+
+ generate_map_case()
+ .skip_category('C#')
+ .skip_category('Rust'),
+
+ generate_non_canonical_map_case()
+ .skip_category('C#')
+ .skip_category('Java') # TODO(ARROW-8715)
+ .skip_category('JS') # TODO(ARROW-8716)
+ .skip_category('Rust'),
+
+ generate_nested_case()
+ .skip_category('C#'),
+
+ generate_recursive_nested_case()
+ .skip_category('C#'),
+
+ generate_nested_large_offsets_case()
+ .skip_category('C#')
+ .skip_category('Go')
+ .skip_category('JS')
+ .skip_category('Rust'),
+
+ generate_unions_case()
+ .skip_category('C#')
+ .skip_category('Go')
+ .skip_category('JS')
+ .skip_category('Rust'),
+
+ generate_custom_metadata_case()
+ .skip_category('C#')
+ .skip_category('JS'),
+
+ generate_duplicate_fieldnames_case()
+ .skip_category('C#')
+ .skip_category('Go')
+ .skip_category('JS'),
+
+ # TODO(ARROW-3039, ARROW-5267): Dictionaries in GO
+ generate_dictionary_case()
+ .skip_category('C#')
+ .skip_category('Go'),
+
+ generate_dictionary_unsigned_case()
+ .skip_category('C#')
+ .skip_category('Go') # TODO(ARROW-9378)
+ .skip_category('Java'), # TODO(ARROW-9377)
+
+ generate_nested_dictionary_case()
+ .skip_category('C#')
+ .skip_category('Go')
+ .skip_category('Java') # TODO(ARROW-7779)
+ .skip_category('JS')
+ .skip_category('Rust'),
+
+ generate_extension_case()
+ .skip_category('C#')
+ .skip_category('Go') # TODO(ARROW-3039): requires dictionaries
+ .skip_category('JS')
+ .skip_category('Rust'),
+ ]
+
+ generated_paths = []
+ for file_obj in file_objs:
+ out_path = os.path.join(tempdir, 'generated_' +
+ file_obj.name + '.json')
+ file_obj.write(out_path)
+ generated_paths.append(file_obj)
+
+ return generated_paths
diff --git a/src/arrow/dev/archery/archery/integration/runner.py b/src/arrow/dev/archery/archery/integration/runner.py
new file mode 100644
index 000000000..463917b81
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/runner.py
@@ -0,0 +1,429 @@
+# licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import namedtuple
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+import glob
+import gzip
+import itertools
+import os
+import sys
+import tempfile
+import traceback
+
+from .scenario import Scenario
+from .tester_cpp import CPPTester
+from .tester_go import GoTester
+from .tester_rust import RustTester
+from .tester_java import JavaTester
+from .tester_js import JSTester
+from .tester_csharp import CSharpTester
+from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT,
+ printer)
+from . import datagen
+
+
+Failure = namedtuple('Failure',
+ ('test_case', 'producer', 'consumer', 'exc_info'))
+
+log = printer.print
+
+
+class Outcome:
+ def __init__(self):
+ self.failure = None
+ self.skipped = False
+
+
+class IntegrationRunner(object):
+
+ def __init__(self, json_files, flight_scenarios, testers, tempdir=None,
+ debug=False, stop_on_error=True, gold_dirs=None,
+ serial=False, match=None, **unused_kwargs):
+ self.json_files = json_files
+ self.flight_scenarios = flight_scenarios
+ self.testers = testers
+ self.temp_dir = tempdir or tempfile.mkdtemp()
+ self.debug = debug
+ self.stop_on_error = stop_on_error
+ self.serial = serial
+ self.gold_dirs = gold_dirs
+ self.failures = []
+ self.match = match
+
+ if self.match is not None:
+ print("-- Only running tests with {} in their name"
+ .format(self.match))
+ self.json_files = [json_file for json_file in self.json_files
+ if self.match in json_file.name]
+
+ def run(self):
+ """
+ Run Arrow IPC integration tests for the matrix of enabled
+ implementations.
+ """
+ for producer, consumer in itertools.product(
+ filter(lambda t: t.PRODUCER, self.testers),
+ filter(lambda t: t.CONSUMER, self.testers)):
+ self._compare_implementations(
+ producer, consumer, self._produce_consume,
+ self.json_files)
+ if self.gold_dirs:
+ for gold_dir, consumer in itertools.product(
+ self.gold_dirs,
+ filter(lambda t: t.CONSUMER, self.testers)):
+ log('\n\n\n\n')
+ log('******************************************************')
+ log('Tests against golden files in {}'.format(gold_dir))
+ log('******************************************************')
+
+ def run_gold(producer, consumer, outcome, test_case):
+ self._run_gold(gold_dir, producer, consumer, outcome,
+ test_case)
+ self._compare_implementations(
+ consumer, consumer, run_gold,
+ self._gold_tests(gold_dir))
+
+ def run_flight(self):
+ """
+ Run Arrow Flight integration tests for the matrix of enabled
+ implementations.
+ """
+ servers = filter(lambda t: t.FLIGHT_SERVER, self.testers)
+ clients = filter(lambda t: (t.FLIGHT_CLIENT and t.CONSUMER),
+ self.testers)
+ for server, client in itertools.product(servers, clients):
+ self._compare_flight_implementations(server, client)
+
+ def _gold_tests(self, gold_dir):
+ prefix = os.path.basename(os.path.normpath(gold_dir))
+ SUFFIX = ".json.gz"
+ golds = [jf for jf in os.listdir(gold_dir) if jf.endswith(SUFFIX)]
+ for json_path in golds:
+ name = json_path[json_path.index('_')+1: -len(SUFFIX)]
+ base_name = prefix + "_" + name + ".gold.json"
+ out_path = os.path.join(self.temp_dir, base_name)
+ with gzip.open(os.path.join(gold_dir, json_path)) as i:
+ with open(out_path, "wb") as out:
+ out.write(i.read())
+
+ try:
+ skip = next(f for f in self.json_files
+ if f.name == name).skip
+ except StopIteration:
+ skip = set()
+ if name == 'union' and prefix == '0.17.1':
+ skip.add("Java")
+ if prefix == '1.0.0-bigendian' or prefix == '1.0.0-littleendian':
+ skip.add("C#")
+ skip.add("Go")
+ skip.add("Java")
+ skip.add("JS")
+ skip.add("Rust")
+ if prefix == '2.0.0-compression':
+ skip.add("C#")
+ skip.add("JS")
+ skip.add("Rust")
+
+ # See https://github.com/apache/arrow/pull/9822 for how to
+ # disable specific compression type tests.
+
+ if prefix == '4.0.0-shareddict':
+ skip.add("C#")
+ skip.add("Go")
+
+ yield datagen.File(name, None, None, skip=skip, path=out_path)
+
+ def _run_test_cases(self, producer, consumer, case_runner,
+ test_cases):
+ def case_wrapper(test_case):
+ with printer.cork():
+ return case_runner(test_case)
+
+ if self.failures and self.stop_on_error:
+ return
+
+ if self.serial:
+ for outcome in map(case_wrapper, test_cases):
+ if outcome.failure is not None:
+ self.failures.append(outcome.failure)
+ if self.stop_on_error:
+ break
+
+ else:
+ with ThreadPoolExecutor() as executor:
+ for outcome in executor.map(case_wrapper, test_cases):
+ if outcome.failure is not None:
+ self.failures.append(outcome.failure)
+ if self.stop_on_error:
+ break
+
+ def _compare_implementations(
+ self, producer, consumer, run_binaries, test_cases):
+ """
+ Compare Arrow IPC for two implementations (one producer, one consumer).
+ """
+ log('##########################################################')
+ log('IPC: {0} producing, {1} consuming'
+ .format(producer.name, consumer.name))
+ log('##########################################################')
+
+ case_runner = partial(self._run_ipc_test_case,
+ producer, consumer, run_binaries)
+ self._run_test_cases(producer, consumer, case_runner, test_cases)
+
+ def _run_ipc_test_case(self, producer, consumer, run_binaries, test_case):
+ """
+ Run one IPC test case.
+ """
+ outcome = Outcome()
+
+ json_path = test_case.path
+ log('==========================================================')
+ log('Testing file {0}'.format(json_path))
+ log('==========================================================')
+
+ if producer.name in test_case.skip:
+ log('-- Skipping test because producer {0} does '
+ 'not support'.format(producer.name))
+ outcome.skipped = True
+
+ elif consumer.name in test_case.skip:
+ log('-- Skipping test because consumer {0} does '
+ 'not support'.format(consumer.name))
+ outcome.skipped = True
+
+ elif SKIP_ARROW in test_case.skip:
+ log('-- Skipping test')
+ outcome.skipped = True
+
+ else:
+ try:
+ run_binaries(producer, consumer, outcome, test_case)
+ except Exception:
+ traceback.print_exc(file=printer.stdout)
+ outcome.failure = Failure(test_case, producer, consumer,
+ sys.exc_info())
+
+ return outcome
+
+ def _produce_consume(self, producer, consumer, outcome, test_case):
+ # Make the random access file
+ json_path = test_case.path
+ file_id = guid()[:8]
+ name = os.path.splitext(os.path.basename(json_path))[0]
+
+ producer_file_path = os.path.join(self.temp_dir, file_id + '_' +
+ name + '.json_as_file')
+ producer_stream_path = os.path.join(self.temp_dir, file_id + '_' +
+ name + '.producer_file_as_stream')
+ consumer_file_path = os.path.join(self.temp_dir, file_id + '_' +
+ name + '.consumer_stream_as_file')
+
+ log('-- Creating binary inputs')
+ producer.json_to_file(json_path, producer_file_path)
+
+ # Validate the file
+ log('-- Validating file')
+ consumer.validate(json_path, producer_file_path)
+
+ log('-- Validating stream')
+ producer.file_to_stream(producer_file_path, producer_stream_path)
+ consumer.stream_to_file(producer_stream_path, consumer_file_path)
+ consumer.validate(json_path, consumer_file_path)
+
+ def _run_gold(self, gold_dir, producer, consumer, outcome, test_case):
+ json_path = test_case.path
+
+ # Validate the file
+ log('-- Validating file')
+ producer_file_path = os.path.join(
+ gold_dir, "generated_" + test_case.name + ".arrow_file")
+ consumer.validate(json_path, producer_file_path)
+
+ log('-- Validating stream')
+ consumer_stream_path = os.path.join(
+ gold_dir, "generated_" + test_case.name + ".stream")
+ file_id = guid()[:8]
+ name = os.path.splitext(os.path.basename(json_path))[0]
+
+ consumer_file_path = os.path.join(self.temp_dir, file_id + '_' +
+ name + '.consumer_stream_as_file')
+
+ consumer.stream_to_file(consumer_stream_path, consumer_file_path)
+ consumer.validate(json_path, consumer_file_path)
+
+ def _compare_flight_implementations(self, producer, consumer):
+ log('##########################################################')
+ log('Flight: {0} serving, {1} requesting'
+ .format(producer.name, consumer.name))
+ log('##########################################################')
+
+ case_runner = partial(self._run_flight_test_case, producer, consumer)
+ self._run_test_cases(producer, consumer, case_runner,
+ self.json_files + self.flight_scenarios)
+
+ def _run_flight_test_case(self, producer, consumer, test_case):
+ """
+ Run one Flight test case.
+ """
+ outcome = Outcome()
+
+ log('=' * 58)
+ log('Testing file {0}'.format(test_case.name))
+ log('=' * 58)
+
+ if producer.name in test_case.skip:
+ log('-- Skipping test because producer {0} does '
+ 'not support'.format(producer.name))
+ outcome.skipped = True
+
+ elif consumer.name in test_case.skip:
+ log('-- Skipping test because consumer {0} does '
+ 'not support'.format(consumer.name))
+ outcome.skipped = True
+
+ elif SKIP_FLIGHT in test_case.skip:
+ log('-- Skipping test')
+ outcome.skipped = True
+
+ else:
+ try:
+ if isinstance(test_case, Scenario):
+ server = producer.flight_server(test_case.name)
+ client_args = {'scenario_name': test_case.name}
+ else:
+ server = producer.flight_server()
+ client_args = {'json_path': test_case.path}
+
+ with server as port:
+ # Have the client upload the file, then download and
+ # compare
+ consumer.flight_request(port, **client_args)
+ except Exception:
+ traceback.print_exc(file=printer.stdout)
+ outcome.failure = Failure(test_case, producer, consumer,
+ sys.exc_info())
+
+ return outcome
+
+
+def get_static_json_files():
+ glob_pattern = os.path.join(ARROW_ROOT_DEFAULT,
+ 'integration', 'data', '*.json')
+ return [
+ datagen.File(name=os.path.basename(p), path=p, skip=set(),
+ schema=None, batches=None)
+ for p in glob.glob(glob_pattern)
+ ]
+
+
+def run_all_tests(with_cpp=True, with_java=True, with_js=True,
+ with_csharp=True, with_go=True, with_rust=False,
+ run_flight=False, tempdir=None, **kwargs):
+ tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
+
+ testers = []
+
+ if with_cpp:
+ testers.append(CPPTester(**kwargs))
+
+ if with_java:
+ testers.append(JavaTester(**kwargs))
+
+ if with_js:
+ testers.append(JSTester(**kwargs))
+
+ if with_csharp:
+ testers.append(CSharpTester(**kwargs))
+
+ if with_go:
+ testers.append(GoTester(**kwargs))
+
+ if with_rust:
+ testers.append(RustTester(**kwargs))
+
+ static_json_files = get_static_json_files()
+ generated_json_files = datagen.get_generated_json_files(tempdir=tempdir)
+ json_files = static_json_files + generated_json_files
+
+ # Additional integration test cases for Arrow Flight.
+ flight_scenarios = [
+ Scenario(
+ "auth:basic_proto",
+ description="Authenticate using the BasicAuth protobuf."),
+ Scenario(
+ "middleware",
+ description="Ensure headers are propagated via middleware.",
+ skip={"Rust"} # TODO(ARROW-10961): tonic upgrade needed
+ ),
+ ]
+
+ runner = IntegrationRunner(json_files, flight_scenarios, testers, **kwargs)
+ runner.run()
+ if run_flight:
+ runner.run_flight()
+
+ fail_count = 0
+ if runner.failures:
+ log("################# FAILURES #################")
+ for test_case, producer, consumer, exc_info in runner.failures:
+ fail_count += 1
+ log("FAILED TEST:", end=" ")
+ log(test_case.name, producer.name, "producing, ",
+ consumer.name, "consuming")
+ if exc_info:
+ traceback.print_exception(*exc_info)
+ log()
+
+ log(fail_count, "failures")
+ if fail_count > 0:
+ sys.exit(1)
+
+
+def write_js_test_json(directory):
+ datagen.generate_map_case().write(
+ os.path.join(directory, 'map.json')
+ )
+ datagen.generate_nested_case().write(
+ os.path.join(directory, 'nested.json')
+ )
+ datagen.generate_decimal128_case().write(
+ os.path.join(directory, 'decimal.json')
+ )
+ datagen.generate_decimal256_case().write(
+ os.path.join(directory, 'decimal256.json')
+ )
+ datagen.generate_datetime_case().write(
+ os.path.join(directory, 'datetime.json')
+ )
+ datagen.generate_dictionary_case().write(
+ os.path.join(directory, 'dictionary.json')
+ )
+ datagen.generate_dictionary_unsigned_case().write(
+ os.path.join(directory, 'dictionary_unsigned.json')
+ )
+ datagen.generate_primitive_case([]).write(
+ os.path.join(directory, 'primitive_no_batches.json')
+ )
+ datagen.generate_primitive_case([7, 10]).write(
+ os.path.join(directory, 'primitive.json')
+ )
+ datagen.generate_primitive_case([0, 0, 0]).write(
+ os.path.join(directory, 'primitive-empty.json')
+ )
diff --git a/src/arrow/dev/archery/archery/integration/scenario.py b/src/arrow/dev/archery/archery/integration/scenario.py
new file mode 100644
index 000000000..1fcbca64e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/scenario.py
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+class Scenario:
+ """
+ An integration test scenario for Arrow Flight.
+
+ Does not correspond to a particular IPC JSON file.
+ """
+
+ def __init__(self, name, description, skip=None):
+ self.name = name
+ self.description = description
+ self.skip = skip or set()
diff --git a/src/arrow/dev/archery/archery/integration/tester.py b/src/arrow/dev/archery/archery/integration/tester.py
new file mode 100644
index 000000000..122e4f2e4
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Base class for language-specific integration test harnesses
+
+import subprocess
+
+from .util import log
+
+
+class Tester(object):
+ PRODUCER = False
+ CONSUMER = False
+ FLIGHT_SERVER = False
+ FLIGHT_CLIENT = False
+
+ def __init__(self, debug=False, **args):
+ self.args = args
+ self.debug = debug
+
+ def run_shell_command(self, cmd):
+ cmd = ' '.join(cmd)
+ if self.debug:
+ log(cmd)
+ subprocess.check_call(cmd, shell=True)
+
+ def json_to_file(self, json_path, arrow_path):
+ raise NotImplementedError
+
+ def stream_to_file(self, stream_path, file_path):
+ raise NotImplementedError
+
+ def file_to_stream(self, file_path, stream_path):
+ raise NotImplementedError
+
+ def validate(self, json_path, arrow_path):
+ raise NotImplementedError
+
+ def flight_server(self, scenario_name=None):
+ """Start the Flight server on a free port.
+
+ This should be a context manager that returns the port as the
+ managed object, and cleans up the server on exit.
+ """
+ raise NotImplementedError
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ raise NotImplementedError
diff --git a/src/arrow/dev/archery/archery/integration/tester_cpp.py b/src/arrow/dev/archery/archery/integration/tester_cpp.py
new file mode 100644
index 000000000..d35c9550e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_cpp.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import os
+import subprocess
+
+from .tester import Tester
+from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+
+
+class CPPTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+ FLIGHT_SERVER = True
+ FLIGHT_CLIENT = True
+
+ EXE_PATH = os.environ.get(
+ 'ARROW_CPP_EXE_PATH',
+ os.path.join(ARROW_ROOT_DEFAULT, 'cpp/build/debug'))
+
+ CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'arrow-json-integration-test')
+ STREAM_TO_FILE = os.path.join(EXE_PATH, 'arrow-stream-to-file')
+ FILE_TO_STREAM = os.path.join(EXE_PATH, 'arrow-file-to-stream')
+
+ FLIGHT_SERVER_CMD = [
+ os.path.join(EXE_PATH, 'flight-test-integration-server')]
+ FLIGHT_CLIENT_CMD = [
+ os.path.join(EXE_PATH, 'flight-test-integration-client'),
+ "-host", "localhost"]
+
+ name = 'C++'
+
+ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+ cmd = [self.CPP_INTEGRATION_EXE, '--integration']
+
+ if arrow_path is not None:
+ cmd.append('--arrow=' + arrow_path)
+
+ if json_path is not None:
+ cmd.append('--json=' + json_path)
+
+ cmd.append('--mode=' + command)
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'VALIDATE')
+
+ def json_to_file(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path]
+ self.run_shell_command(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path]
+ self.run_shell_command(cmd)
+
+ @contextlib.contextmanager
+ def flight_server(self, scenario_name=None):
+ cmd = self.FLIGHT_SERVER_CMD + ['-port=0']
+ if scenario_name:
+ cmd = cmd + ["-scenario", scenario_name]
+ if self.debug:
+ log(' '.join(cmd))
+ server = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ output = server.stdout.readline().decode()
+ if not output.startswith("Server listening on localhost:"):
+ server.kill()
+ out, err = server.communicate()
+ raise RuntimeError(
+ "Flight-C++ server did not start properly, "
+ "stdout:\n{}\n\nstderr:\n{}\n"
+ .format(output + out.decode(), err.decode()))
+ port = int(output.split(":")[1])
+ yield port
+ finally:
+ server.kill()
+ server.wait(5)
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ cmd = self.FLIGHT_CLIENT_CMD + [
+ '-port=' + str(port),
+ ]
+ if json_path:
+ cmd.extend(('-path', json_path))
+ elif scenario_name:
+ cmd.extend(('-scenario', scenario_name))
+ else:
+ raise TypeError("Must provide one of json_path or scenario_name")
+
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
diff --git a/src/arrow/dev/archery/archery/integration/tester_csharp.py b/src/arrow/dev/archery/archery/integration/tester_csharp.py
new file mode 100644
index 000000000..130c49cfe
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_csharp.py
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from .tester import Tester
+from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+
+
+class CSharpTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+
+ EXE_PATH = os.path.join(
+ ARROW_ROOT_DEFAULT,
+ 'csharp/artifacts/Apache.Arrow.IntegrationTest',
+ 'Debug/netcoreapp3.1/Apache.Arrow.IntegrationTest')
+
+ name = 'C#'
+
+ def _run(self, json_path=None, arrow_path=None, command='validate'):
+ cmd = [self.EXE_PATH]
+
+ cmd.extend(['--mode', command])
+
+ if json_path is not None:
+ cmd.extend(['-j', json_path])
+
+ if arrow_path is not None:
+ cmd.extend(['-a', arrow_path])
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(json_path, arrow_path, 'validate')
+
+ def json_to_file(self, json_path, arrow_path):
+ return self._run(json_path, arrow_path, 'json-to-arrow')
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = [self.EXE_PATH]
+ cmd.extend(['--mode', 'stream-to-file', '-a', file_path])
+ cmd.extend(['<', stream_path])
+ self.run_shell_command(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = [self.EXE_PATH]
+ cmd.extend(['--mode', 'file-to-stream'])
+ cmd.extend(['-a', file_path, '>', stream_path])
+ self.run_shell_command(cmd)
diff --git a/src/arrow/dev/archery/archery/integration/tester_go.py b/src/arrow/dev/archery/archery/integration/tester_go.py
new file mode 100644
index 000000000..eeba38fe5
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_go.py
@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import os
+import subprocess
+
+from .tester import Tester
+from .util import run_cmd, log
+
+
+class GoTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+ FLIGHT_SERVER = True
+ FLIGHT_CLIENT = True
+
+ # FIXME(sbinet): revisit for Go modules
+ HOME = os.getenv('HOME', '~')
+ GOPATH = os.getenv('GOPATH', os.path.join(HOME, 'go'))
+ GOBIN = os.environ.get('GOBIN', os.path.join(GOPATH, 'bin'))
+
+ GO_INTEGRATION_EXE = os.path.join(GOBIN, 'arrow-json-integration-test')
+ STREAM_TO_FILE = os.path.join(GOBIN, 'arrow-stream-to-file')
+ FILE_TO_STREAM = os.path.join(GOBIN, 'arrow-file-to-stream')
+
+ FLIGHT_SERVER_CMD = [
+ os.path.join(GOBIN, 'arrow-flight-integration-server')]
+ FLIGHT_CLIENT_CMD = [
+ os.path.join(GOBIN, 'arrow-flight-integration-client'),
+ '-host', 'localhost']
+
+ name = 'Go'
+
+ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+ cmd = [self.GO_INTEGRATION_EXE]
+
+ if arrow_path is not None:
+ cmd.extend(['-arrow', arrow_path])
+
+ if json_path is not None:
+ cmd.extend(['-json', json_path])
+
+ cmd.extend(['-mode', command])
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'VALIDATE')
+
+ def json_to_file(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path]
+ self.run_shell_command(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path]
+ self.run_shell_command(cmd)
+
+ @contextlib.contextmanager
+ def flight_server(self, scenario_name=None):
+ cmd = self.FLIGHT_SERVER_CMD + ['-port=0']
+ if scenario_name:
+ cmd = cmd + ['-scenario', scenario_name]
+ if self.debug:
+ log(' '.join(cmd))
+ server = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ try:
+ output = server.stdout.readline().decode()
+ if not output.startswith("Server listening on localhost:"):
+ server.kill()
+ out, err = server.communicate()
+ raise RuntimeError(
+ "Flight-Go server did not start properly, "
+ "stdout: \n{}\n\nstderr:\n{}\n"
+ .format(output + out.decode(), err.decode())
+ )
+ port = int(output.split(":")[1])
+ yield port
+ finally:
+ server.kill()
+ server.wait(5)
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ cmd = self.FLIGHT_CLIENT_CMD + [
+ '-port=' + str(port),
+ ]
+ if json_path:
+ cmd.extend(('-path', json_path))
+ elif scenario_name:
+ cmd.extend(('-scenario', scenario_name))
+ else:
+ raise TypeError("Must provide one of json_path or scenario_name")
+
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
diff --git a/src/arrow/dev/archery/archery/integration/tester_java.py b/src/arrow/dev/archery/archery/integration/tester_java.py
new file mode 100644
index 000000000..f283f6cd2
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_java.py
@@ -0,0 +1,140 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import os
+import subprocess
+
+from .tester import Tester
+from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+
+
+def load_version_from_pom():
+ import xml.etree.ElementTree as ET
+ tree = ET.parse(os.path.join(ARROW_ROOT_DEFAULT, 'java', 'pom.xml'))
+ tag_pattern = '{http://maven.apache.org/POM/4.0.0}version'
+ version_tag = list(tree.getroot().findall(tag_pattern))[0]
+ return version_tag.text
+
+
+class JavaTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+ FLIGHT_SERVER = True
+ FLIGHT_CLIENT = True
+
+ JAVA_OPTS = ['-Dio.netty.tryReflectionSetAccessible=true',
+ '-Darrow.struct.conflict.policy=CONFLICT_APPEND']
+
+ _arrow_version = load_version_from_pom()
+ ARROW_TOOLS_JAR = os.environ.get(
+ 'ARROW_JAVA_INTEGRATION_JAR',
+ os.path.join(ARROW_ROOT_DEFAULT,
+ 'java/tools/target/arrow-tools-{}-'
+ 'jar-with-dependencies.jar'.format(_arrow_version)))
+ ARROW_FLIGHT_JAR = os.environ.get(
+ 'ARROW_FLIGHT_JAVA_INTEGRATION_JAR',
+ os.path.join(ARROW_ROOT_DEFAULT,
+ 'java/flight/flight-core/target/flight-core-{}-'
+ 'jar-with-dependencies.jar'.format(_arrow_version)))
+ ARROW_FLIGHT_SERVER = ('org.apache.arrow.flight.example.integration.'
+ 'IntegrationTestServer')
+ ARROW_FLIGHT_CLIENT = ('org.apache.arrow.flight.example.integration.'
+ 'IntegrationTestClient')
+
+ name = 'Java'
+
+ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+ cmd = ['java'] + self.JAVA_OPTS + \
+ ['-cp', self.ARROW_TOOLS_JAR, 'org.apache.arrow.tools.Integration']
+
+ if arrow_path is not None:
+ cmd.extend(['-a', arrow_path])
+
+ if json_path is not None:
+ cmd.extend(['-j', json_path])
+
+ cmd.extend(['-c', command])
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'VALIDATE')
+
+ def json_to_file(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = ['java'] + self.JAVA_OPTS + \
+ ['-cp', self.ARROW_TOOLS_JAR,
+ 'org.apache.arrow.tools.StreamToFile', stream_path, file_path]
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = ['java'] + self.JAVA_OPTS + \
+ ['-cp', self.ARROW_TOOLS_JAR,
+ 'org.apache.arrow.tools.FileToStream', file_path, stream_path]
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ cmd = ['java'] + self.JAVA_OPTS + \
+ ['-cp', self.ARROW_FLIGHT_JAR, self.ARROW_FLIGHT_CLIENT,
+ '-port', str(port)]
+
+ if json_path:
+ cmd.extend(('-j', json_path))
+ elif scenario_name:
+ cmd.extend(('-scenario', scenario_name))
+ else:
+ raise TypeError("Must provide one of json_path or scenario_name")
+
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
+
+ @contextlib.contextmanager
+ def flight_server(self, scenario_name=None):
+ cmd = ['java'] + self.JAVA_OPTS + \
+ ['-cp', self.ARROW_FLIGHT_JAR, self.ARROW_FLIGHT_SERVER,
+ '-port', '0']
+ if scenario_name:
+ cmd.extend(('-scenario', scenario_name))
+ if self.debug:
+ log(' '.join(cmd))
+ server = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ output = server.stdout.readline().decode()
+ if not output.startswith("Server listening on localhost:"):
+ server.kill()
+ out, err = server.communicate()
+ raise RuntimeError(
+ "Flight-Java server did not start properly, "
+ "stdout:\n{}\n\nstderr:\n{}\n"
+ .format(output + out.decode(), err.decode()))
+ port = int(output.split(":")[1])
+ yield port
+ finally:
+ server.kill()
+ server.wait(5)
diff --git a/src/arrow/dev/archery/archery/integration/tester_js.py b/src/arrow/dev/archery/archery/integration/tester_js.py
new file mode 100644
index 000000000..e24eec0ca
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_js.py
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from .tester import Tester
+from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+
+
+class JSTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+
+ EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
+ VALIDATE = os.path.join(EXE_PATH, 'integration.js')
+ JSON_TO_ARROW = os.path.join(EXE_PATH, 'json-to-arrow.js')
+ STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file.js')
+ FILE_TO_STREAM = os.path.join(EXE_PATH, 'file-to-stream.js')
+
+ name = 'JS'
+
+ def _run(self, exe_cmd, arrow_path=None, json_path=None,
+ command='VALIDATE'):
+ cmd = [exe_cmd]
+
+ if arrow_path is not None:
+ cmd.extend(['-a', arrow_path])
+
+ if json_path is not None:
+ cmd.extend(['-j', json_path])
+
+ cmd.extend(['--mode', command])
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(self.VALIDATE, arrow_path, json_path, 'VALIDATE')
+
+ def json_to_file(self, json_path, arrow_path):
+ cmd = ['node',
+ '--no-warnings', self.JSON_TO_ARROW,
+ '-a', arrow_path,
+ '-j', json_path]
+ self.run_shell_command(cmd)
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = ['node', '--no-warnings', self.STREAM_TO_FILE,
+ '<', stream_path,
+ '>', file_path]
+ self.run_shell_command(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = ['node', '--no-warnings', self.FILE_TO_STREAM,
+ '<', file_path,
+ '>', stream_path]
+ self.run_shell_command(cmd)
diff --git a/src/arrow/dev/archery/archery/integration/tester_rust.py b/src/arrow/dev/archery/archery/integration/tester_rust.py
new file mode 100644
index 000000000..bca80ebae
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/tester_rust.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import os
+import subprocess
+
+from .tester import Tester
+from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+
+
+class RustTester(Tester):
+ PRODUCER = True
+ CONSUMER = True
+ FLIGHT_SERVER = True
+ FLIGHT_CLIENT = True
+
+ EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'rust/target/debug')
+
+ RUST_INTEGRATION_EXE = os.path.join(EXE_PATH,
+ 'arrow-json-integration-test')
+ STREAM_TO_FILE = os.path.join(EXE_PATH, 'arrow-stream-to-file')
+ FILE_TO_STREAM = os.path.join(EXE_PATH, 'arrow-file-to-stream')
+
+ FLIGHT_SERVER_CMD = [
+ os.path.join(EXE_PATH, 'flight-test-integration-server')]
+ FLIGHT_CLIENT_CMD = [
+ os.path.join(EXE_PATH, 'flight-test-integration-client'),
+ "--host", "localhost"]
+
+ name = 'Rust'
+
+ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
+ cmd = [self.RUST_INTEGRATION_EXE, '--integration']
+
+ if arrow_path is not None:
+ cmd.append('--arrow=' + arrow_path)
+
+ if json_path is not None:
+ cmd.append('--json=' + json_path)
+
+ cmd.append('--mode=' + command)
+
+ if self.debug:
+ log(' '.join(cmd))
+
+ run_cmd(cmd)
+
+ def validate(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'VALIDATE')
+
+ def json_to_file(self, json_path, arrow_path):
+ return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
+
+ def stream_to_file(self, stream_path, file_path):
+ cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path]
+ self.run_shell_command(cmd)
+
+ def file_to_stream(self, file_path, stream_path):
+ cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path]
+ self.run_shell_command(cmd)
+
+ @contextlib.contextmanager
+ def flight_server(self, scenario_name=None):
+ cmd = self.FLIGHT_SERVER_CMD + ['--port=0']
+ if scenario_name:
+ cmd = cmd + ["--scenario", scenario_name]
+ if self.debug:
+ log(' '.join(cmd))
+ server = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ output = server.stdout.readline().decode()
+ if not output.startswith("Server listening on localhost:"):
+ server.kill()
+ out, err = server.communicate()
+ raise RuntimeError(
+ "Flight-Rust server did not start properly, "
+ "stdout:\n{}\n\nstderr:\n{}\n"
+ .format(output + out.decode(), err.decode()))
+ port = int(output.split(":")[1])
+ yield port
+ finally:
+ server.kill()
+ server.wait(5)
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ cmd = self.FLIGHT_CLIENT_CMD + [
+ '--port=' + str(port),
+ ]
+ if json_path:
+ cmd.extend(('--path', json_path))
+ elif scenario_name:
+ cmd.extend(('--scenario', scenario_name))
+ else:
+ raise TypeError("Must provide one of json_path or scenario_name")
+
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
diff --git a/src/arrow/dev/archery/archery/integration/util.py b/src/arrow/dev/archery/archery/integration/util.py
new file mode 100644
index 000000000..a4c4982ec
--- /dev/null
+++ b/src/arrow/dev/archery/archery/integration/util.py
@@ -0,0 +1,166 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import io
+import os
+import random
+import socket
+import subprocess
+import sys
+import threading
+import uuid
+
+import numpy as np
+
+
+def guid():
+ return uuid.uuid4().hex
+
+
+# SKIP categories
+SKIP_ARROW = 'arrow'
+SKIP_FLIGHT = 'flight'
+
+ARROW_ROOT_DEFAULT = os.environ.get(
+ 'ARROW_ROOT',
+ os.path.abspath(__file__).rsplit("/", 5)[0]
+)
+
+
+class _Printer:
+ """
+ A print()-providing object that can override the stream output on
+ a per-thread basis.
+ """
+
+ def __init__(self):
+ self._tls = threading.local()
+
+ def _get_stdout(self):
+ try:
+ return self._tls.stdout
+ except AttributeError:
+ self._tls.stdout = sys.stdout
+ self._tls.corked = False
+ return self._tls.stdout
+
+ def print(self, *args, **kwargs):
+ """
+ A variant of print() that writes to a thread-local stream.
+ """
+ print(*args, file=self._get_stdout(), **kwargs)
+
+ @property
+ def stdout(self):
+ """
+ A thread-local stdout wrapper that may be temporarily buffered
+ using `cork()`.
+ """
+ return self._get_stdout()
+
+ @contextlib.contextmanager
+ def cork(self):
+ """
+ Temporarily buffer this thread's stream and write out its contents
+ at the end of the context manager. Useful to avoid interleaved
+ output when multiple threads output progress information.
+ """
+ outer_stdout = self._get_stdout()
+ assert not self._tls.corked, "reentrant call"
+ inner_stdout = self._tls.stdout = io.StringIO()
+ self._tls.corked = True
+ try:
+ yield
+ finally:
+ self._tls.stdout = outer_stdout
+ self._tls.corked = False
+ outer_stdout.write(inner_stdout.getvalue())
+ outer_stdout.flush()
+
+
+printer = _Printer()
+log = printer.print
+
+
+_RAND_CHARS = np.array(list("abcdefghijklmnop123456Ârrôwµ£°€矢"), dtype="U")
+
+
+def random_utf8(nchars):
+ """
+ Generate one random UTF8 string.
+ """
+ return ''.join(np.random.choice(_RAND_CHARS, nchars))
+
+
+def random_bytes(nbytes):
+ """
+ Generate one random binary string.
+ """
+ # NOTE getrandbits(0) fails
+ if nbytes > 0:
+ return random.getrandbits(nbytes * 8).to_bytes(nbytes,
+ byteorder='little')
+ else:
+ return b""
+
+
+def tobytes(o):
+ if isinstance(o, str):
+ return o.encode('utf8')
+ return o
+
+
+def frombytes(o):
+ if isinstance(o, bytes):
+ return o.decode('utf8')
+ return o
+
+
+def run_cmd(cmd):
+ if isinstance(cmd, str):
+ cmd = cmd.split(' ')
+
+ try:
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ # this avoids hiding the stdout / stderr of failed processes
+ sio = io.StringIO()
+ print('Command failed:', " ".join(cmd), file=sio)
+ print('With output:', file=sio)
+ print('--------------', file=sio)
+ print(frombytes(e.output), file=sio)
+ print('--------------', file=sio)
+ raise RuntimeError(sio.getvalue())
+
+ return frombytes(output)
+
+
+# Adapted from CPython
+def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
+ """Returns an unused port that should be suitable for binding. This is
+ achieved by creating a temporary socket with the same family and type as
+ the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to
+ the specified host address (defaults to 0.0.0.0) with the port set to 0,
+ eliciting an unused ephemeral port from the OS. The temporary socket is
+ then closed and deleted, and the ephemeral port is returned.
+ """
+ with socket.socket(family, socktype) as tempsock:
+ tempsock.bind(('', 0))
+ port = tempsock.getsockname()[1]
+ del tempsock
+ return port
diff --git a/src/arrow/dev/archery/archery/lang/__init__.py b/src/arrow/dev/archery/archery/lang/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/lang/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/arrow/dev/archery/archery/lang/cpp.py b/src/arrow/dev/archery/archery/lang/cpp.py
new file mode 100644
index 000000000..c2b1ca680
--- /dev/null
+++ b/src/arrow/dev/archery/archery/lang/cpp.py
@@ -0,0 +1,296 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from ..utils.cmake import CMakeDefinition
+
+
+def truthifier(value):
+ return "ON" if value else "OFF"
+
+
+def or_else(value, default):
+ return value if value else default
+
+
+def coalesce(value, fallback):
+ return fallback if value is None else value
+
+
+LLVM_VERSION = 7
+
+
+class CppConfiguration:
+ def __init__(self,
+
+ # toolchain
+ cc=None, cxx=None, cxx_flags=None,
+ build_type=None, warn_level=None,
+ cpp_package_prefix=None, install_prefix=None, use_conda=None,
+ build_static=False, build_shared=True, build_unity=True,
+ # tests & examples
+ with_tests=None, with_benchmarks=None, with_examples=None,
+ with_integration=None,
+ # static checks
+ use_asan=None, use_tsan=None, use_ubsan=None,
+ with_fuzzing=None,
+ # Components
+ with_compute=None, with_csv=None, with_cuda=None,
+ with_dataset=None, with_filesystem=None, with_flight=None,
+ with_gandiva=None, with_hdfs=None, with_hiveserver2=None,
+ with_ipc=True, with_json=None, with_jni=None,
+ with_mimalloc=None,
+ with_parquet=None, with_plasma=None, with_python=True,
+ with_r=None, with_s3=None,
+ # Compressions
+ with_brotli=None, with_bz2=None, with_lz4=None,
+ with_snappy=None, with_zlib=None, with_zstd=None,
+ # extras
+ with_lint_only=False,
+ use_gold_linker=True,
+ simd_level="SSE4_2",
+ cmake_extras=None):
+ self._cc = cc
+ self._cxx = cxx
+ self.cxx_flags = cxx_flags
+
+ self._build_type = build_type
+ self.warn_level = warn_level
+ self._install_prefix = install_prefix
+ self._package_prefix = cpp_package_prefix
+ self._use_conda = use_conda
+ self.build_static = build_static
+ self.build_shared = build_shared
+ self.build_unity = build_unity
+
+ self.with_tests = with_tests
+ self.with_benchmarks = with_benchmarks
+ self.with_examples = with_examples
+ self.with_integration = with_integration
+
+ self.use_asan = use_asan
+ self.use_tsan = use_tsan
+ self.use_ubsan = use_ubsan
+ self.with_fuzzing = with_fuzzing
+
+ self.with_compute = with_compute
+ self.with_csv = with_csv
+ self.with_cuda = with_cuda
+ self.with_dataset = with_dataset
+ self.with_filesystem = with_filesystem
+ self.with_flight = with_flight
+ self.with_gandiva = with_gandiva
+ self.with_hdfs = with_hdfs
+ self.with_hiveserver2 = with_hiveserver2
+ self.with_ipc = with_ipc
+ self.with_json = with_json
+ self.with_jni = with_jni
+ self.with_mimalloc = with_mimalloc
+ self.with_parquet = with_parquet
+ self.with_plasma = with_plasma
+ self.with_python = with_python
+ self.with_r = with_r
+ self.with_s3 = with_s3
+
+ self.with_brotli = with_brotli
+ self.with_bz2 = with_bz2
+ self.with_lz4 = with_lz4
+ self.with_snappy = with_snappy
+ self.with_zlib = with_zlib
+ self.with_zstd = with_zstd
+
+ self.with_lint_only = with_lint_only
+ self.use_gold_linker = use_gold_linker
+ self.simd_level = simd_level
+
+ self.cmake_extras = cmake_extras
+
+ # Fixup required dependencies by providing sane defaults if the caller
+ # didn't specify the option.
+ if self.with_r:
+ self.with_csv = coalesce(with_csv, True)
+ self.with_dataset = coalesce(with_dataset, True)
+ self.with_filesystem = coalesce(with_filesystem, True)
+ self.with_ipc = coalesce(with_ipc, True)
+ self.with_json = coalesce(with_json, True)
+ self.with_parquet = coalesce(with_parquet, True)
+
+ if self.with_python:
+ self.with_zlib = coalesce(with_zlib, True)
+ self.with_lz4 = coalesce(with_lz4, True)
+
+ if self.with_dataset:
+ self.with_filesystem = coalesce(with_filesystem, True)
+ self.with_parquet = coalesce(with_parquet, True)
+
+ if self.with_parquet:
+ self.with_snappy = coalesce(with_snappy, True)
+
+ @property
+ def build_type(self):
+ if self._build_type:
+ return self._build_type
+
+ if self.with_fuzzing:
+ return "relwithdebinfo"
+
+ return "release"
+
+ @property
+ def cc(self):
+ if self._cc:
+ return self._cc
+
+ if self.with_fuzzing:
+ return "clang-{}".format(LLVM_VERSION)
+
+ return None
+
+ @property
+ def cxx(self):
+ if self._cxx:
+ return self._cxx
+
+ if self.with_fuzzing:
+ return "clang++-{}".format(LLVM_VERSION)
+
+ return None
+
+ def _gen_defs(self):
+ if self.cxx_flags:
+ yield ("ARROW_CXXFLAGS", self.cxx_flags)
+
+ yield ("CMAKE_EXPORT_COMPILE_COMMANDS", truthifier(True))
+ yield ("CMAKE_BUILD_TYPE", self.build_type)
+
+ if not self.with_lint_only:
+ yield ("BUILD_WARNING_LEVEL",
+ or_else(self.warn_level, "production"))
+
+ # if not ctx.quiet:
+ # yield ("ARROW_VERBOSE_THIRDPARTY_BUILD", "ON")
+
+ maybe_prefix = self.install_prefix
+ if maybe_prefix:
+ yield ("CMAKE_INSTALL_PREFIX", maybe_prefix)
+
+ if self._package_prefix is not None:
+ yield ("ARROW_DEPENDENCY_SOURCE", "SYSTEM")
+ yield ("ARROW_PACKAGE_PREFIX", self._package_prefix)
+
+ yield ("ARROW_BUILD_STATIC", truthifier(self.build_static))
+ yield ("ARROW_BUILD_SHARED", truthifier(self.build_shared))
+ yield ("CMAKE_UNITY_BUILD", truthifier(self.build_unity))
+
+ # Tests and benchmarks
+ yield ("ARROW_BUILD_TESTS", truthifier(self.with_tests))
+ yield ("ARROW_BUILD_BENCHMARKS", truthifier(self.with_benchmarks))
+ yield ("ARROW_BUILD_EXAMPLES", truthifier(self.with_examples))
+ yield ("ARROW_BUILD_INTEGRATION", truthifier(self.with_integration))
+
+ # Static checks
+ yield ("ARROW_USE_ASAN", truthifier(self.use_asan))
+ yield ("ARROW_USE_TSAN", truthifier(self.use_tsan))
+ yield ("ARROW_USE_UBSAN", truthifier(self.use_ubsan))
+ yield ("ARROW_FUZZING", truthifier(self.with_fuzzing))
+
+ # Components
+ yield ("ARROW_COMPUTE", truthifier(self.with_compute))
+ yield ("ARROW_CSV", truthifier(self.with_csv))
+ yield ("ARROW_CUDA", truthifier(self.with_cuda))
+ yield ("ARROW_DATASET", truthifier(self.with_dataset))
+ yield ("ARROW_FILESYSTEM", truthifier(self.with_filesystem))
+ yield ("ARROW_FLIGHT", truthifier(self.with_flight))
+ yield ("ARROW_GANDIVA", truthifier(self.with_gandiva))
+ yield ("ARROW_PARQUET", truthifier(self.with_parquet))
+ yield ("ARROW_HDFS", truthifier(self.with_hdfs))
+ yield ("ARROW_HIVESERVER2", truthifier(self.with_hiveserver2))
+ yield ("ARROW_IPC", truthifier(self.with_ipc))
+ yield ("ARROW_JSON", truthifier(self.with_json))
+ yield ("ARROW_JNI", truthifier(self.with_jni))
+ yield ("ARROW_MIMALLOC", truthifier(self.with_mimalloc))
+ yield ("ARROW_PLASMA", truthifier(self.with_plasma))
+ yield ("ARROW_PYTHON", truthifier(self.with_python))
+ yield ("ARROW_S3", truthifier(self.with_s3))
+
+ # Compressions
+ yield ("ARROW_WITH_BROTLI", truthifier(self.with_brotli))
+ yield ("ARROW_WITH_BZ2", truthifier(self.with_bz2))
+ yield ("ARROW_WITH_LZ4", truthifier(self.with_lz4))
+ yield ("ARROW_WITH_SNAPPY", truthifier(self.with_snappy))
+ yield ("ARROW_WITH_ZLIB", truthifier(self.with_zlib))
+ yield ("ARROW_WITH_ZSTD", truthifier(self.with_zstd))
+
+ yield ("ARROW_LINT_ONLY", truthifier(self.with_lint_only))
+
+ # Some configurations don't like gnu gold linker.
+ broken_with_gold_ld = [self.with_fuzzing, self.with_gandiva]
+ if self.use_gold_linker and not any(broken_with_gold_ld):
+ yield ("ARROW_USE_LD_GOLD", truthifier(self.use_gold_linker))
+ yield ("ARROW_SIMD_LEVEL", or_else(self.simd_level, "SSE4_2"))
+
+ # Detect custom conda toolchain
+ if self.use_conda:
+ for d, v in [('CMAKE_AR', 'AR'), ('CMAKE_RANLIB', 'RANLIB')]:
+ v = os.environ.get(v)
+ if v:
+ yield (d, v)
+
+ @property
+ def install_prefix(self):
+ if self._install_prefix:
+ return self._install_prefix
+
+ if self.use_conda:
+ return os.environ.get("CONDA_PREFIX")
+
+ return None
+
+ @property
+ def use_conda(self):
+ # If the user didn't specify a preference, guess via environment
+ if self._use_conda is None:
+ return os.environ.get("CONDA_PREFIX") is not None
+
+ return self._use_conda
+
+ @property
+ def definitions(self):
+ extras = list(self.cmake_extras) if self.cmake_extras else []
+ definitions = ["-D{}={}".format(d[0], d[1]) for d in self._gen_defs()]
+ return definitions + extras
+
+ @property
+ def environment(self):
+ env = os.environ.copy()
+
+ if self.cc:
+ env["CC"] = self.cc
+
+ if self.cxx:
+ env["CXX"] = self.cxx
+
+ return env
+
+
+class CppCMakeDefinition(CMakeDefinition):
+ def __init__(self, source, conf, **kwargs):
+ self.configuration = conf
+ super().__init__(source, **kwargs,
+ definitions=conf.definitions, env=conf.environment,
+ build_type=conf.build_type)
diff --git a/src/arrow/dev/archery/archery/lang/java.py b/src/arrow/dev/archery/archery/lang/java.py
new file mode 100644
index 000000000..bc169adf6
--- /dev/null
+++ b/src/arrow/dev/archery/archery/lang/java.py
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from ..utils.command import Command, CommandStackMixin, default_bin
+from ..utils.maven import MavenDefinition
+
+
+class Java(Command):
+ def __init__(self, java_bin=None):
+ self.bin = default_bin(java_bin, "java")
+
+
+class Jar(CommandStackMixin, Java):
+ def __init__(self, jar, *args, **kwargs):
+ self.jar = jar
+ self.argv = ("-jar", jar)
+ Java.__init__(self, *args, **kwargs)
+
+
+class JavaConfiguration:
+ def __init__(self,
+
+ # toolchain
+ java_home=None, java_options=None,
+ # build & benchmark
+ build_extras=None, benchmark_extras=None):
+ self.java_home = java_home
+ self.java_options = java_options
+
+ self.build_extras = list(build_extras) if build_extras else []
+ self.benchmark_extras = list(
+ benchmark_extras) if benchmark_extras else []
+
+ @property
+ def build_definitions(self):
+ return self.build_extras
+
+ @property
+ def benchmark_definitions(self):
+ return self.benchmark_extras
+
+ @property
+ def environment(self):
+ env = os.environ.copy()
+
+ if self.java_home:
+ env["JAVA_HOME"] = self.java_home
+
+ if self.java_options:
+ env["JAVA_OPTIONS"] = self.java_options
+
+ return env
+
+
+class JavaMavenDefinition(MavenDefinition):
+ def __init__(self, source, conf, **kwargs):
+ self.configuration = conf
+ super().__init__(source, **kwargs,
+ build_definitions=conf.build_definitions,
+ benchmark_definitions=conf.benchmark_definitions,
+ env=conf.environment)
diff --git a/src/arrow/dev/archery/archery/lang/python.py b/src/arrow/dev/archery/archery/lang/python.py
new file mode 100644
index 000000000..c6ebbe650
--- /dev/null
+++ b/src/arrow/dev/archery/archery/lang/python.py
@@ -0,0 +1,223 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import inspect
+import tokenize
+from contextlib import contextmanager
+
+try:
+ from numpydoc.validate import Docstring, validate
+except ImportError:
+ have_numpydoc = False
+else:
+ have_numpydoc = True
+
+from ..utils.logger import logger
+from ..utils.command import Command, capture_stdout, default_bin
+
+
+class Flake8(Command):
+ def __init__(self, flake8_bin=None):
+ self.bin = default_bin(flake8_bin, "flake8")
+
+
+class Autopep8(Command):
+ def __init__(self, autopep8_bin=None):
+ self.bin = default_bin(autopep8_bin, "autopep8")
+
+ @capture_stdout()
+ def run_captured(self, *args, **kwargs):
+ return self.run(*args, **kwargs)
+
+
+def _tokenize_signature(s):
+ lines = s.encode('ascii').splitlines()
+ generator = iter(lines).__next__
+ return tokenize.tokenize(generator)
+
+
+def _convert_typehint(tokens):
+ names = []
+ opening_bracket_reached = False
+ for token in tokens:
+ # omit the tokens before the opening bracket
+ if not opening_bracket_reached:
+ if token.string == '(':
+ opening_bracket_reached = True
+ else:
+ continue
+
+ if token.type == 1: # type 1 means NAME token
+ names.append(token)
+ else:
+ if len(names) == 1:
+ yield (names[0].type, names[0].string)
+ elif len(names) == 2:
+ # two "NAME" tokens follow each other which means a cython
+ # typehint like `bool argument`, so remove the typehint
+ # note that we could convert it to python typehints, but hints
+ # are not supported by _signature_fromstr
+ yield (names[1].type, names[1].string)
+ elif len(names) > 2:
+ raise ValueError('More than two NAME tokens follow each other')
+ names = []
+ yield (token.type, token.string)
+
+
+def inspect_signature(obj):
+ """
+ Custom signature inspection primarily for cython generated callables.
+
+ Cython puts the signatures to the first line of the docstrings, which we
+ can reuse to parse the python signature from, but some gymnastics are
+ required, like removing the cython typehints.
+
+ It converts the cython signature:
+ array(obj, type=None, mask=None, size=None, from_pandas=None,
+ bool safe=True, MemoryPool memory_pool=None)
+ To:
+ <Signature (obj, type=None, mask=None, size=None, from_pandas=None,
+ safe=True, memory_pool=None)>
+ """
+ cython_signature = obj.__doc__.splitlines()[0]
+ cython_tokens = _tokenize_signature(cython_signature)
+ python_tokens = _convert_typehint(cython_tokens)
+ python_signature = tokenize.untokenize(python_tokens)
+ return inspect._signature_fromstr(inspect.Signature, obj, python_signature)
+
+
+class NumpyDoc:
+
+ def __init__(self, symbols=None):
+ if not have_numpydoc:
+ raise RuntimeError(
+ 'Numpydoc is not available, install the development version '
+ 'with command: pip install numpydoc==1.1.0'
+ )
+ self.symbols = set(symbols or {'pyarrow'})
+
+ def traverse(self, fn, obj, from_package):
+ """Apply a function on publicly exposed API components.
+
+ Recursively iterates over the members of the passed object. It omits
+ any '_' prefixed and thirdparty (non pyarrow) symbols.
+
+ Parameters
+ ----------
+ obj : Any
+ from_package : string, default 'pyarrow'
+ Predicate to only consider objects from this package.
+ """
+ todo = [obj]
+ seen = set()
+
+ while todo:
+ obj = todo.pop()
+ if obj in seen:
+ continue
+ else:
+ seen.add(obj)
+
+ fn(obj)
+
+ for name in dir(obj):
+ if name.startswith('_'):
+ continue
+
+ member = getattr(obj, name)
+ module = getattr(member, '__module__', None)
+ if not (module and module.startswith(from_package)):
+ continue
+
+ todo.append(member)
+
+ @contextmanager
+ def _apply_patches(self):
+ """
+ Patch Docstring class to bypass loading already loaded python objects.
+ """
+ orig_load_obj = Docstring._load_obj
+ orig_signature = inspect.signature
+
+ @staticmethod
+ def _load_obj(obj):
+ # By default it expects a qualname and import the object, but we
+ # have already loaded object after the API traversal.
+ if isinstance(obj, str):
+ return orig_load_obj(obj)
+ else:
+ return obj
+
+ def signature(obj):
+ # inspect.signature tries to parse __text_signature__ if other
+ # properties like __signature__ doesn't exists, but cython
+ # doesn't set that property despite that embedsignature cython
+ # directive is set. The only way to inspect a cython compiled
+ # callable's signature to parse it from __doc__ while
+ # embedsignature directive is set during the build phase.
+ # So path inspect.signature function to attempt to parse the first
+ # line of callable.__doc__ as a signature.
+ try:
+ return orig_signature(obj)
+ except Exception as orig_error:
+ try:
+ return inspect_signature(obj)
+ except Exception:
+ raise orig_error
+
+ try:
+ Docstring._load_obj = _load_obj
+ inspect.signature = signature
+ yield
+ finally:
+ Docstring._load_obj = orig_load_obj
+ inspect.signature = orig_signature
+
+ def validate(self, from_package='', allow_rules=None,
+ disallow_rules=None):
+ results = []
+
+ def callback(obj):
+ try:
+ result = validate(obj)
+ except OSError as e:
+ symbol = f"{obj.__module__}.{obj.__name__}"
+ logger.warning(f"Unable to validate `{symbol}` due to `{e}`")
+ return
+
+ errors = []
+ for errcode, errmsg in result.get('errors', []):
+ if allow_rules and errcode not in allow_rules:
+ continue
+ if disallow_rules and errcode in disallow_rules:
+ continue
+ errors.append((errcode, errmsg))
+
+ if len(errors):
+ result['errors'] = errors
+ results.append((obj, result))
+
+ with self._apply_patches():
+ for symbol in self.symbols:
+ try:
+ obj = Docstring._load_obj(symbol)
+ except (ImportError, AttributeError):
+ print('{} is not available for import'.format(symbol))
+ else:
+ self.traverse(callback, obj, from_package=from_package)
+
+ return results
diff --git a/src/arrow/dev/archery/archery/linking.py b/src/arrow/dev/archery/archery/linking.py
new file mode 100644
index 000000000..c2e6f1772
--- /dev/null
+++ b/src/arrow/dev/archery/archery/linking.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import platform
+import subprocess
+
+from .utils.command import Command
+
+
+_ldd = Command("ldd")
+_otool = Command("otool")
+
+
+class DependencyError(Exception):
+ pass
+
+
+class DynamicLibrary:
+
+ def __init__(self, path):
+ self.path = path
+
+ def list_dependencies(self):
+ """
+ List the full name of the library dependencies.
+ """
+ system = platform.system()
+ if system == "Linux":
+ result = _ldd.run(self.path, stdout=subprocess.PIPE)
+ lines = result.stdout.splitlines()
+ return [ll.split(None, 1)[0].decode() for ll in lines]
+ elif system == "Darwin":
+ result = _otool.run("-L", self.path, stdout=subprocess.PIPE)
+ lines = result.stdout.splitlines()
+ return [dl.split(None, 1)[0].decode() for dl in lines]
+ else:
+ raise ValueError(f"{platform} is not supported")
+
+ def list_dependency_names(self):
+ """
+ List the truncated names of the dynamic library dependencies.
+ """
+ names = []
+ for dependency in self.list_dependencies():
+ *_, library = dependency.rsplit("/", 1)
+ name, *_ = library.split(".", 1)
+ names.append(name)
+ return names
+
+
+def check_dynamic_library_dependencies(path, allowed, disallowed):
+ dylib = DynamicLibrary(path)
+ for dep in dylib.list_dependency_names():
+ if allowed and dep not in allowed:
+ raise DependencyError(
+ f"Unexpected shared dependency found in {dylib.path}: `{dep}`"
+ )
+ if disallowed and dep in disallowed:
+ raise DependencyError(
+ f"Disallowed shared dependency found in {dylib.path}: `{dep}`"
+ )
diff --git a/src/arrow/dev/archery/archery/release.py b/src/arrow/dev/archery/archery/release.py
new file mode 100644
index 000000000..6baeabc9d
--- /dev/null
+++ b/src/arrow/dev/archery/archery/release.py
@@ -0,0 +1,535 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import defaultdict
+import functools
+import os
+import re
+import pathlib
+import shelve
+import warnings
+
+from git import Repo
+from jira import JIRA
+from semver import VersionInfo as SemVer
+
+from .utils.source import ArrowSources
+from .utils.report import JinjaReport
+
+
+def cached_property(fn):
+ return property(functools.lru_cache(maxsize=1)(fn))
+
+
+class Version(SemVer):
+
+ __slots__ = ('released', 'release_date')
+
+ def __init__(self, released=False, release_date=None, **kwargs):
+ super().__init__(**kwargs)
+ self.released = released
+ self.release_date = release_date
+
+ @classmethod
+ def parse(cls, version, **kwargs):
+ return cls(**SemVer.parse(version).to_dict(), **kwargs)
+
+ @classmethod
+ def from_jira(cls, jira_version):
+ return cls.parse(
+ jira_version.name,
+ released=jira_version.released,
+ release_date=getattr(jira_version, 'releaseDate', None)
+ )
+
+
+class Issue:
+
+ def __init__(self, key, type, summary):
+ self.key = key
+ self.type = type
+ self.summary = summary
+
+ @classmethod
+ def from_jira(cls, jira_issue):
+ return cls(
+ key=jira_issue.key,
+ type=jira_issue.fields.issuetype.name,
+ summary=jira_issue.fields.summary
+ )
+
+ @property
+ def project(self):
+ return self.key.split('-')[0]
+
+ @property
+ def number(self):
+ return int(self.key.split('-')[1])
+
+
+class Jira(JIRA):
+
+ def __init__(self, user=None, password=None,
+ url='https://issues.apache.org/jira'):
+ user = user or os.environ.get('APACHE_JIRA_USER')
+ password = password or os.environ.get('APACHE_JIRA_PASSWORD')
+ super().__init__(url, basic_auth=(user, password))
+
+ def project_version(self, version_string, project='ARROW'):
+ # query version from jira to populated with additional metadata
+ versions = {str(v): v for v in self.project_versions(project)}
+ return versions[version_string]
+
+ def project_versions(self, project):
+ versions = []
+ for v in super().project_versions(project):
+ try:
+ versions.append(Version.from_jira(v))
+ except ValueError:
+ # ignore invalid semantic versions like JS-0.4.0
+ continue
+ return sorted(versions, reverse=True)
+
+ def issue(self, key):
+ return Issue.from_jira(super().issue(key))
+
+ def project_issues(self, version, project='ARROW'):
+ query = "project={} AND fixVersion={}".format(project, version)
+ issues = super().search_issues(query, maxResults=False)
+ return list(map(Issue.from_jira, issues))
+
+
+class CachedJira:
+
+ def __init__(self, cache_path, jira=None):
+ self.jira = jira or Jira()
+ self.cache_path = cache_path
+
+ def __getattr__(self, name):
+ attr = getattr(self.jira, name)
+ return self._cached(name, attr) if callable(attr) else attr
+
+ def _cached(self, name, method):
+ def wrapper(*args, **kwargs):
+ key = str((name, args, kwargs))
+ with shelve.open(self.cache_path) as cache:
+ try:
+ result = cache[key]
+ except KeyError:
+ cache[key] = result = method(*args, **kwargs)
+ return result
+ return wrapper
+
+
+_TITLE_REGEX = re.compile(
+ r"(?P<issue>(?P<project>(ARROW|PARQUET))\-\d+)?\s*:?\s*"
+ r"(?P<components>\[.*\])?\s*(?P<summary>.*)"
+)
+_COMPONENT_REGEX = re.compile(r"\[([^\[\]]+)\]")
+
+
+class CommitTitle:
+
+ def __init__(self, summary, project=None, issue=None, components=None):
+ self.project = project
+ self.issue = issue
+ self.components = components or []
+ self.summary = summary
+
+ def __str__(self):
+ out = ""
+ if self.issue:
+ out += "{}: ".format(self.issue)
+ if self.components:
+ for component in self.components:
+ out += "[{}]".format(component)
+ out += " "
+ out += self.summary
+ return out
+
+ def __eq__(self, other):
+ return (
+ self.summary == other.summary and
+ self.project == other.project and
+ self.issue == other.issue and
+ self.components == other.components
+ )
+
+ def __hash__(self):
+ return hash(
+ (self.summary, self.project, self.issue, tuple(self.components))
+ )
+
+ @classmethod
+ def parse(cls, headline):
+ matches = _TITLE_REGEX.match(headline)
+ if matches is None:
+ warnings.warn(
+ "Unable to parse commit message `{}`".format(headline)
+ )
+ return CommitTitle(headline)
+
+ values = matches.groupdict()
+ components = values.get('components') or ''
+ components = _COMPONENT_REGEX.findall(components)
+
+ return CommitTitle(
+ values['summary'],
+ project=values.get('project'),
+ issue=values.get('issue'),
+ components=components
+ )
+
+
+class Commit:
+
+ def __init__(self, wrapped):
+ self._title = CommitTitle.parse(wrapped.summary)
+ self._wrapped = wrapped
+
+ def __getattr__(self, attr):
+ if hasattr(self._title, attr):
+ return getattr(self._title, attr)
+ else:
+ return getattr(self._wrapped, attr)
+
+ def __repr__(self):
+ template = '<Commit sha={!r} issue={!r} components={!r} summary={!r}>'
+ return template.format(self.hexsha, self.issue, self.components,
+ self.summary)
+
+ @property
+ def url(self):
+ return 'https://github.com/apache/arrow/commit/{}'.format(self.hexsha)
+
+ @property
+ def title(self):
+ return self._title
+
+
+class ReleaseCuration(JinjaReport):
+ templates = {
+ 'console': 'release_curation.txt.j2'
+ }
+ fields = [
+ 'release',
+ 'within',
+ 'outside',
+ 'nojira',
+ 'parquet',
+ 'nopatch'
+ ]
+
+
+class JiraChangelog(JinjaReport):
+ templates = {
+ 'markdown': 'release_changelog.md.j2',
+ 'html': 'release_changelog.html.j2'
+ }
+ fields = [
+ 'release',
+ 'categories'
+ ]
+
+
+class Release:
+
+ def __init__(self):
+ raise TypeError("Do not initialize Release class directly, use "
+ "Release.from_jira(version) instead.")
+
+ def __repr__(self):
+ if self.version.released:
+ status = "released_at={!r}".format(self.version.release_date)
+ else:
+ status = "pending"
+ return "<{} {!r} {}>".format(self.__class__.__name__,
+ str(self.version), status)
+
+ @staticmethod
+ def from_jira(version, jira=None, repo=None):
+ if jira is None:
+ jira = Jira()
+ elif isinstance(jira, str):
+ jira = Jira(jira)
+ elif not isinstance(jira, (Jira, CachedJira)):
+ raise TypeError("`jira` argument must be a server url or a valid "
+ "Jira instance")
+
+ if repo is None:
+ arrow = ArrowSources.find()
+ repo = Repo(arrow.path)
+ elif isinstance(repo, (str, pathlib.Path)):
+ repo = Repo(repo)
+ elif not isinstance(repo, Repo):
+ raise TypeError("`repo` argument must be a path or a valid Repo "
+ "instance")
+
+ if isinstance(version, str):
+ version = jira.project_version(version, project='ARROW')
+ elif not isinstance(version, Version):
+ raise TypeError(version)
+
+ # decide the type of the release based on the version number
+ if version.patch == 0:
+ if version.minor == 0:
+ klass = MajorRelease
+ elif version.major == 0:
+ # handle minor releases before 1.0 as major releases
+ klass = MajorRelease
+ else:
+ klass = MinorRelease
+ else:
+ klass = PatchRelease
+
+ # prevent instantiating release object directly
+ obj = klass.__new__(klass)
+ obj.version = version
+ obj.jira = jira
+ obj.repo = repo
+
+ return obj
+
+ @property
+ def is_released(self):
+ return self.version.released
+
+ @property
+ def tag(self):
+ return "apache-arrow-{}".format(str(self.version))
+
+ @property
+ def branch(self):
+ raise NotImplementedError()
+
+ @property
+ def siblings(self):
+ """
+ Releases to consider when calculating previous and next releases.
+ """
+ raise NotImplementedError()
+
+ @cached_property
+ def previous(self):
+ # select all non-patch releases
+ position = self.siblings.index(self.version)
+ try:
+ previous = self.siblings[position + 1]
+ except IndexError:
+ # first release doesn't have a previous one
+ return None
+ else:
+ return Release.from_jira(previous, jira=self.jira, repo=self.repo)
+
+ @cached_property
+ def next(self):
+ # select all non-patch releases
+ position = self.siblings.index(self.version)
+ if position <= 0:
+ raise ValueError("There is no upcoming release set in JIRA after "
+ "version {}".format(self.version))
+ upcoming = self.siblings[position - 1]
+ return Release.from_jira(upcoming, jira=self.jira, repo=self.repo)
+
+ @cached_property
+ def issues(self):
+ issues = self.jira.project_issues(self.version, project='ARROW')
+ return {i.key: i for i in issues}
+
+ @cached_property
+ def commits(self):
+ """
+ All commits applied between two versions.
+ """
+ if self.previous is None:
+ # first release
+ lower = ''
+ else:
+ lower = self.repo.tags[self.previous.tag]
+
+ if self.version.released:
+ upper = self.repo.tags[self.tag]
+ else:
+ try:
+ upper = self.repo.branches[self.branch]
+ except IndexError:
+ warnings.warn("Release branch `{}` doesn't exist."
+ .format(self.branch))
+ return []
+
+ commit_range = "{}..{}".format(lower, upper)
+ return list(map(Commit, self.repo.iter_commits(commit_range)))
+
+ def curate(self):
+ # handle commits with parquet issue key specially and query them from
+ # jira and add it to the issues
+ release_issues = self.issues
+
+ within, outside, nojira, parquet = [], [], [], []
+ for c in self.commits:
+ if c.issue is None:
+ nojira.append(c)
+ elif c.issue in release_issues:
+ within.append((release_issues[c.issue], c))
+ elif c.project == 'PARQUET':
+ parquet.append((self.jira.issue(c.issue), c))
+ else:
+ outside.append((self.jira.issue(c.issue), c))
+
+ # remaining jira tickets
+ within_keys = {i.key for i, c in within}
+ nopatch = [issue for key, issue in release_issues.items()
+ if key not in within_keys]
+
+ return ReleaseCuration(release=self, within=within, outside=outside,
+ nojira=nojira, parquet=parquet, nopatch=nopatch)
+
+ def changelog(self):
+ release_issues = []
+
+ # get organized report for the release
+ curation = self.curate()
+
+ # jira tickets having patches in the release
+ for issue, _ in curation.within:
+ release_issues.append(issue)
+
+ # jira tickets without patches
+ for issue in curation.nopatch:
+ release_issues.append(issue)
+
+ # parquet patches in the release
+ for issue, _ in curation.parquet:
+ release_issues.append(issue)
+
+ # organize issues into categories
+ issue_types = {
+ 'Bug': 'Bug Fixes',
+ 'Improvement': 'New Features and Improvements',
+ 'New Feature': 'New Features and Improvements',
+ 'Sub-task': 'New Features and Improvements',
+ 'Task': 'New Features and Improvements',
+ 'Test': 'Bug Fixes',
+ 'Wish': 'New Features and Improvements',
+ }
+ categories = defaultdict(list)
+ for issue in release_issues:
+ categories[issue_types[issue.type]].append(issue)
+
+ # sort issues by the issue key in ascending order
+ for name, issues in categories.items():
+ issues.sort(key=lambda issue: (issue.project, issue.number))
+
+ return JiraChangelog(release=self, categories=categories)
+
+
+class MaintenanceMixin:
+ """
+ Utility methods for cherry-picking commits from the main branch.
+ """
+
+ def commits_to_pick(self, exclude_already_applied=True):
+ # collect commits applied on the main branch since the root of the
+ # maintenance branch (the previous major release)
+ if self.version.major == 0:
+ # treat minor releases as major releases preceeding 1.0.0 release
+ commit_range = "apache-arrow-0.{}.0..master".format(
+ self.version.minor
+ )
+ else:
+ commit_range = "apache-arrow-{}.0.0..master".format(
+ self.version.major
+ )
+
+ # keeping the original order of the commits helps to minimize the merge
+ # conflicts during cherry-picks
+ commits = map(Commit, self.repo.iter_commits(commit_range))
+
+ # exclude patches that have been already applied to the maintenance
+ # branch, we cannot identify patches based on sha because it changes
+ # after the cherry pick so use commit title instead
+ if exclude_already_applied:
+ already_applied = {c.title for c in self.commits}
+ else:
+ already_applied = set()
+
+ # iterate over the commits applied on the main branch and filter out
+ # the ones that are included in the jira release
+ patches_to_pick = [c for c in commits if
+ c.issue in self.issues and
+ c.title not in already_applied]
+
+ return reversed(patches_to_pick)
+
+ def cherry_pick_commits(self, recreate_branch=True):
+ if recreate_branch:
+ # delete, create and checkout the maintenance branch based off of
+ # the previous tag
+ if self.branch in self.repo.branches:
+ self.repo.git.branch('-D', self.branch)
+ self.repo.git.checkout(self.previous.tag, b=self.branch)
+ else:
+ # just checkout the already existing maintenance branch
+ self.repo.git.checkout(self.branch)
+
+ # cherry pick the commits based on the jira tickets
+ for commit in self.commits_to_pick():
+ self.repo.git.cherry_pick(commit.hexsha)
+
+
+class MajorRelease(Release):
+
+ @property
+ def branch(self):
+ return "master"
+
+ @cached_property
+ def siblings(self):
+ """
+ Filter only the major releases.
+ """
+ # handle minor releases before 1.0 as major releases
+ return [v for v in self.jira.project_versions('ARROW')
+ if v.patch == 0 and (v.major == 0 or v.minor == 0)]
+
+
+class MinorRelease(Release, MaintenanceMixin):
+
+ @property
+ def branch(self):
+ return "maint-{}.x.x".format(self.version.major)
+
+ @cached_property
+ def siblings(self):
+ """
+ Filter the major and minor releases.
+ """
+ return [v for v in self.jira.project_versions('ARROW') if v.patch == 0]
+
+
+class PatchRelease(Release, MaintenanceMixin):
+
+ @property
+ def branch(self):
+ return "maint-{}.{}.x".format(self.version.major, self.version.minor)
+
+ @cached_property
+ def siblings(self):
+ """
+ No filtering, consider all releases.
+ """
+ return self.jira.project_versions('ARROW')
diff --git a/src/arrow/dev/archery/archery/templates/release_changelog.md.j2 b/src/arrow/dev/archery/archery/templates/release_changelog.md.j2
new file mode 100644
index 000000000..c0406ddf4
--- /dev/null
+++ b/src/arrow/dev/archery/archery/templates/release_changelog.md.j2
@@ -0,0 +1,29 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#}
+# Apache Arrow {{ release.version }} ({{ release.version.release_date or today() }})
+
+{% for category, issues in categories.items() -%}
+
+## {{ category }}
+
+{% for issue in issues -%}
+* [{{ issue.key }}](https://issues.apache.org/jira/browse/{{ issue.key }}) - {{ issue.summary | md }}
+{% endfor %}
+
+{% endfor %}
diff --git a/src/arrow/dev/archery/archery/templates/release_curation.txt.j2 b/src/arrow/dev/archery/archery/templates/release_curation.txt.j2
new file mode 100644
index 000000000..a5d11e9d4
--- /dev/null
+++ b/src/arrow/dev/archery/archery/templates/release_curation.txt.j2
@@ -0,0 +1,41 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#}
+Total number of JIRA tickets assigned to version {{ release.version }}: {{ release.issues|length }}
+
+Total number of applied patches since version {{ release.previous.version }}: {{ release.commits|length }}
+
+Patches with assigned issue in version {{ release.version }}:
+{% for issue, commit in within -%}
+ - {{ commit.url }} {{ commit.title }}
+{% endfor %}
+
+Patches with assigned issue outside of version {{ release.version }}:
+{% for issue, commit in outside -%}
+ - {{ commit.url }} {{ commit.title }}
+{% endfor %}
+
+Patches in version {{ release.version }} without a linked issue:
+{% for commit in nojira -%}
+ - {{ commit.url }} {{ commit.title }}
+{% endfor %}
+
+JIRA issues in version {{ release.version }} without a linked patch:
+{% for issue in nopatch -%}
+ - https://issues.apache.org/jira/browse/{{ issue.key }}
+{% endfor %}
diff --git a/src/arrow/dev/archery/archery/testing.py b/src/arrow/dev/archery/archery/testing.py
new file mode 100644
index 000000000..471a54d4c
--- /dev/null
+++ b/src/arrow/dev/archery/archery/testing.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from contextlib import contextmanager
+import os
+from unittest import mock
+import re
+
+
+class DotDict(dict):
+
+ def __getattr__(self, key):
+ try:
+ item = self[key]
+ except KeyError:
+ raise AttributeError(key)
+ if isinstance(item, dict):
+ return DotDict(item)
+ else:
+ return item
+
+
+class PartialEnv(dict):
+
+ def __eq__(self, other):
+ return self.items() <= other.items()
+
+
+_mock_call_type = type(mock.call())
+
+
+def _ensure_mock_call_object(obj, **kwargs):
+ if isinstance(obj, _mock_call_type):
+ return obj
+ elif isinstance(obj, str):
+ cmd = re.split(r"\s+", obj)
+ return mock.call(cmd, **kwargs)
+ elif isinstance(obj, list):
+ return mock.call(obj, **kwargs)
+ else:
+ raise TypeError(obj)
+
+
+class SuccessfulSubprocessResult:
+
+ def check_returncode(self):
+ return
+
+
+@contextmanager
+def assert_subprocess_calls(expected_commands_or_calls, **kwargs):
+ calls = [
+ _ensure_mock_call_object(obj, **kwargs)
+ for obj in expected_commands_or_calls
+ ]
+ with mock.patch('subprocess.run', autospec=True) as run:
+ run.return_value = SuccessfulSubprocessResult()
+ yield run
+ run.assert_has_calls(calls)
+
+
+@contextmanager
+def override_env(mapping):
+ original = os.environ
+ try:
+ os.environ = dict(os.environ, **mapping)
+ yield os.environ
+ finally:
+ os.environ = original
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl
new file mode 100644
index 000000000..5854eb75c
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl
@@ -0,0 +1,6 @@
+{"benchmark": "RegressionSumKernel/32768/10", "change": 0.0046756468886368545, "regression": false, "baseline": 13265442258.099466, "contender": 13327466781.91994, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"}
+{"benchmark": "RegressionSumKernel/32768/1", "change": 0.0025108399115900733, "regression": false, "baseline": 15181891659.539782, "contender": 15220010959.05199, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"}
+
+{"benchmark": "RegressionSumKernel/32768/50", "change": 0.00346735806287155, "regression": false, "baseline": 11471825667.817123, "contender": 11511602595.042286, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"}
+
+{"benchmark": "RegressionSumKernel/32768/0", "change": 0.010140954727954987, "regression": false, "baseline": 18316987019.994465, "contender": 18502738756.116768, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"}
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl
new file mode 100644
index 000000000..1e25810d7
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl
@@ -0,0 +1,4 @@
+{"benchmark":"RegressionSumKernel/32768/50","change":-0.001550846227215492,"regression":false,"baseline":19241207435.428757,"contender":19211367281.47045,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"}
+{"benchmark":"RegressionSumKernel/32768/1","change":0.0020681767923465765,"regression":true,"baseline":24823170673.777943,"contender":24771831968.277977,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"}
+{"benchmark":"RegressionSumKernel/32768/10","change":0.0033323376378746905,"regression":false,"baseline":21902707565.968014,"contender":21975694782.76145,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"}
+{"benchmark":"RegressionSumKernel/32768/0","change":-0.004918126090954414,"regression":true,"baseline":27685006611.446762,"contender":27821164964.790764,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"}
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json
new file mode 100644
index 000000000..d591105f0
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json
@@ -0,0 +1,212 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "MEMBER",
+ "body": "@ursabot build",
+ "created_at": "2019-04-05T11:55:43Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726",
+ "id": 480248726,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==",
+ "updated_at": "2019-04-05T11:55:43Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "MEMBER",
+ "body": "",
+ "closed_at": null,
+ "comments": 3,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "created_at": "2019-04-05T11:22:15Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "pull_request": {
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Unittests for GithubHook",
+ "updated_at": "2019-04-05T11:55:43Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json
new file mode 100644
index 000000000..5a8f3461c
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json
@@ -0,0 +1,212 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "NONE",
+ "body": "Unknown command \"\"",
+ "created_at": "2019-04-05T11:35:47Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815",
+ "id": 480243815,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==",
+ "updated_at": "2019-04-05T11:35:47Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815",
+ "user": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4",
+ "events_url": "https://api.github.com/users/ursabot/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursabot/followers",
+ "following_url": "https://api.github.com/users/ursabot/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursabot",
+ "id": 49275095,
+ "login": "someone",
+ "node_id": "MDQ6VXNlcjQ5Mjc1MDk1",
+ "organizations_url": "https://api.github.com/users/ursabot/orgs",
+ "received_events_url": "https://api.github.com/users/ursabot/received_events",
+ "repos_url": "https://api.github.com/users/ursabot/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/ursabot"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "NONE",
+ "body": "",
+ "closed_at": null,
+ "comments": 2,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "created_at": "2019-04-05T11:22:15Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "pull_request": {
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Unittests for GithubHook",
+ "updated_at": "2019-04-05T11:35:47Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "someone",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4",
+ "events_url": "https://api.github.com/users/ursabot/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursabot/followers",
+ "following_url": "https://api.github.com/users/ursabot/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursabot",
+ "id": 49275095,
+ "login": "someone",
+ "node_id": "MDQ6VXNlcjQ5Mjc1MDk1",
+ "organizations_url": "https://api.github.com/users/ursabot/orgs",
+ "received_events_url": "https://api.github.com/users/ursabot/received_events",
+ "repos_url": "https://api.github.com/users/ursabot/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/ursabot"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json
new file mode 100644
index 000000000..bfb7210df
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json
@@ -0,0 +1,212 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "NONE",
+ "body": "Unknown command \"\"",
+ "created_at": "2019-04-05T11:35:47Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815",
+ "id": 480243815,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==",
+ "updated_at": "2019-04-05T11:35:47Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815",
+ "user": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4",
+ "events_url": "https://api.github.com/users/ursabot/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursabot/followers",
+ "following_url": "https://api.github.com/users/ursabot/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursabot",
+ "id": 49275095,
+ "login": "ursabot",
+ "node_id": "MDQ6VXNlcjQ5Mjc1MDk1",
+ "organizations_url": "https://api.github.com/users/ursabot/orgs",
+ "received_events_url": "https://api.github.com/users/ursabot/received_events",
+ "repos_url": "https://api.github.com/users/ursabot/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/ursabot"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "MEMBER",
+ "body": "",
+ "closed_at": null,
+ "comments": 2,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "created_at": "2019-04-05T11:22:15Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "pull_request": {
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Unittests for GithubHook",
+ "updated_at": "2019-04-05T11:35:47Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4",
+ "events_url": "https://api.github.com/users/ursabot/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursabot/followers",
+ "following_url": "https://api.github.com/users/ursabot/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursabot",
+ "id": 49275095,
+ "login": "ursabot",
+ "node_id": "MDQ6VXNlcjQ5Mjc1MDk1",
+ "organizations_url": "https://api.github.com/users/ursabot/orgs",
+ "received_events_url": "https://api.github.com/users/ursabot/received_events",
+ "repos_url": "https://api.github.com/users/ursabot/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/ursabot"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json
new file mode 100644
index 000000000..a3d450078
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json
@@ -0,0 +1,212 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "MEMBER",
+ "body": "bear is no game",
+ "created_at": "2019-04-05T11:26:56Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727",
+ "id": 480241727,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==",
+ "updated_at": "2019-04-05T11:26:56Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "MEMBER",
+ "body": "",
+ "closed_at": null,
+ "comments": 0,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "created_at": "2019-04-05T11:22:15Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "pull_request": {
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Unittests for GithubHook",
+ "updated_at": "2019-04-05T11:26:56Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json
new file mode 100644
index 000000000..c88197c8e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json
@@ -0,0 +1,217 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "MEMBER",
+ "body": "@ursabot ",
+ "body_html": "",
+ "body_text": "",
+ "created_at": "2019-04-05T11:35:46Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811",
+ "id": 480243811,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==",
+ "updated_at": "2019-04-05T11:35:46Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "MEMBER",
+ "body": "",
+ "body_html": "",
+ "body_text": "",
+ "closed_at": null,
+ "closed_by": null,
+ "comments": 1,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "created_at": "2019-04-05T11:22:15Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "pull_request": {
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Unittests for GithubHook",
+ "updated_at": "2019-04-05T11:35:46Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json
new file mode 100644
index 000000000..9e362fc0e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json
@@ -0,0 +1,206 @@
+{
+ "action": "created",
+ "comment": {
+ "author_association": "MEMBER",
+ "body": "@ursabot build",
+ "created_at": "2019-04-05T13:07:57Z",
+ "html_url": "https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480268708",
+ "id": 480268708,
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19",
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI2ODcwOA==",
+ "updated_at": "2019-04-05T13:07:57Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480268708",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "issue": {
+ "assignee": null,
+ "assignees": [],
+ "author_association": "MEMBER",
+ "body": "",
+ "closed_at": null,
+ "comments": 5,
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments",
+ "created_at": "2019-04-02T09:56:41Z",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/issues/19",
+ "id": 428131685,
+ "labels": [],
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}",
+ "locked": false,
+ "milestone": null,
+ "node_id": "MDU6SXNzdWU0MjgxMzE2ODU=",
+ "number": 19,
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "state": "open",
+ "title": "Build ursabot itself via ursabot",
+ "updated_at": "2019-04-05T13:07:57Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19",
+ "user": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+ },
+ "organization": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "description": "Innovation lab for open source data science tools, powered by Apache Arrow",
+ "events_url": "https://api.github.com/orgs/ursa-labs/events",
+ "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks",
+ "id": 46514972,
+ "issues_url": "https://api.github.com/orgs/ursa-labs/issues",
+ "login": "ursa-labs",
+ "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}",
+ "repos_url": "https://api.github.com/orgs/ursa-labs/repos",
+ "url": "https://api.github.com/orgs/ursa-labs"
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T12:01:40Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 898,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json b/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json
new file mode 100644
index 000000000..9cf5c0dda
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json
@@ -0,0 +1,445 @@
+{
+ "action": "opened",
+ "number": 26,
+ "pull_request": {
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26",
+ "id": 267785552,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "number": 26,
+ "state": "open",
+ "locked": false,
+ "title": "Unittests for GithubHook",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "body": "",
+ "created_at": "2019-04-05T11:22:15Z",
+ "updated_at": "2019-04-05T12:01:40Z",
+ "closed_at": null,
+ "merged_at": null,
+ "merge_commit_sha": "cc5dc3606988b3824be54df779ed2028776113cb",
+ "assignee": null,
+ "assignees": [],
+ "requested_reviewers": [],
+ "requested_teams": [],
+ "labels": [],
+ "milestone": null,
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits",
+ "review_comments_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments",
+ "review_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "head": {
+ "label": "ursa-labs:test-hook",
+ "ref": "test-hook",
+ "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "user": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "repo": {
+ "id": 169101701,
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "name": "ursabot",
+ "full_name": "ursa-labs/ursabot",
+ "private": false,
+ "owner": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "description": null,
+ "fork": false,
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "created_at": "2019-02-04T15:40:31Z",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "pushed_at": "2019-04-05T12:01:40Z",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "homepage": null,
+ "size": 898,
+ "stargazers_count": 1,
+ "watchers_count": 1,
+ "language": "Jupyter Notebook",
+ "has_issues": true,
+ "has_projects": true,
+ "has_downloads": true,
+ "has_wiki": true,
+ "has_pages": false,
+ "forks_count": 0,
+ "mirror_url": null,
+ "archived": false,
+ "disabled": false,
+ "open_issues_count": 19,
+ "license": null,
+ "forks": 0,
+ "open_issues": 19,
+ "watchers": 1,
+ "default_branch": "master"
+ }
+ },
+ "base": {
+ "label": "ursa-labs:master",
+ "ref": "master",
+ "sha": "a162ad254b589b924db47e057791191b39613fd5",
+ "user": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "repo": {
+ "id": 169101701,
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "name": "ursabot",
+ "full_name": "ursa-labs/ursabot",
+ "private": false,
+ "owner": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "description": null,
+ "fork": false,
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "created_at": "2019-02-04T15:40:31Z",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "pushed_at": "2019-04-05T12:01:40Z",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "homepage": null,
+ "size": 898,
+ "stargazers_count": 1,
+ "watchers_count": 1,
+ "language": "Jupyter Notebook",
+ "has_issues": true,
+ "has_projects": true,
+ "has_downloads": true,
+ "has_wiki": true,
+ "has_pages": false,
+ "forks_count": 0,
+ "mirror_url": null,
+ "archived": false,
+ "disabled": false,
+ "open_issues_count": 19,
+ "license": null,
+ "forks": 0,
+ "open_issues": 19,
+ "watchers": 1,
+ "default_branch": "master"
+ }
+ },
+ "_links": {
+ "self": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "html": {
+ "href": "https://github.com/ursa-labs/ursabot/pull/26"
+ },
+ "issue": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26"
+ },
+ "comments": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments"
+ },
+ "review_comments": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments"
+ },
+ "review_comment": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}"
+ },
+ "commits": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits"
+ },
+ "statuses": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d"
+ }
+ },
+ "author_association": "MEMBER",
+ "merged": false,
+ "mergeable": true,
+ "rebaseable": true,
+ "mergeable_state": "unstable",
+ "merged_by": null,
+ "comments": 5,
+ "review_comments": 0,
+ "maintainer_can_modify": false,
+ "commits": 2,
+ "additions": 1124,
+ "deletions": 0,
+ "changed_files": 7
+ },
+ "repository": {
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "archived": false,
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "created_at": "2019-02-04T15:40:31Z",
+ "default_branch": "master",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "description": null,
+ "disabled": false,
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "fork": false,
+ "forks": 0,
+ "forks_count": 0,
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "full_name": "ursa-labs/ursabot",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "has_downloads": true,
+ "has_issues": true,
+ "has_pages": false,
+ "has_projects": true,
+ "has_wiki": true,
+ "homepage": null,
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "id": 169101701,
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "language": "Jupyter Notebook",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "license": null,
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "mirror_url": null,
+ "name": "ursabot",
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "open_issues": 19,
+ "open_issues_count": 19,
+ "owner": {
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/ursa-labs",
+ "id": 46514972,
+ "login": "ursa-labs",
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "type": "Organization",
+ "url": "https://api.github.com/users/ursa-labs"
+ },
+ "private": false,
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "pushed_at": "2019-04-05T11:22:16Z",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "size": 892,
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "stargazers_count": 1,
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "watchers": 1,
+ "watchers_count": 1
+ },
+ "sender": {
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "gravatar_id": "",
+ "html_url": "https://github.com/kszucs",
+ "id": 961747,
+ "login": "kszucs",
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "site_admin": false,
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "type": "User",
+ "url": "https://api.github.com/users/kszucs"
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json
new file mode 100644
index 000000000..1e4939776
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json
@@ -0,0 +1,64 @@
+{
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19",
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/issues/19",
+ "id": 428131685,
+ "node_id": "MDU6SXNzdWU0MjgxMzE2ODU=",
+ "number": 19,
+ "title": "Build ursabot itself via ursabot",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "labels": [],
+ "state": "closed",
+ "locked": false,
+ "assignee": null,
+ "assignees": [],
+ "milestone": null,
+ "comments": 8,
+ "created_at": "2019-04-02T09:56:41Z",
+ "updated_at": "2019-04-05T13:30:49Z",
+ "closed_at": "2019-04-05T13:30:49Z",
+ "author_association": "MEMBER",
+ "body": "",
+ "closed_by": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json
new file mode 100644
index 000000000..44c4d3bed
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json
@@ -0,0 +1,70 @@
+{
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "repository_url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "id": 429706959,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "number": 26,
+ "title": "Unittests for GithubHook + native asyncio syntax",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "labels": [],
+ "state": "closed",
+ "locked": false,
+ "assignee": null,
+ "assignees": [],
+ "milestone": null,
+ "comments": 9,
+ "created_at": "2019-04-05T11:22:15Z",
+ "updated_at": "2019-08-28T00:34:19Z",
+ "closed_at": "2019-04-05T13:54:34Z",
+ "author_association": "MEMBER",
+ "pull_request": {
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch"
+ },
+ "body": "Resolves:\r\n- #26 Unittests for GithubHook + native asyncio syntax\r\n- #27 Use native async/await keywords instead of @inlineCallbacks and yield\r\n",
+ "closed_by": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ }
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json
new file mode 100644
index 000000000..93ee4b13c
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json
@@ -0,0 +1,31 @@
+{
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/479081273",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/21#issuecomment-479081273",
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/21",
+ "id": 480243811,
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ3OTA4MTI3Mw==",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "created_at": "2019-04-02T16:29:46Z",
+ "updated_at": "2019-04-02T16:29:46Z",
+ "author_association": "MEMBER",
+ "body": "@ursabot"
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json
new file mode 100644
index 000000000..f3cd34083
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json
@@ -0,0 +1,31 @@
+{
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726",
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "id": 480248726,
+ "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "created_at": "2019-04-05T11:55:43Z",
+ "updated_at": "2019-04-05T11:55:43Z",
+ "author_association": "MEMBER",
+ "body": "@ursabot build"
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json
new file mode 100644
index 000000000..ffc48943a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json
@@ -0,0 +1,158 @@
+{
+ "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "node_id": "MDY6Q29tbWl0MTY5MTAxNzAxOjI3MDVkYTJiNjE2Yjk4ZmE2MDEwYTI1ODEzYzVhN2EyNzQ1NmY3MWQ=",
+ "commit": {
+ "author": {
+ "name": "Krisztián Szűcs",
+ "email": "szucs.krisztian@gmail.com",
+ "date": "2019-04-05T12:01:31Z"
+ },
+ "committer": {
+ "name": "Krisztián Szűcs",
+ "email": "szucs.krisztian@gmail.com",
+ "date": "2019-04-05T12:01:31Z"
+ },
+ "message": "add recorded event requests",
+ "tree": {
+ "sha": "16a7bb186833a67e9c2d84a58393503b85500ceb",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees/16a7bb186833a67e9c2d84a58393503b85500ceb"
+ },
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits/2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "comment_count": 0,
+ "verification": {
+ "verified": true,
+ "reason": "valid",
+ "signature": "-----BEGIN PGP SIGNATURE-----\n\niQFOBAABCAA4FiEEOOW2r8dr6sA77zHlgjqBKYe1QKUFAlynQ58aHHN6dWNzLmty\naXN6dGlhbkBnbWFpbC5jb20ACgkQgjqBKYe1QKUYKwf6AiXDMaLqNLNSjRY7lIXX\nudioewz0hSb4bgIXBv30nswu9CoOA0+mHCokEVtZhYbXzXDsZ1KJrilSC4j+Ws4q\nkRGA6iEmrne2HcSKNZXzcVnwV9zpwKxlVh2QCTNb1PuOYFBLH0kwE704uWIWMGDN\nbo8cjQPwegePCRguCvPh/5wa5J3uiq5gmJLG6bC/d1XYE+FJVtlnyzqzLMIryGKe\ntIciw+wwkF413Q/YVbZ49vLUeCX9H8PHC4mZYGDWuvjFW1WTfkjK5bAH+oaTVM6h\n350I5ZFloHmMA/QeRge5qFxXoEBMDGiXHHktzYZDXnliFOQNxzqwirA5lQQ6LRSS\naQ==\n=7rqi\n-----END PGP SIGNATURE-----",
+ "payload": "tree 16a7bb186833a67e9c2d84a58393503b85500ceb\nparent 446ae69b9385e8d0f40aa9595f723d34383af2f7\nauthor Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\ncommitter Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\n\nadd recorded event requests\n"
+ }
+ },
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "html_url": "https://github.com/ursa-labs/ursabot/commit/2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d/comments",
+ "author": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "committer": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "parents": [
+ {
+ "sha": "446ae69b9385e8d0f40aa9595f723d34383af2f7",
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/commits/446ae69b9385e8d0f40aa9595f723d34383af2f7",
+ "html_url": "https://github.com/ursa-labs/ursabot/commit/446ae69b9385e8d0f40aa9595f723d34383af2f7"
+ }
+ ],
+ "stats": {
+ "total": 1062,
+ "additions": 1058,
+ "deletions": 4
+ },
+ "files": [
+ {
+ "sha": "dfae6eeaef384ae6180c6302a58b49e39982dc33",
+ "filename": "ursabot/tests/fixtures/issue-comment-build-command.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"I've successfully started builds for this PR\",\n+ \"created_at\": \"2019-04-05T11:55:44Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248730\",\n+ \"id\": 480248730,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODczMA==\",\n+ \"updated_at\": \"2019-04-05T11:55:44Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248730\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 4,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:55:44Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}"
+ },
+ {
+ "sha": "7ef554e333327f0e62aa1fd76b4b17844a39adeb",
+ "filename": "ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Unknown command \\\"\\\"\",\n+ \"created_at\": \"2019-04-05T11:35:47Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\",\n+ \"id\": 480243815,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 2,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}"
+ },
+ {
+ "sha": "a8082dbc91fdfe815b795e49ec10e49000771ef5",
+ "filename": "ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"bear is no game\",\n+ \"created_at\": \"2019-04-05T11:26:56Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\",\n+ \"id\": 480241727,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 0,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "2770e29ba9086394455315e590c0b433d08e437e",
+ "filename": "ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot \",\n+ \"created_at\": \"2019-04-05T11:35:46Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\",\n+ \"id\": 480243811,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 1,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "80ff46510a2f39ae60f7c3a98e5fdaef8e688784",
+ "filename": "ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "status": "added",
+ "additions": 206,
+ "deletions": 0,
+ "changes": 206,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -0,0 +1,206 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Ursabot only listens to pull request comments!\",\n+ \"created_at\": \"2019-04-05T11:53:43Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480248217\",\n+ \"id\": 480248217,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODIxNw==\",\n+ \"updated_at\": \"2019-04-05T11:53:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248217\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 4,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\",\n+ \"created_at\": \"2019-04-02T09:56:41Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19\",\n+ \"id\": 428131685,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDU6SXNzdWU0MjgxMzE2ODU=\",\n+ \"number\": 19,\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Build ursabot itself via ursabot\",\n+ \"updated_at\": \"2019-04-05T11:53:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}"
+ },
+ {
+ "sha": "c738bb0eb54c87ba0f23e97e827d77c2be74d0b6",
+ "filename": "ursabot/tests/test_hooks.py",
+ "status": "modified",
+ "additions": 4,
+ "deletions": 4,
+ "changes": 8,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "patch": "@@ -54,7 +54,7 @@ class TestGithubHook(ChangeHookTestCase):\n await self.request('ping', {})\n assert len(self.hook.master.data.updates.changesAdded) == 0\n \n- @ensure_deferred\n- async def test_issue_comment(self):\n- payload = {}\n- await self.request('issue_comment', payload)\n+ # @ensure_deferred\n+ # async def test_issue_comment(self):\n+ # payload = {}\n+ # await self.request('issue_comment', payload)"
+ }
+ ]
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json
new file mode 100644
index 000000000..b039b3d10
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json
@@ -0,0 +1,170 @@
+[
+ {
+ "sha": "ebfe3f6c5e98723f9751c99ce8ce798f1ba529c5",
+ "filename": ".travis.yml",
+ "status": "modified",
+ "additions": 4,
+ "deletions": 1,
+ "changes": 5,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/.travis.yml",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/.travis.yml",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/.travis.yml?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -4,7 +4,10 @@ services:\n python:\n - 3.6\n script:\n- - pip install \"pytest>=3.9\" flake8 -e .\n+ # --no-binary buildbot is required because buildbot doesn't bundle its tests\n+ # to binary wheels, but ursabot's test suite depends on buildbot's so install\n+ # it from source\n+ - pip install --no-binary buildbot \"pytest>=3.9\" mock flake8 -e .\n \n # run linter\n - flake8 ursabot"
+ },
+ {
+ "sha": "86ad809d3f74c175b92ac58c6c645b0fbf5fa2c5",
+ "filename": "setup.py",
+ "status": "modified",
+ "additions": 6,
+ "deletions": 1,
+ "changes": 7,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/setup.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/setup.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/setup.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -1,8 +1,13 @@\n #!/usr/bin/env python\n \n+import sys\n from setuptools import setup\n \n \n+if sys.version_info < (3, 6):\n+ sys.exit('Python < 3.6 is not supported due to missing asyncio support')\n+\n+\n # TODO(kszucs): add package data, change maintainer\n setup(\n name='ursabot',\n@@ -15,7 +20,7 @@\n setup_requires=['setuptools_scm'],\n install_requires=['click', 'dask', 'docker', 'docker-map', 'toolz',\n 'buildbot', 'treq'],\n- tests_require=['pytest>=3.9'],\n+ tests_require=['pytest>=3.9', 'mock'],\n entry_points='''\n [console_scripts]\n ursabot=ursabot.cli:ursabot"
+ },
+ {
+ "sha": "c884f3f85bba499d77d9ad28bcd0ff5edf80f957",
+ "filename": "ursabot/factories.py",
+ "status": "modified",
+ "additions": 6,
+ "deletions": 2,
+ "changes": 8,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/factories.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/factories.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/factories.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -79,8 +79,12 @@ def prepend_step(self, step):\n repourl='https://github.com/ursa-labs/ursabot',\n mode='full'),\n ShellCommand(command=['ls', '-lah']),\n- ShellCommand(command=['pip', 'install', 'pytest', 'flake8']),\n- ShellCommand(command=['pip', 'install', '-e', '.']),\n+ ShellCommand(command=['pip', 'install', 'pytest', 'flake8', 'mock']),\n+ # --no-binary buildbot is required because buildbot doesn't bundle its\n+ # tests to binary wheels, but ursabot's test suite depends on buildbot's\n+ # so install it from source\n+ ShellCommand(command=['pip', 'install', '--no-binary', 'buildbot',\n+ '-e', '.']),\n ShellCommand(command=['flake8']),\n ShellCommand(command=['pytest', '-v', '-m', 'not docker', 'ursabot']),\n ShellCommand(command=['buildbot', 'checkconfig', '.'])"
+ },
+ {
+ "sha": "0265cfbd9c2882f492469882a7bf513a1c1b5af4",
+ "filename": "ursabot/hooks.py",
+ "status": "modified",
+ "additions": 17,
+ "deletions": 19,
+ "changes": 36,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/hooks.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/hooks.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/hooks.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -1,11 +1,11 @@\n from urllib.parse import urlparse\n \n from twisted.python import log\n-from twisted.internet import defer\n \n from buildbot.www.hooks.github import GitHubEventHandler\n from buildbot.util.httpclientservice import HTTPClientService\n \n+from .utils import ensure_deferred\n \n BOTNAME = 'ursabot'\n \n@@ -22,20 +22,18 @@ def _client(self):\n self.master, self.github_api_endpoint, headers=headers,\n debug=self.debug, verify=self.verify)\n \n- @defer.inlineCallbacks\n- def _get(self, url):\n+ async def _get(self, url):\n url = urlparse(url)\n- client = yield self._client()\n- response = yield client.get(url.path)\n- result = yield response.json()\n+ client = await self._client()\n+ response = await client.get(url.path)\n+ result = await response.json()\n return result\n \n- @defer.inlineCallbacks\n- def _post(self, url, data):\n+ async def _post(self, url, data):\n url = urlparse(url)\n- client = yield self._client()\n- response = yield client.post(url.path, json=data)\n- result = yield response.json()\n+ client = await self._client()\n+ response = await client.post(url.path, json=data)\n+ result = await response.json()\n log.msg(f'POST to {url} with the following result: {result}')\n return result\n \n@@ -46,8 +44,8 @@ def _parse_command(self, message):\n return message.split(mention)[-1].lower().strip()\n return None\n \n- @defer.inlineCallbacks\n- def handle_issue_comment(self, payload, event):\n+ @ensure_deferred\n+ async def handle_issue_comment(self, payload, event):\n issue = payload['issue']\n comments_url = issue['comments_url']\n command = self._parse_command(payload['comment']['body'])\n@@ -64,16 +62,16 @@ def handle_issue_comment(self, payload, event):\n elif command == 'build':\n if 'pull_request' not in issue:\n message = 'Ursabot only listens to pull request comments!'\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return [], 'git'\n else:\n message = f'Unknown command \"{command}\"'\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return [], 'git'\n \n try:\n- pull_request = yield self._get(issue['pull_request']['url'])\n- changes, _ = yield self.handle_pull_request({\n+ pull_request = await self._get(issue['pull_request']['url'])\n+ changes, _ = await self.handle_pull_request({\n 'action': 'synchronize',\n 'sender': payload['sender'],\n 'repository': payload['repository'],\n@@ -82,11 +80,11 @@ def handle_issue_comment(self, payload, event):\n }, event)\n except Exception as e:\n message = \"I've failed to start builds for this PR\"\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n raise e\n else:\n message = \"I've successfully started builds for this PR\"\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return changes, 'git'\n \n # TODO(kszucs):"
+ },
+ {
+ "sha": "1e1ecf2ce47da929dbf1b93632640e7e6ae1cfe0",
+ "filename": "ursabot/steps.py",
+ "status": "modified",
+ "additions": 13,
+ "deletions": 13,
+ "changes": 26,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/steps.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/steps.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/steps.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -1,9 +1,9 @@\n-from twisted.internet import defer\n-\n from buildbot.plugins import steps, util\n from buildbot.process import buildstep\n from buildbot.process.results import SUCCESS\n \n+from .utils import ensure_deferred\n+\n \n class ShellMixin(buildstep.ShellMixin):\n \"\"\"Run command in a login bash shell\n@@ -49,10 +49,10 @@ def __init__(self, **kwargs):\n kwargs = self.setupShellMixin(kwargs)\n super().__init__(**kwargs)\n \n- @defer.inlineCallbacks\n- def run(self):\n- cmd = yield self.makeRemoteShellCommand(command=self.command)\n- yield self.runCommand(cmd)\n+ @ensure_deferred\n+ async def run(self):\n+ cmd = await self.makeRemoteShellCommand(command=self.command)\n+ await self.runCommand(cmd)\n return cmd.results()\n \n \n@@ -71,8 +71,8 @@ class CMake(ShellMixin, steps.CMake):\n \n name = 'CMake'\n \n- @defer.inlineCallbacks\n- def run(self):\n+ @ensure_deferred\n+ async def run(self):\n \"\"\"Create and run CMake command\n \n Copied from the original CMake implementation to handle None values as\n@@ -94,8 +94,8 @@ def run(self):\n if self.options is not None:\n command.extend(self.options)\n \n- cmd = yield self.makeRemoteShellCommand(command=command)\n- yield self.runCommand(cmd)\n+ cmd = await self.makeRemoteShellCommand(command=command)\n+ await self.runCommand(cmd)\n \n return cmd.results()\n \n@@ -117,8 +117,8 @@ def __init__(self, variables, source='WorkerEnvironment', **kwargs):\n self.source = source\n super().__init__(**kwargs)\n \n- @defer.inlineCallbacks\n- def run(self):\n+ @ensure_deferred\n+ async def run(self):\n # on Windows, environment variables are case-insensitive, but we have\n # a case-sensitive dictionary in worker_environ. Fortunately, that\n # dictionary is also folded to uppercase, so we can simply fold the\n@@ -139,7 +139,7 @@ def run(self):\n # TODO(kszucs) try with self.setProperty similarly like in\n # SetProperties\n properties.setProperty(prop, value, self.source, runtime=True)\n- yield self.addCompleteLog('set-prop', f'{prop}: {value}')\n+ await self.addCompleteLog('set-prop', f'{prop}: {value}')\n \n return SUCCESS\n "
+ },
+ {
+ "sha": "6a7d5308be6608f542a810d410f9240157a1340f",
+ "filename": "ursabot/tests/fixtures/issue-comment-build-command.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-build-command.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-build-command.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot build\",\n+ \"created_at\": \"2019-04-05T11:55:43Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726\",\n+ \"id\": 480248726,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==\",\n+ \"updated_at\": \"2019-04-05T11:55:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 3,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:55:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "7ef554e333327f0e62aa1fd76b4b17844a39adeb",
+ "filename": "ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-by-ursabot.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Unknown command \\\"\\\"\",\n+ \"created_at\": \"2019-04-05T11:35:47Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\",\n+ \"id\": 480243815,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 2,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}"
+ },
+ {
+ "sha": "a8082dbc91fdfe815b795e49ec10e49000771ef5",
+ "filename": "ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"bear is no game\",\n+ \"created_at\": \"2019-04-05T11:26:56Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\",\n+ \"id\": 480241727,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 0,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "2770e29ba9086394455315e590c0b433d08e437e",
+ "filename": "ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "status": "added",
+ "additions": 212,
+ "deletions": 0,
+ "changes": 212,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-with-empty-command.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot \",\n+ \"created_at\": \"2019-04-05T11:35:46Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\",\n+ \"id\": 480243811,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 1,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "b7de8d838332944101812ee2a46c08dd0144efe3",
+ "filename": "ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "status": "added",
+ "additions": 206,
+ "deletions": 0,
+ "changes": 206,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-without-pull-request.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,206 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot build\",\n+ \"created_at\": \"2019-04-05T13:07:57Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480268708\",\n+ \"id\": 480268708,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI2ODcwOA==\",\n+ \"updated_at\": \"2019-04-05T13:07:57Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480268708\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 5,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\",\n+ \"created_at\": \"2019-04-02T09:56:41Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19\",\n+ \"id\": 428131685,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDU6SXNzdWU0MjgxMzE2ODU=\",\n+ \"number\": 19,\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Build ursabot itself via ursabot\",\n+ \"updated_at\": \"2019-04-05T13:07:57Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 898,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}"
+ },
+ {
+ "sha": "33e051455e866fb4774a16ae02ad40dcf9e6a7fd",
+ "filename": "ursabot/tests/fixtures/pull-request-26-commit.json",
+ "status": "added",
+ "additions": 158,
+ "deletions": 0,
+ "changes": 158,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26-commit.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26-commit.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/pull-request-26-commit.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,158 @@\n+{\n+ \"sha\": \"2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"node_id\": \"MDY6Q29tbWl0MTY5MTAxNzAxOjI3MDVkYTJiNjE2Yjk4ZmE2MDEwYTI1ODEzYzVhN2EyNzQ1NmY3MWQ=\",\n+ \"commit\": {\n+ \"author\": {\n+ \"name\": \"Krisztián Szűcs\",\n+ \"email\": \"szucs.krisztian@gmail.com\",\n+ \"date\": \"2019-04-05T12:01:31Z\"\n+ },\n+ \"committer\": {\n+ \"name\": \"Krisztián Szűcs\",\n+ \"email\": \"szucs.krisztian@gmail.com\",\n+ \"date\": \"2019-04-05T12:01:31Z\"\n+ },\n+ \"message\": \"add recorded event requests\",\n+ \"tree\": {\n+ \"sha\": \"16a7bb186833a67e9c2d84a58393503b85500ceb\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees/16a7bb186833a67e9c2d84a58393503b85500ceb\"\n+ },\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"comment_count\": 0,\n+ \"verification\": {\n+ \"verified\": true,\n+ \"reason\": \"valid\",\n+ \"signature\": \"-----BEGIN PGP SIGNATURE-----\\n\\niQFOBAABCAA4FiEEOOW2r8dr6sA77zHlgjqBKYe1QKUFAlynQ58aHHN6dWNzLmty\\naXN6dGlhbkBnbWFpbC5jb20ACgkQgjqBKYe1QKUYKwf6AiXDMaLqNLNSjRY7lIXX\\nudioewz0hSb4bgIXBv30nswu9CoOA0+mHCokEVtZhYbXzXDsZ1KJrilSC4j+Ws4q\\nkRGA6iEmrne2HcSKNZXzcVnwV9zpwKxlVh2QCTNb1PuOYFBLH0kwE704uWIWMGDN\\nbo8cjQPwegePCRguCvPh/5wa5J3uiq5gmJLG6bC/d1XYE+FJVtlnyzqzLMIryGKe\\ntIciw+wwkF413Q/YVbZ49vLUeCX9H8PHC4mZYGDWuvjFW1WTfkjK5bAH+oaTVM6h\\n350I5ZFloHmMA/QeRge5qFxXoEBMDGiXHHktzYZDXnliFOQNxzqwirA5lQQ6LRSS\\naQ==\\n=7rqi\\n-----END PGP SIGNATURE-----\",\n+ \"payload\": \"tree 16a7bb186833a67e9c2d84a58393503b85500ceb\\nparent 446ae69b9385e8d0f40aa9595f723d34383af2f7\\nauthor Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\\ncommitter Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\\n\\nadd recorded event requests\\n\"\n+ }\n+ },\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/commit/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d/comments\",\n+ \"author\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"committer\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"parents\": [\n+ {\n+ \"sha\": \"446ae69b9385e8d0f40aa9595f723d34383af2f7\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/446ae69b9385e8d0f40aa9595f723d34383af2f7\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/commit/446ae69b9385e8d0f40aa9595f723d34383af2f7\"\n+ }\n+ ],\n+ \"stats\": {\n+ \"total\": 1062,\n+ \"additions\": 1058,\n+ \"deletions\": 4\n+ },\n+ \"files\": [\n+ {\n+ \"sha\": \"dfae6eeaef384ae6180c6302a58b49e39982dc33\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"I've successfully started builds for this PR\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248730\\\",\\n+ \\\"id\\\": 480248730,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODczMA==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248730\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 4,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"7ef554e333327f0e62aa1fd76b4b17844a39adeb\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"Unknown command \\\\\\\"\\\\\\\"\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\\\",\\n+ \\\"id\\\": 480243815,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 2,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"a8082dbc91fdfe815b795e49ec10e49000771ef5\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"bear is no game\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\\\",\\n+ \\\"id\\\": 480241727,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 0,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"2770e29ba9086394455315e590c0b433d08e437e\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"@ursabot \\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\\\",\\n+ \\\"id\\\": 480243811,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 1,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"80ff46510a2f39ae60f7c3a98e5fdaef8e688784\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"status\": \"added\",\n+ \"additions\": 206,\n+ \"deletions\": 0,\n+ \"changes\": 206,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,206 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"Ursabot only listens to pull request comments!\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480248217\\\",\\n+ \\\"id\\\": 480248217,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODIxNw==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248217\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 4,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-02T09:56:41Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"id\\\": 428131685,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDU6SXNzdWU0MjgxMzE2ODU=\\\",\\n+ \\\"number\\\": 19,\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Build ursabot itself via ursabot\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"c738bb0eb54c87ba0f23e97e827d77c2be74d0b6\",\n+ \"filename\": \"ursabot/tests/test_hooks.py\",\n+ \"status\": \"modified\",\n+ \"additions\": 4,\n+ \"deletions\": 4,\n+ \"changes\": 8,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -54,7 +54,7 @@ class TestGithubHook(ChangeHookTestCase):\\n await self.request('ping', {})\\n assert len(self.hook.master.data.updates.changesAdded) == 0\\n \\n- @ensure_deferred\\n- async def test_issue_comment(self):\\n- payload = {}\\n- await self.request('issue_comment', payload)\\n+ # @ensure_deferred\\n+ # async def test_issue_comment(self):\\n+ # payload = {}\\n+ # await self.request('issue_comment', payload)\"\n+ }\n+ ]\n+}"
+ },
+ {
+ "sha": "ad061d7244b917e6ea3853698dc3bc2a8c9c6857",
+ "filename": "ursabot/tests/fixtures/pull-request-26.json",
+ "status": "added",
+ "additions": 335,
+ "deletions": 0,
+ "changes": 335,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26.json",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26.json",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/pull-request-26.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,335 @@\n+{\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\",\n+ \"id\": 267785552,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"number\": 26,\n+ \"state\": \"open\",\n+ \"locked\": false,\n+ \"title\": \"Unittests for GithubHook\",\n+ \"user\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"body\": \"\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"updated_at\": \"2019-04-05T12:01:40Z\",\n+ \"closed_at\": null,\n+ \"merged_at\": null,\n+ \"merge_commit_sha\": \"cc5dc3606988b3824be54df779ed2028776113cb\",\n+ \"assignee\": null,\n+ \"assignees\": [\n+\n+ ],\n+ \"requested_reviewers\": [\n+\n+ ],\n+ \"requested_teams\": [\n+\n+ ],\n+ \"labels\": [\n+\n+ ],\n+ \"milestone\": null,\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits\",\n+ \"review_comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments\",\n+ \"review_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"head\": {\n+ \"label\": \"ursa-labs:test-hook\",\n+ \"ref\": \"test-hook\",\n+ \"sha\": \"2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"user\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"repo\": {\n+ \"id\": 169101701,\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"name\": \"ursabot\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"private\": false,\n+ \"owner\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"description\": null,\n+ \"fork\": false,\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"homepage\": null,\n+ \"size\": 898,\n+ \"stargazers_count\": 1,\n+ \"watchers_count\": 1,\n+ \"language\": \"Jupyter Notebook\",\n+ \"has_issues\": true,\n+ \"has_projects\": true,\n+ \"has_downloads\": true,\n+ \"has_wiki\": true,\n+ \"has_pages\": false,\n+ \"forks_count\": 0,\n+ \"mirror_url\": null,\n+ \"archived\": false,\n+ \"disabled\": false,\n+ \"open_issues_count\": 19,\n+ \"license\": null,\n+ \"forks\": 0,\n+ \"open_issues\": 19,\n+ \"watchers\": 1,\n+ \"default_branch\": \"master\"\n+ }\n+ },\n+ \"base\": {\n+ \"label\": \"ursa-labs:master\",\n+ \"ref\": \"master\",\n+ \"sha\": \"a162ad254b589b924db47e057791191b39613fd5\",\n+ \"user\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"repo\": {\n+ \"id\": 169101701,\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"name\": \"ursabot\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"private\": false,\n+ \"owner\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"description\": null,\n+ \"fork\": false,\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"homepage\": null,\n+ \"size\": 898,\n+ \"stargazers_count\": 1,\n+ \"watchers_count\": 1,\n+ \"language\": \"Jupyter Notebook\",\n+ \"has_issues\": true,\n+ \"has_projects\": true,\n+ \"has_downloads\": true,\n+ \"has_wiki\": true,\n+ \"has_pages\": false,\n+ \"forks_count\": 0,\n+ \"mirror_url\": null,\n+ \"archived\": false,\n+ \"disabled\": false,\n+ \"open_issues_count\": 19,\n+ \"license\": null,\n+ \"forks\": 0,\n+ \"open_issues\": 19,\n+ \"watchers\": 1,\n+ \"default_branch\": \"master\"\n+ }\n+ },\n+ \"_links\": {\n+ \"self\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"html\": {\n+ \"href\": \"https://github.com/ursa-labs/ursabot/pull/26\"\n+ },\n+ \"issue\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\"\n+ },\n+ \"comments\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\"\n+ },\n+ \"review_comments\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments\"\n+ },\n+ \"review_comment\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}\"\n+ },\n+ \"commits\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits\"\n+ },\n+ \"statuses\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d\"\n+ }\n+ },\n+ \"author_association\": \"MEMBER\",\n+ \"merged\": false,\n+ \"mergeable\": true,\n+ \"rebaseable\": true,\n+ \"mergeable_state\": \"unstable\",\n+ \"merged_by\": null,\n+ \"comments\": 5,\n+ \"review_comments\": 0,\n+ \"maintainer_can_modify\": false,\n+ \"commits\": 2,\n+ \"additions\": 1124,\n+ \"deletions\": 0,\n+ \"changed_files\": 7\n+}"
+ },
+ {
+ "sha": "e87b27d2d7b4956d15f7468488b96cf6a06686f4",
+ "filename": "ursabot/tests/test_hooks.py",
+ "status": "added",
+ "additions": 116,
+ "deletions": 0,
+ "changes": 116,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/test_hooks.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/test_hooks.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,116 @@\n+import json\n+from pathlib import Path\n+from twisted.trial import unittest\n+\n+from buildbot.test.util.misc import TestReactorMixin\n+from buildbot.test.fake.httpclientservice import \\\n+ HTTPClientService as FakeHTTPClientService\n+from buildbot.test.unit.test_www_hooks_github import (\n+ _prepare_request, _prepare_github_change_hook)\n+\n+from ursabot.utils import ensure_deferred\n+from ursabot.hooks import GithubHook\n+\n+\n+class ChangeHookTestCase(unittest.TestCase, TestReactorMixin):\n+\n+ klass = None\n+\n+ @ensure_deferred\n+ async def setUp(self):\n+ self.setUpTestReactor()\n+\n+ assert self.klass is not None\n+ self.hook = _prepare_github_change_hook(self, **{'class': self.klass})\n+ self.master = self.hook.master\n+ self.http = await FakeHTTPClientService.getFakeService(\n+ self.master, self, 'https://api.github.com',\n+ headers={'User-Agent': 'Buildbot'}, debug=False, verify=False)\n+\n+ await self.master.startService()\n+\n+ @ensure_deferred\n+ async def tearDown(self):\n+ await self.master.stopService()\n+\n+ async def trigger(self, event, payload, headers=None, _secret=None):\n+ payload = json.dumps(payload).encode()\n+ request = _prepare_request(event, payload, _secret=_secret,\n+ headers=headers)\n+ await request.test_render(self.hook)\n+ return request\n+\n+ def load_fixture(self, name):\n+ path = Path(__file__).parent / 'fixtures' / f'{name}.json'\n+ with path.open('r') as fp:\n+ return json.load(fp)\n+\n+\n+class TestGithubHook(ChangeHookTestCase):\n+\n+ klass = GithubHook\n+\n+ @ensure_deferred\n+ async def test_ping(self):\n+ await self.trigger('ping', {})\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_not_mentioning_ursabot(self):\n+ payload = self.load_fixture('issue-comment-not-mentioning-ursabot')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_by_ursabot(self):\n+ payload = self.load_fixture('issue-comment-by-ursabot')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_with_empty_command(self):\n+ # responds to the comment\n+ request_json = {'body': 'Unknown command \"\"'}\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/26/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-with-empty-command')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_without_pull_request(self):\n+ # responds to the comment\n+ request_json = {\n+ 'body': 'Ursabot only listens to pull request comments!'\n+ }\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/19/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-without-pull-request')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_build_command(self):\n+ # handle_issue_comment queries the pull request\n+ request_json = self.load_fixture('pull-request-26')\n+ self.http.expect('get', '/repos/ursa-labs/ursabot/pulls/26',\n+ content_json=request_json)\n+ # tigger handle_pull_request which fetches the commit\n+ request_json = self.load_fixture('pull-request-26-commit')\n+ commit = '2705da2b616b98fa6010a25813c5a7a27456f71d'\n+ self.http.expect('get', f'/repos/ursa-labs/ursabot/commits/{commit}',\n+ content_json=request_json)\n+\n+ # then responds to the comment\n+ request_json = {'body': \"I've successfully started builds for this PR\"}\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/26/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-build-command')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 1"
+ },
+ {
+ "sha": "3ff0e88660cf186420e8bc672735e4d446963192",
+ "filename": "ursabot/utils.py",
+ "status": "added",
+ "additions": 10,
+ "deletions": 0,
+ "changes": 10,
+ "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/utils.py",
+ "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/utils.py",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/utils.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0",
+ "patch": "@@ -0,0 +1,10 @@\n+import functools\n+from twisted.internet import defer\n+\n+\n+def ensure_deferred(f):\n+ @functools.wraps(f)\n+ def wrapper(*args, **kwargs):\n+ result = f(*args, **kwargs)\n+ return defer.ensureDeferred(result)\n+ return wrapper"
+ }
+] \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json
new file mode 100644
index 000000000..d295afb39
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json
@@ -0,0 +1,329 @@
+{
+ "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26",
+ "id": 267785552,
+ "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy",
+ "html_url": "https://github.com/ursa-labs/ursabot/pull/26",
+ "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff",
+ "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch",
+ "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26",
+ "number": 26,
+ "state": "open",
+ "locked": false,
+ "title": "Unittests for GithubHook",
+ "user": {
+ "login": "kszucs",
+ "id": 961747,
+ "node_id": "MDQ6VXNlcjk2MTc0Nw==",
+ "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/kszucs",
+ "html_url": "https://github.com/kszucs",
+ "followers_url": "https://api.github.com/users/kszucs/followers",
+ "following_url": "https://api.github.com/users/kszucs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions",
+ "organizations_url": "https://api.github.com/users/kszucs/orgs",
+ "repos_url": "https://api.github.com/users/kszucs/repos",
+ "events_url": "https://api.github.com/users/kszucs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/kszucs/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "body": "",
+ "body_html": "",
+ "body_text": "",
+ "created_at": "2019-04-05T11:22:15Z",
+ "updated_at": "2019-04-05T12:01:40Z",
+ "closed_at": null,
+ "merged_at": null,
+ "merge_commit_sha": "cc5dc3606988b3824be54df779ed2028776113cb",
+ "assignee": null,
+ "assignees": [],
+ "requested_reviewers": [],
+ "requested_teams": [],
+ "labels": [],
+ "milestone": null,
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits",
+ "review_comments_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments",
+ "review_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "head": {
+ "label": "ursa-labs:test-hook",
+ "ref": "test-hook",
+ "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d",
+ "user": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "repo": {
+ "id": 169101701,
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "name": "ursabot",
+ "full_name": "ursa-labs/ursabot",
+ "private": false,
+ "owner": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "description": null,
+ "fork": false,
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "created_at": "2019-02-04T15:40:31Z",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "pushed_at": "2019-04-05T12:01:40Z",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "homepage": null,
+ "size": 898,
+ "stargazers_count": 1,
+ "watchers_count": 1,
+ "language": "Jupyter Notebook",
+ "has_issues": true,
+ "has_projects": true,
+ "has_downloads": true,
+ "has_wiki": true,
+ "has_pages": false,
+ "forks_count": 0,
+ "mirror_url": null,
+ "archived": false,
+ "disabled": false,
+ "open_issues_count": 19,
+ "license": null,
+ "forks": 0,
+ "open_issues": 19,
+ "watchers": 1,
+ "default_branch": "master"
+ }
+ },
+ "base": {
+ "label": "ursa-labs:master",
+ "ref": "master",
+ "sha": "a162ad254b589b924db47e057791191b39613fd5",
+ "user": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "repo": {
+ "id": 169101701,
+ "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=",
+ "name": "ursabot",
+ "full_name": "ursa-labs/ursabot",
+ "private": false,
+ "owner": {
+ "login": "ursa-labs",
+ "id": 46514972,
+ "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy",
+ "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/ursa-labs",
+ "html_url": "https://github.com/ursa-labs",
+ "followers_url": "https://api.github.com/users/ursa-labs/followers",
+ "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}",
+ "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions",
+ "organizations_url": "https://api.github.com/users/ursa-labs/orgs",
+ "repos_url": "https://api.github.com/users/ursa-labs/repos",
+ "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/ursa-labs/received_events",
+ "type": "Organization",
+ "site_admin": false
+ },
+ "html_url": "https://github.com/ursa-labs/ursabot",
+ "description": null,
+ "fork": false,
+ "url": "https://api.github.com/repos/ursa-labs/ursabot",
+ "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks",
+ "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}",
+ "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}",
+ "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams",
+ "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks",
+ "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}",
+ "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events",
+ "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}",
+ "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}",
+ "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags",
+ "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}",
+ "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}",
+ "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}",
+ "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}",
+ "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages",
+ "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers",
+ "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors",
+ "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers",
+ "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription",
+ "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}",
+ "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}",
+ "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}",
+ "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}",
+ "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}",
+ "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}",
+ "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges",
+ "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}",
+ "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads",
+ "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}",
+ "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}",
+ "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}",
+ "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}",
+ "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}",
+ "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}",
+ "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments",
+ "created_at": "2019-02-04T15:40:31Z",
+ "updated_at": "2019-04-04T17:49:10Z",
+ "pushed_at": "2019-04-05T12:01:40Z",
+ "git_url": "git://github.com/ursa-labs/ursabot.git",
+ "ssh_url": "git@github.com:ursa-labs/ursabot.git",
+ "clone_url": "https://github.com/ursa-labs/ursabot.git",
+ "svn_url": "https://github.com/ursa-labs/ursabot",
+ "homepage": null,
+ "size": 898,
+ "stargazers_count": 1,
+ "watchers_count": 1,
+ "language": "Jupyter Notebook",
+ "has_issues": true,
+ "has_projects": true,
+ "has_downloads": true,
+ "has_wiki": true,
+ "has_pages": false,
+ "forks_count": 0,
+ "mirror_url": null,
+ "archived": false,
+ "disabled": false,
+ "open_issues_count": 19,
+ "license": null,
+ "forks": 0,
+ "open_issues": 19,
+ "watchers": 1,
+ "default_branch": "master"
+ }
+ },
+ "_links": {
+ "self": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26"
+ },
+ "html": {
+ "href": "https://github.com/ursa-labs/ursabot/pull/26"
+ },
+ "issue": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26"
+ },
+ "comments": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments"
+ },
+ "review_comments": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments"
+ },
+ "review_comment": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}"
+ },
+ "commits": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits"
+ },
+ "statuses": {
+ "href": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d"
+ }
+ },
+ "author_association": "MEMBER",
+ "merged": false,
+ "mergeable": true,
+ "rebaseable": true,
+ "mergeable_state": "unstable",
+ "merged_by": null,
+ "comments": 5,
+ "review_comments": 0,
+ "maintainer_can_modify": false,
+ "commits": 2,
+ "additions": 1124,
+ "deletions": 0,
+ "changed_files": 7
+} \ No newline at end of file
diff --git a/src/arrow/dev/archery/archery/tests/test_benchmarks.py b/src/arrow/dev/archery/archery/tests/test_benchmarks.py
new file mode 100644
index 000000000..fab1e8d44
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/test_benchmarks.py
@@ -0,0 +1,383 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from archery.benchmark.codec import JsonEncoder
+from archery.benchmark.core import Benchmark, median
+from archery.benchmark.compare import (
+ BenchmarkComparator, RunnerComparator
+)
+from archery.benchmark.google import (
+ GoogleBenchmark, GoogleBenchmarkObservation
+)
+from archery.benchmark.runner import StaticBenchmarkRunner
+
+
+def test_benchmark_comparator():
+ unit = "micros"
+
+ assert not BenchmarkComparator(
+ Benchmark("contender", unit, True, [10], unit, [1]),
+ Benchmark("baseline", unit, True, [20], unit, [1]),
+ ).regression
+
+ assert BenchmarkComparator(
+ Benchmark("contender", unit, False, [10], unit, [1]),
+ Benchmark("baseline", unit, False, [20], unit, [1]),
+ ).regression
+
+ assert BenchmarkComparator(
+ Benchmark("contender", unit, True, [20], unit, [1]),
+ Benchmark("baseline", unit, True, [10], unit, [1]),
+ ).regression
+
+ assert not BenchmarkComparator(
+ Benchmark("contender", unit, False, [20], unit, [1]),
+ Benchmark("baseline", unit, False, [10], unit, [1]),
+ ).regression
+
+
+def test_static_runner_from_json_not_a_regression():
+ archery_result = {
+ "suites": [
+ {
+ "name": "arrow-value-parsing-benchmark",
+ "benchmarks": [
+ {
+ "name": "FloatParsing<DoubleType>",
+ "unit": "items_per_second",
+ "less_is_better": False,
+ "values": [
+ 109941112.87296811
+ ],
+ "time_unit": "ns",
+ "times": [
+ 9095.800104330105
+ ]
+ },
+ ]
+ }
+ ]
+ }
+
+ contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+ baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+ [comparison] = RunnerComparator(contender, baseline).comparisons
+ assert not comparison.regression
+
+
+def test_static_runner_from_json_regression():
+ archery_result = {
+ "suites": [
+ {
+ "name": "arrow-value-parsing-benchmark",
+ "benchmarks": [
+ {
+ "name": "FloatParsing<DoubleType>",
+ "unit": "items_per_second",
+ "less_is_better": False,
+ "values": [
+ 109941112.87296811
+ ],
+ "time_unit": "ns",
+ "times": [
+ 9095.800104330105
+ ]
+ },
+ ]
+ }
+ ]
+ }
+
+ contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+
+ # introduce artificial regression
+ archery_result['suites'][0]['benchmarks'][0]['values'][0] *= 2
+ baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result))
+
+ [comparison] = RunnerComparator(contender, baseline).comparisons
+ assert comparison.regression
+
+
+def test_benchmark_median():
+ assert median([10]) == 10
+ assert median([1, 2, 3]) == 2
+ assert median([1, 2]) == 1.5
+ assert median([1, 2, 3, 4]) == 2.5
+ assert median([1, 1, 1, 1]) == 1
+ try:
+ median([])
+ assert False
+ except ValueError:
+ pass
+
+
+def assert_benchmark(name, google_result, archery_result):
+ observation = GoogleBenchmarkObservation(**google_result)
+ benchmark = GoogleBenchmark(name, [observation])
+ result = json.dumps(benchmark, cls=JsonEncoder)
+ assert json.loads(result) == archery_result
+
+
+def test_items_per_second():
+ name = "ArrayArrayKernel<AddChecked, UInt8Type>/32768/0"
+ google_result = {
+ "cpu_time": 116292.58886653671,
+ "items_per_second": 281772039.9844759,
+ "iterations": 5964,
+ "name": name,
+ "null_percent": 0.0,
+ "real_time": 119811.77313729875,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "size": 32768.0,
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 5964,
+ "null_percent": 0.0,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "items_per_second",
+ "less_is_better": False,
+ "values": [281772039.9844759],
+ "time_unit": "ns",
+ "times": [119811.77313729875],
+ }
+ assert "items_per_second" in google_result
+ assert "bytes_per_second" not in google_result
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_bytes_per_second():
+ name = "BufferOutputStreamLargeWrites/real_time"
+ google_result = {
+ "bytes_per_second": 1890209037.3405428,
+ "cpu_time": 17018127.659574457,
+ "iterations": 47,
+ "name": name,
+ "real_time": 17458386.53190963,
+ "repetition_index": 1,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 47,
+ "repetition_index": 1,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "bytes_per_second",
+ "less_is_better": False,
+ "values": [1890209037.3405428],
+ "time_unit": "ns",
+ "times": [17458386.53190963],
+ }
+ assert "items_per_second" not in google_result
+ assert "bytes_per_second" in google_result
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_both_items_and_bytes_per_second():
+ name = "ArrayArrayKernel<AddChecked, UInt8Type>/32768/0"
+ google_result = {
+ "bytes_per_second": 281772039.9844759,
+ "cpu_time": 116292.58886653671,
+ "items_per_second": 281772039.9844759,
+ "iterations": 5964,
+ "name": name,
+ "null_percent": 0.0,
+ "real_time": 119811.77313729875,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "size": 32768.0,
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ # Note that bytes_per_second trumps items_per_second
+ archery_result = {
+ "counters": {"iterations": 5964,
+ "null_percent": 0.0,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "bytes_per_second",
+ "less_is_better": False,
+ "values": [281772039.9844759],
+ "time_unit": "ns",
+ "times": [119811.77313729875],
+ }
+ assert "items_per_second" in google_result
+ assert "bytes_per_second" in google_result
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_neither_items_nor_bytes_per_second():
+ name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time"
+ google_result = {
+ "cpu_time": 1778.6004847419827,
+ "iterations": 352765,
+ "name": name,
+ "real_time": 1835.3137357788837,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 352765,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "ns",
+ "less_is_better": True,
+ "values": [1835.3137357788837],
+ "time_unit": "ns",
+ "times": [1835.3137357788837],
+ }
+ assert "items_per_second" not in google_result
+ assert "bytes_per_second" not in google_result
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_prefer_real_time():
+ name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time"
+ google_result = {
+ "cpu_time": 1778.6004847419827,
+ "iterations": 352765,
+ "name": name,
+ "real_time": 1835.3137357788837,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 352765,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "ns",
+ "less_is_better": True,
+ "values": [1835.3137357788837],
+ "time_unit": "ns",
+ "times": [1835.3137357788837],
+ }
+ assert name.endswith("/real_time")
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_prefer_cpu_time():
+ name = "AllocateDeallocate<Jemalloc>/size:1048576"
+ google_result = {
+ "cpu_time": 1778.6004847419827,
+ "iterations": 352765,
+ "name": name,
+ "real_time": 1835.3137357788837,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 352765,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "ns",
+ "less_is_better": True,
+ "values": [1778.6004847419827],
+ "time_unit": "ns",
+ "times": [1835.3137357788837],
+ }
+ assert not name.endswith("/real_time")
+ assert_benchmark(name, google_result, archery_result)
+
+
+def test_omits_aggregates():
+ name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time"
+ google_aggregate = {
+ "aggregate_name": "mean",
+ "cpu_time": 1757.428694267678,
+ "iterations": 3,
+ "name": "AllocateDeallocate<Jemalloc>/size:1048576/real_time_mean",
+ "real_time": 1849.3869337041162,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "aggregate",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ google_result = {
+ "cpu_time": 1778.6004847419827,
+ "iterations": 352765,
+ "name": name,
+ "real_time": 1835.3137357788837,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "run_type": "iteration",
+ "threads": 1,
+ "time_unit": "ns",
+ }
+ archery_result = {
+ "counters": {"iterations": 352765,
+ "repetition_index": 0,
+ "repetitions": 0,
+ "run_name": name,
+ "threads": 1},
+ "name": name,
+ "unit": "ns",
+ "less_is_better": True,
+ "values": [1835.3137357788837],
+ "time_unit": "ns",
+ "times": [1835.3137357788837],
+ }
+ assert google_aggregate["run_type"] == "aggregate"
+ assert google_result["run_type"] == "iteration"
+ observation1 = GoogleBenchmarkObservation(**google_aggregate)
+ observation2 = GoogleBenchmarkObservation(**google_result)
+ benchmark = GoogleBenchmark(name, [observation1, observation2])
+ result = json.dumps(benchmark, cls=JsonEncoder)
+ assert json.loads(result) == archery_result
diff --git a/src/arrow/dev/archery/archery/tests/test_bot.py b/src/arrow/dev/archery/archery/tests/test_bot.py
new file mode 100644
index 000000000..e84fb7e27
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/test_bot.py
@@ -0,0 +1,215 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from unittest.mock import Mock
+
+import click
+import pytest
+import responses as rsps
+
+from archery.bot import CommentBot, CommandError, group
+
+
+@pytest.fixture
+def responses():
+ with rsps.RequestsMock() as mock:
+ yield mock
+
+
+def github_url(path):
+ return 'https://api.github.com:443/{}'.format(path.strip('/'))
+
+
+@group()
+def custom_handler():
+ pass
+
+
+@custom_handler.command()
+@click.pass_obj
+def extra(obj):
+ return obj
+
+
+@custom_handler.command()
+@click.option('--force', '-f', is_flag=True)
+def build(force):
+ return force
+
+
+@custom_handler.command()
+@click.option('--name', required=True)
+def benchmark(name):
+ return name
+
+
+def test_click_based_commands():
+ assert custom_handler('build') is False
+ assert custom_handler('build -f') is True
+
+ assert custom_handler('benchmark --name strings') == 'strings'
+ with pytest.raises(CommandError):
+ assert custom_handler('benchmark')
+
+ assert custom_handler('extra', extra='data') == {'extra': 'data'}
+
+
+@pytest.mark.parametrize('fixture_name', [
+ # the bot is not mentioned, nothing to do
+ 'event-issue-comment-not-mentioning-ursabot.json',
+ # don't respond to itself, it prevents recursive comment storms!
+ 'event-issue-comment-by-ursabot.json',
+ # non-authorized user sent the comment, do not respond
+ 'event-issue-comment-by-non-authorized-user.json',
+])
+def test_noop_events(load_fixture, fixture_name):
+ payload = load_fixture(fixture_name)
+
+ handler = Mock()
+ bot = CommentBot(name='ursabot', token='', handler=handler)
+ bot.handle('issue_comment', payload)
+
+ handler.assert_not_called()
+
+
+def test_issue_comment_without_pull_request(load_fixture, responses):
+ responses.add(
+ responses.GET,
+ github_url('/repositories/169101701/issues/19'),
+ json=load_fixture('issue-19.json'),
+ status=200
+ )
+ responses.add(
+ responses.GET,
+ github_url('repos/ursa-labs/ursabot/pulls/19'),
+ json={},
+ status=404
+ )
+ responses.add(
+ responses.POST,
+ github_url('/repos/ursa-labs/ursabot/issues/19/comments'),
+ json={}
+ )
+
+ def handler(command, **kwargs):
+ pass
+
+ payload = load_fixture('event-issue-comment-without-pull-request.json')
+ bot = CommentBot(name='ursabot', token='', handler=handler)
+ bot.handle('issue_comment', payload)
+
+ post = responses.calls[2]
+ assert json.loads(post.request.body) == {
+ 'body': "The comment bot only listens to pull request comments!"
+ }
+
+
+def test_respond_with_usage(load_fixture, responses):
+ responses.add(
+ responses.GET,
+ github_url('/repositories/169101701/issues/26'),
+ json=load_fixture('issue-26.json'),
+ status=200
+ )
+ responses.add(
+ responses.GET,
+ github_url('/repos/ursa-labs/ursabot/pulls/26'),
+ json=load_fixture('pull-request-26.json'),
+ status=200
+ )
+ responses.add(
+ responses.GET,
+ github_url('/repos/ursa-labs/ursabot/issues/comments/480243811'),
+ json=load_fixture('issue-comment-480243811.json')
+ )
+ responses.add(
+ responses.POST,
+ github_url('/repos/ursa-labs/ursabot/issues/26/comments'),
+ json={}
+ )
+
+ def handler(command, **kwargs):
+ raise CommandError('test-usage')
+
+ payload = load_fixture('event-issue-comment-with-empty-command.json')
+ bot = CommentBot(name='ursabot', token='', handler=handler)
+ bot.handle('issue_comment', payload)
+
+ post = responses.calls[3]
+ assert json.loads(post.request.body) == {'body': '```\ntest-usage\n```'}
+
+
+@pytest.mark.parametrize(('command', 'reaction'), [
+ ('@ursabot build', '+1'),
+ ('@ursabot build\nwith a comment', '+1'),
+ ('@ursabot listen', '-1'),
+])
+def test_issue_comment_with_commands(load_fixture, responses, command,
+ reaction):
+ responses.add(
+ responses.GET,
+ github_url('/repositories/169101701/issues/26'),
+ json=load_fixture('issue-26.json'),
+ status=200
+ )
+ responses.add(
+ responses.GET,
+ github_url('/repos/ursa-labs/ursabot/pulls/26'),
+ json=load_fixture('pull-request-26.json'),
+ status=200
+ )
+ responses.add(
+ responses.GET,
+ github_url('/repos/ursa-labs/ursabot/issues/comments/480248726'),
+ json=load_fixture('issue-comment-480248726.json')
+ )
+ responses.add(
+ responses.POST,
+ github_url(
+ '/repos/ursa-labs/ursabot/issues/comments/480248726/reactions'
+ ),
+ json={}
+ )
+
+ def handler(command, **kwargs):
+ if command == 'build':
+ return True
+ else:
+ raise ValueError('Only `build` command is supported.')
+
+ payload = load_fixture('event-issue-comment-build-command.json')
+ payload["comment"]["body"] = command
+
+ bot = CommentBot(name='ursabot', token='', handler=handler)
+ bot.handle('issue_comment', payload)
+
+ post = responses.calls[3]
+ assert json.loads(post.request.body) == {'content': reaction}
+
+
+def test_issue_comment_with_commands_bot_not_first(load_fixture, responses):
+ # when the @-mention is not first, this is a no-op
+ handler = Mock()
+
+ payload = load_fixture('event-issue-comment-build-command.json')
+ payload["comment"]["body"] = 'with a comment\n@ursabot build'
+
+ bot = CommentBot(name='ursabot', token='', handler=handler)
+ bot.handle('issue_comment', payload)
+
+ handler.assert_not_called()
diff --git a/src/arrow/dev/archery/archery/tests/test_cli.py b/src/arrow/dev/archery/archery/tests/test_cli.py
new file mode 100644
index 000000000..3891a2c28
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/test_cli.py
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+from unittest.mock import patch
+
+from click.testing import CliRunner
+
+from archery.cli import archery
+
+
+@patch("archery.linking.check_dynamic_library_dependencies")
+def test_linking_check_dependencies(fn):
+ args = [
+ "linking",
+ "check-dependencies",
+ "-a", "libarrow",
+ "-d", "libcurl",
+ "somelib.so"
+ ]
+ result = CliRunner().invoke(archery, args)
+ assert result.exit_code == 0
+ fn.assert_called_once_with(
+ Path('somelib.so'), allowed={'libarrow'}, disallowed={'libcurl'}
+ )
diff --git a/src/arrow/dev/archery/archery/tests/test_release.py b/src/arrow/dev/archery/archery/tests/test_release.py
new file mode 100644
index 000000000..75aac8921
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/test_release.py
@@ -0,0 +1,333 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from archery.release import (
+ Release, MajorRelease, MinorRelease, PatchRelease,
+ Jira, Version, Issue, CommitTitle, Commit
+)
+from archery.testing import DotDict
+
+
+# subset of issues per revision
+_issues = {
+ "1.0.1": [
+ Issue("ARROW-9684", type="Bug", summary="[C++] Title"),
+ Issue("ARROW-9667", type="New Feature", summary="[Crossbow] Title"),
+ Issue("ARROW-9659", type="Bug", summary="[C++] Title"),
+ Issue("ARROW-9644", type="Bug", summary="[C++][Dataset] Title"),
+ Issue("ARROW-9643", type="Bug", summary="[C++] Title"),
+ Issue("ARROW-9609", type="Bug", summary="[C++] Title"),
+ Issue("ARROW-9606", type="Bug", summary="[C++][Dataset] Title")
+ ],
+ "1.0.0": [
+ Issue("ARROW-300", type="New Feature", summary="[Format] Title"),
+ Issue("ARROW-4427", type="Task", summary="[Doc] Title"),
+ Issue("ARROW-5035", type="Improvement", summary="[C#] Title"),
+ Issue("ARROW-8473", type="Bug", summary="[Rust] Title"),
+ Issue("ARROW-8472", type="Bug", summary="[Go][Integration] Title"),
+ Issue("ARROW-8471", type="Bug", summary="[C++][Integration] Title"),
+ Issue("ARROW-8974", type="Improvement", summary="[C++] Title"),
+ Issue("ARROW-8973", type="New Feature", summary="[Java] Title")
+ ],
+ "0.17.1": [
+ Issue("ARROW-8684", type="Bug", summary="[Python] Title"),
+ Issue("ARROW-8657", type="Bug", summary="[C++][Parquet] Title"),
+ Issue("ARROW-8641", type="Bug", summary="[Python] Title"),
+ Issue("ARROW-8609", type="Bug", summary="[C++] Title"),
+ ],
+ "0.17.0": [
+ Issue("ARROW-2882", type="New Feature", summary="[C++][Python] Title"),
+ Issue("ARROW-2587", type="Bug", summary="[Python] Title"),
+ Issue("ARROW-2447", type="Improvement", summary="[C++] Title"),
+ Issue("ARROW-2255", type="Bug", summary="[Integration] Title"),
+ Issue("ARROW-1907", type="Bug", summary="[C++/Python] Title"),
+ Issue("ARROW-1636", type="New Feature", summary="[Format] Title")
+ ]
+}
+
+
+class FakeJira(Jira):
+
+ def __init__(self):
+ pass
+
+ def project_versions(self, project='ARROW'):
+ return [
+ Version.parse("3.0.0", released=False),
+ Version.parse("2.0.0", released=False),
+ Version.parse("1.1.0", released=False),
+ Version.parse("1.0.1", released=False),
+ Version.parse("1.0.0", released=True),
+ Version.parse("0.17.1", released=True),
+ Version.parse("0.17.0", released=True),
+ Version.parse("0.16.0", released=True),
+ Version.parse("0.15.2", released=True),
+ Version.parse("0.15.1", released=True),
+ Version.parse("0.15.0", released=True),
+ ]
+
+ def project_issues(self, version, project='ARROW'):
+ return _issues[str(version)]
+
+
+@pytest.fixture
+def fake_jira():
+ return FakeJira()
+
+
+def test_version(fake_jira):
+ v = Version.parse("1.2.5")
+ assert str(v) == "1.2.5"
+ assert v.major == 1
+ assert v.minor == 2
+ assert v.patch == 5
+ assert v.released is False
+ assert v.release_date is None
+
+ v = Version.parse("1.0.0", released=True, release_date="2020-01-01")
+ assert str(v) == "1.0.0"
+ assert v.major == 1
+ assert v.minor == 0
+ assert v.patch == 0
+ assert v.released is True
+ assert v.release_date == "2020-01-01"
+
+
+def test_issue(fake_jira):
+ i = Issue("ARROW-1234", type='Bug', summary="title")
+ assert i.key == "ARROW-1234"
+ assert i.type == "Bug"
+ assert i.summary == "title"
+ assert i.project == "ARROW"
+ assert i.number == 1234
+
+ i = Issue("PARQUET-1111", type='Improvement', summary="another title")
+ assert i.key == "PARQUET-1111"
+ assert i.type == "Improvement"
+ assert i.summary == "another title"
+ assert i.project == "PARQUET"
+ assert i.number == 1111
+
+ fake_jira_issue = DotDict({
+ 'key': 'ARROW-2222',
+ 'fields': {
+ 'issuetype': {
+ 'name': 'Feature'
+ },
+ 'summary': 'Issue title'
+ }
+ })
+ i = Issue.from_jira(fake_jira_issue)
+ assert i.key == "ARROW-2222"
+ assert i.type == "Feature"
+ assert i.summary == "Issue title"
+ assert i.project == "ARROW"
+ assert i.number == 2222
+
+
+def test_commit_title():
+ t = CommitTitle.parse(
+ "ARROW-9598: [C++][Parquet] Fix writing nullable structs"
+ )
+ assert t.project == "ARROW"
+ assert t.issue == "ARROW-9598"
+ assert t.components == ["C++", "Parquet"]
+ assert t.summary == "Fix writing nullable structs"
+
+ t = CommitTitle.parse(
+ "ARROW-8002: [C++][Dataset][R] Support partitioned dataset writing"
+ )
+ assert t.project == "ARROW"
+ assert t.issue == "ARROW-8002"
+ assert t.components == ["C++", "Dataset", "R"]
+ assert t.summary == "Support partitioned dataset writing"
+
+ t = CommitTitle.parse(
+ "ARROW-9600: [Rust][Arrow] pin older version of proc-macro2 during "
+ "build"
+ )
+ assert t.project == "ARROW"
+ assert t.issue == "ARROW-9600"
+ assert t.components == ["Rust", "Arrow"]
+ assert t.summary == "pin older version of proc-macro2 during build"
+
+ t = CommitTitle.parse("[Release] Update versions for 1.0.0")
+ assert t.project is None
+ assert t.issue is None
+ assert t.components == ["Release"]
+ assert t.summary == "Update versions for 1.0.0"
+
+ t = CommitTitle.parse("[Python][Doc] Fix rst role dataset.rst (#7725)")
+ assert t.project is None
+ assert t.issue is None
+ assert t.components == ["Python", "Doc"]
+ assert t.summary == "Fix rst role dataset.rst (#7725)"
+
+ t = CommitTitle.parse(
+ "PARQUET-1882: [C++] Buffered Reads should allow for 0 length"
+ )
+ assert t.project == 'PARQUET'
+ assert t.issue == 'PARQUET-1882'
+ assert t.components == ["C++"]
+ assert t.summary == "Buffered Reads should allow for 0 length"
+
+ t = CommitTitle.parse(
+ "ARROW-9340 [R] Use CRAN version of decor package "
+ "\nsomething else\n"
+ "\nwhich should be truncated"
+ )
+ assert t.project == 'ARROW'
+ assert t.issue == 'ARROW-9340'
+ assert t.components == ["R"]
+ assert t.summary == "Use CRAN version of decor package "
+
+
+def test_release_basics(fake_jira):
+ r = Release.from_jira("1.0.0", jira=fake_jira)
+ assert isinstance(r, MajorRelease)
+ assert r.is_released is True
+ assert r.branch == 'master'
+ assert r.tag == 'apache-arrow-1.0.0'
+
+ r = Release.from_jira("1.1.0", jira=fake_jira)
+ assert isinstance(r, MinorRelease)
+ assert r.is_released is False
+ assert r.branch == 'maint-1.x.x'
+ assert r.tag == 'apache-arrow-1.1.0'
+
+ # minor releases before 1.0 are treated as major releases
+ r = Release.from_jira("0.17.0", jira=fake_jira)
+ assert isinstance(r, MajorRelease)
+ assert r.is_released is True
+ assert r.branch == 'master'
+ assert r.tag == 'apache-arrow-0.17.0'
+
+ r = Release.from_jira("0.17.1", jira=fake_jira)
+ assert isinstance(r, PatchRelease)
+ assert r.is_released is True
+ assert r.branch == 'maint-0.17.x'
+ assert r.tag == 'apache-arrow-0.17.1'
+
+
+def test_previous_and_next_release(fake_jira):
+ r = Release.from_jira("3.0.0", jira=fake_jira)
+ assert isinstance(r.previous, MajorRelease)
+ assert r.previous.version == Version.parse("2.0.0")
+ with pytest.raises(ValueError, match="There is no upcoming release set"):
+ assert r.next
+
+ r = Release.from_jira("2.0.0", jira=fake_jira)
+ assert isinstance(r.previous, MajorRelease)
+ assert isinstance(r.next, MajorRelease)
+ assert r.previous.version == Version.parse("1.0.0")
+ assert r.next.version == Version.parse("3.0.0")
+
+ r = Release.from_jira("1.1.0", jira=fake_jira)
+ assert isinstance(r.previous, MajorRelease)
+ assert isinstance(r.next, MajorRelease)
+ assert r.previous.version == Version.parse("1.0.0")
+ assert r.next.version == Version.parse("2.0.0")
+
+ r = Release.from_jira("1.0.0", jira=fake_jira)
+ assert isinstance(r.next, MajorRelease)
+ assert isinstance(r.previous, MajorRelease)
+ assert r.previous.version == Version.parse("0.17.0")
+ assert r.next.version == Version.parse("2.0.0")
+
+ r = Release.from_jira("0.17.0", jira=fake_jira)
+ assert isinstance(r.previous, MajorRelease)
+ assert r.previous.version == Version.parse("0.16.0")
+
+ r = Release.from_jira("0.15.2", jira=fake_jira)
+ assert isinstance(r.previous, PatchRelease)
+ assert isinstance(r.next, MajorRelease)
+ assert r.previous.version == Version.parse("0.15.1")
+ assert r.next.version == Version.parse("0.16.0")
+
+ r = Release.from_jira("0.15.1", jira=fake_jira)
+ assert isinstance(r.previous, MajorRelease)
+ assert isinstance(r.next, PatchRelease)
+ assert r.previous.version == Version.parse("0.15.0")
+ assert r.next.version == Version.parse("0.15.2")
+
+
+def test_release_issues(fake_jira):
+ # major release issues
+ r = Release.from_jira("1.0.0", jira=fake_jira)
+ assert r.issues.keys() == set([
+ "ARROW-300",
+ "ARROW-4427",
+ "ARROW-5035",
+ "ARROW-8473",
+ "ARROW-8472",
+ "ARROW-8471",
+ "ARROW-8974",
+ "ARROW-8973"
+ ])
+ # minor release issues
+ r = Release.from_jira("0.17.0", jira=fake_jira)
+ assert r.issues.keys() == set([
+ "ARROW-2882",
+ "ARROW-2587",
+ "ARROW-2447",
+ "ARROW-2255",
+ "ARROW-1907",
+ "ARROW-1636",
+ ])
+ # patch release issues
+ r = Release.from_jira("1.0.1", jira=fake_jira)
+ assert r.issues.keys() == set([
+ "ARROW-9684",
+ "ARROW-9667",
+ "ARROW-9659",
+ "ARROW-9644",
+ "ARROW-9643",
+ "ARROW-9609",
+ "ARROW-9606"
+ ])
+
+
+@pytest.mark.parametrize(('version', 'ncommits'), [
+ ("1.0.0", 771),
+ ("0.17.1", 27),
+ ("0.17.0", 569),
+ ("0.15.1", 41)
+])
+def test_release_commits(fake_jira, version, ncommits):
+ r = Release.from_jira(version, jira=fake_jira)
+ assert len(r.commits) == ncommits
+ for c in r.commits:
+ assert isinstance(c, Commit)
+ assert isinstance(c.title, CommitTitle)
+ assert c.url.endswith(c.hexsha)
+
+
+def test_maintenance_patch_selection(fake_jira):
+ r = Release.from_jira("0.17.1", jira=fake_jira)
+
+ shas_to_pick = [
+ c.hexsha for c in r.commits_to_pick(exclude_already_applied=False)
+ ]
+ expected = [
+ '8939b4bd446ee406d5225c79d563a27d30fd7d6d',
+ 'bcef6c95a324417e85e0140f9745d342cd8784b3',
+ '6002ec388840de5622e39af85abdc57a2cccc9b2',
+ '9123dadfd123bca7af4eaa9455f5b0d1ca8b929d',
+ ]
+ assert shas_to_pick == expected
diff --git a/src/arrow/dev/archery/archery/tests/test_testing.py b/src/arrow/dev/archery/archery/tests/test_testing.py
new file mode 100644
index 000000000..117b9288d
--- /dev/null
+++ b/src/arrow/dev/archery/archery/tests/test_testing.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+
+import pytest
+
+from archery.testing import PartialEnv, assert_subprocess_calls
+
+
+def test_partial_env():
+ assert PartialEnv(a=1, b=2) == {'a': 1, 'b': 2, 'c': 3}
+ assert PartialEnv(a=1) == {'a': 1, 'b': 2, 'c': 3}
+ assert PartialEnv(a=1, b=2) == {'a': 1, 'b': 2}
+ assert PartialEnv(a=1, b=2) != {'b': 2, 'c': 3}
+ assert PartialEnv(a=1, b=2) != {'a': 1, 'c': 3}
+
+
+def test_assert_subprocess_calls():
+ expected_calls = [
+ "echo Hello",
+ ["echo", "World"]
+ ]
+ with assert_subprocess_calls(expected_calls):
+ subprocess.run(['echo', 'Hello'])
+ subprocess.run(['echo', 'World'])
+
+ expected_env = PartialEnv(
+ CUSTOM_ENV_A='a',
+ CUSTOM_ENV_C='c'
+ )
+ with assert_subprocess_calls(expected_calls, env=expected_env):
+ env = {
+ 'CUSTOM_ENV_A': 'a',
+ 'CUSTOM_ENV_B': 'b',
+ 'CUSTOM_ENV_C': 'c'
+ }
+ subprocess.run(['echo', 'Hello'], env=env)
+ subprocess.run(['echo', 'World'], env=env)
+
+ with pytest.raises(AssertionError):
+ with assert_subprocess_calls(expected_calls, env=expected_env):
+ env = {
+ 'CUSTOM_ENV_B': 'b',
+ 'CUSTOM_ENV_C': 'c'
+ }
+ subprocess.run(['echo', 'Hello'], env=env)
+ subprocess.run(['echo', 'World'], env=env)
diff --git a/src/arrow/dev/archery/archery/utils/__init__.py b/src/arrow/dev/archery/archery/utils/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/arrow/dev/archery/archery/utils/cache.py b/src/arrow/dev/archery/archery/utils/cache.py
new file mode 100644
index 000000000..d92c5f32e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/cache.py
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+import os
+from urllib.request import urlopen
+
+from .logger import logger
+
+ARCHERY_CACHE_DIR = Path.home() / ".cache" / "archery"
+
+
+class Cache:
+ """ Cache stores downloaded objects, notably apache-rat.jar. """
+
+ def __init__(self, path=ARCHERY_CACHE_DIR):
+ self.root = path
+
+ if not path.exists():
+ os.makedirs(path)
+
+ def key_path(self, key):
+ """ Return the full path of a key. """
+ return self.root/key
+
+ def get(self, key):
+ """ Return the full path of a key if cached, None otherwise. """
+ path = self.key_path(key)
+ return path if path.exists() else None
+
+ def delete(self, key):
+ """ Remove a key (and the file) from the cache. """
+ path = self.get(key)
+ if path:
+ path.unlink()
+
+ def get_or_insert(self, key, create):
+ """
+ Get or Insert a key from the cache. If the key is not found, the
+ `create` closure will be evaluated.
+
+ The `create` closure takes a single parameter, the path where the
+ object should be store. The file should only be created upon success.
+ """
+ path = self.key_path(key)
+
+ if not path.exists():
+ create(path)
+
+ return path
+
+ def get_or_insert_from_url(self, key, url):
+ """
+ Get or Insert a key from the cache. If the key is not found, the file
+ is downloaded from `url`.
+ """
+ def download(path):
+ """ Tiny wrapper that download a file and save as key. """
+ logger.debug("Downloading {} as {}".format(url, path))
+ conn = urlopen(url)
+ # Ensure the download is completed before writing to disks.
+ content = conn.read()
+ with open(path, "wb") as path_fd:
+ path_fd.write(content)
+
+ return self.get_or_insert(key, download)
diff --git a/src/arrow/dev/archery/archery/utils/cli.py b/src/arrow/dev/archery/archery/utils/cli.py
new file mode 100644
index 000000000..701abe925
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/cli.py
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import importlib
+
+import click
+
+from .source import ArrowSources, InvalidArrowSource
+
+
+class ArrowBool(click.types.BoolParamType):
+ """
+ ArrowBool supports the 'ON' and 'OFF' values on top of the values
+ supported by BoolParamType. This is convenient to port script which exports
+ CMake options variables.
+ """
+ name = "boolean"
+
+ def convert(self, value, param, ctx):
+ if isinstance(value, str):
+ lowered = value.lower()
+ if lowered == "on":
+ return True
+ elif lowered == "off":
+ return False
+
+ return super().convert(value, param, ctx)
+
+
+def validate_arrow_sources(ctx, param, src):
+ """
+ Ensure a directory contains Arrow cpp sources.
+ """
+ try:
+ return ArrowSources.find(src)
+ except InvalidArrowSource as e:
+ raise click.BadParameter(str(e))
+
+
+def add_optional_command(name, module, function, parent):
+ try:
+ module = importlib.import_module(module, package="archery")
+ command = getattr(module, function)
+ except ImportError as exc:
+ error_message = exc.name
+
+ @parent.command(
+ name,
+ context_settings={
+ "allow_extra_args": True,
+ "ignore_unknown_options": True,
+ }
+ )
+ def command():
+ raise click.ClickException(
+ f"Couldn't import command `{name}` due to {error_message}"
+ )
+ else:
+ parent.add_command(command)
diff --git a/src/arrow/dev/archery/archery/utils/cmake.py b/src/arrow/dev/archery/archery/utils/cmake.py
new file mode 100644
index 000000000..f93895b1a
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/cmake.py
@@ -0,0 +1,215 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+from shutil import rmtree, which
+
+from .command import Command, default_bin
+
+
+class CMake(Command):
+ def __init__(self, cmake_bin=None):
+ self.bin = default_bin(cmake_bin, "cmake")
+
+ @staticmethod
+ def default_generator():
+ """ Infer default generator.
+
+ Gives precedence to ninja if there exists an executable named `ninja`
+ in the search path.
+ """
+ found_ninja = which("ninja")
+ return "Ninja" if found_ninja else "Unix Makefiles"
+
+
+cmake = CMake()
+
+
+class CMakeDefinition:
+ """ CMakeDefinition captures the cmake invocation arguments.
+
+ It allows creating build directories with the same definition, e.g.
+ ```
+ build_1 = cmake_def.build("/tmp/build-1")
+ build_2 = cmake_def.build("/tmp/build-2")
+
+ ...
+
+ build1.all()
+ build2.all()
+ """
+
+ def __init__(self, source, build_type="release", generator=None,
+ definitions=None, env=None):
+ """ Initialize a CMakeDefinition
+
+ Parameters
+ ----------
+ source : str
+ Source directory where the top-level CMakeLists.txt is
+ located. This is usually the root of the project.
+ generator : str, optional
+ definitions: list(str), optional
+ env : dict(str,str), optional
+ Environment to use when invoking cmake. This can be required to
+ work around cmake deficiencies, e.g. CC and CXX.
+ """
+ self.source = os.path.abspath(source)
+ self.build_type = build_type
+ self.generator = generator if generator else cmake.default_generator()
+ self.definitions = definitions if definitions else []
+ self.env = env
+
+ @property
+ def arguments(self):
+ """" Return the arguments to cmake invocation. """
+ arguments = [
+ "-G{}".format(self.generator),
+ ] + self.definitions + [
+ self.source
+ ]
+ return arguments
+
+ def build(self, build_dir, force=False, cmd_kwargs=None, **kwargs):
+ """ Invoke cmake into a build directory.
+
+ Parameters
+ ----------
+ build_dir : str
+ Directory in which the CMake build will be instantiated.
+ force : bool
+ If the build folder exists, delete it before. Otherwise if it's
+ present, an error will be returned.
+ """
+ if os.path.exists(build_dir):
+ # Extra safety to ensure we're deleting a build folder.
+ if not CMakeBuild.is_build_dir(build_dir):
+ raise FileExistsError(
+ "{} is not a cmake build".format(build_dir)
+ )
+ if not force:
+ raise FileExistsError(
+ "{} exists use force=True".format(build_dir)
+ )
+ rmtree(build_dir)
+
+ os.mkdir(build_dir)
+
+ cmd_kwargs = cmd_kwargs if cmd_kwargs else {}
+ cmake(*self.arguments, cwd=build_dir, env=self.env, **cmd_kwargs)
+ return CMakeBuild(build_dir, self.build_type, definition=self,
+ **kwargs)
+
+ def __repr__(self):
+ return "CMakeDefinition[source={}]".format(self.source)
+
+
+CMAKE_BUILD_TYPE_RE = re.compile("CMAKE_BUILD_TYPE:STRING=([a-zA-Z]+)")
+
+
+class CMakeBuild(CMake):
+ """ CMakeBuild represents a build directory initialized by cmake.
+
+ The build instance can be used to build/test/install. It alleviates the
+ user to know which generator is used.
+ """
+
+ def __init__(self, build_dir, build_type, definition=None):
+ """ Initialize a CMakeBuild.
+
+ The caller must ensure that cmake was invoked in the build directory.
+
+ Parameters
+ ----------
+ definition : CMakeDefinition
+ The definition to build from.
+ build_dir : str
+ The build directory to setup into.
+ """
+ assert CMakeBuild.is_build_dir(build_dir)
+ super().__init__()
+ self.build_dir = os.path.abspath(build_dir)
+ self.build_type = build_type
+ self.definition = definition
+
+ @property
+ def binaries_dir(self):
+ return os.path.join(self.build_dir, self.build_type)
+
+ def run(self, *argv, verbose=False, **kwargs):
+ cmake_args = ["--build", self.build_dir, "--"]
+ extra = []
+ if verbose:
+ extra.append("-v" if self.bin.endswith("ninja") else "VERBOSE=1")
+ # Commands must be ran under the build directory
+ return super().run(*cmake_args, *extra,
+ *argv, **kwargs, cwd=self.build_dir)
+
+ def all(self):
+ return self.run("all")
+
+ def clean(self):
+ return self.run("clean")
+
+ def install(self):
+ return self.run("install")
+
+ def test(self):
+ return self.run("test")
+
+ @staticmethod
+ def is_build_dir(path):
+ """ Indicate if a path is CMake build directory.
+
+ This method only checks for the existence of paths and does not do any
+ validation whatsoever.
+ """
+ cmake_cache = os.path.join(path, "CMakeCache.txt")
+ cmake_files = os.path.join(path, "CMakeFiles")
+ return os.path.exists(cmake_cache) and os.path.exists(cmake_files)
+
+ @staticmethod
+ def from_path(path):
+ """ Instantiate a CMakeBuild from a path.
+
+ This is used to recover from an existing physical directory (created
+ with or without CMakeBuild).
+
+ Note that this method is not idempotent as the original definition will
+ be lost. Only build_type is recovered.
+ """
+ if not CMakeBuild.is_build_dir(path):
+ raise ValueError("Not a valid CMakeBuild path: {}".format(path))
+
+ build_type = None
+ # Infer build_type by looking at CMakeCache.txt and looking for a magic
+ # definition
+ cmake_cache_path = os.path.join(path, "CMakeCache.txt")
+ with open(cmake_cache_path, "r") as cmake_cache:
+ candidates = CMAKE_BUILD_TYPE_RE.findall(cmake_cache.read())
+ build_type = candidates[0].lower() if candidates else "release"
+
+ return CMakeBuild(path, build_type)
+
+ def __repr__(self):
+ return ("CMakeBuild["
+ "build = {},"
+ "build_type = {},"
+ "definition = {}]".format(self.build_dir,
+ self.build_type,
+ self.definition))
diff --git a/src/arrow/dev/archery/archery/utils/command.py b/src/arrow/dev/archery/archery/utils/command.py
new file mode 100644
index 000000000..f655e2ef2
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/command.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shlex
+import shutil
+import subprocess
+
+from .logger import logger, ctx
+
+
+def default_bin(name, default):
+ assert(default)
+ env_name = "ARCHERY_{0}_BIN".format(default.upper())
+ return name if name else os.environ.get(env_name, default)
+
+
+# Decorator running a command and returning stdout
+class capture_stdout:
+ def __init__(self, strip=False, listify=False):
+ self.strip = strip
+ self.listify = listify
+
+ def __call__(self, f):
+ def strip_it(x):
+ return x.strip() if self.strip else x
+
+ def list_it(x):
+ return x.decode('utf-8').splitlines() if self.listify else x
+
+ def wrapper(*argv, **kwargs):
+ # Ensure stdout is captured
+ kwargs["stdout"] = subprocess.PIPE
+ return list_it(strip_it(f(*argv, **kwargs).stdout))
+ return wrapper
+
+
+class Command:
+ """
+ A runnable command.
+
+ Class inheriting from the Command class must provide the bin
+ property/attribute.
+ """
+
+ def __init__(self, bin):
+ self.bin = bin
+
+ def run(self, *argv, **kwargs):
+ assert hasattr(self, "bin")
+ invocation = shlex.split(self.bin)
+ invocation.extend(argv)
+
+ for key in ["stdout", "stderr"]:
+ # Preserve caller intention, otherwise silence
+ if key not in kwargs and ctx.quiet:
+ kwargs[key] = subprocess.PIPE
+
+ # Prefer safe by default
+ if "check" not in kwargs:
+ kwargs["check"] = True
+
+ logger.debug("Executing `{}`".format(invocation))
+ return subprocess.run(invocation, **kwargs)
+
+ @property
+ def available(self):
+ """
+ Indicate if the command binary is found in PATH.
+ """
+ binary = shlex.split(self.bin)[0]
+ return shutil.which(binary) is not None
+
+ def __call__(self, *argv, **kwargs):
+ return self.run(*argv, **kwargs)
+
+
+class CommandStackMixin:
+ def run(self, *argv, **kwargs):
+ stacked_args = self.argv + argv
+ return super(CommandStackMixin, self).run(*stacked_args, **kwargs)
+
+
+class Bash(Command):
+ def __init__(self, bash_bin=None):
+ self.bin = default_bin(bash_bin, "bash")
diff --git a/src/arrow/dev/archery/archery/utils/git.py b/src/arrow/dev/archery/archery/utils/git.py
new file mode 100644
index 000000000..798bc5d70
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/git.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .command import Command, capture_stdout, default_bin
+from ..compat import _stringify_path
+
+
+# Decorator prepending argv with the git sub-command found with the method
+# name.
+def git_cmd(fn):
+ # function name is the subcommand
+ sub_cmd = fn.__name__.replace("_", "-")
+
+ def wrapper(self, *argv, **kwargs):
+ return fn(self, sub_cmd, *argv, **kwargs)
+ return wrapper
+
+
+class Git(Command):
+ def __init__(self, git_bin=None):
+ self.bin = default_bin(git_bin, "git")
+
+ def run_cmd(self, cmd, *argv, git_dir=None, **kwargs):
+ """ Inject flags before sub-command in argv. """
+ opts = []
+ if git_dir is not None:
+ opts.extend(["-C", _stringify_path(git_dir)])
+
+ return self.run(*opts, cmd, *argv, **kwargs)
+
+ @capture_stdout(strip=False)
+ @git_cmd
+ def archive(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @git_cmd
+ def clone(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @git_cmd
+ def fetch(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @git_cmd
+ def checkout(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ def dirty(self, **kwargs):
+ return len(self.status("--short", **kwargs)) > 0
+
+ @git_cmd
+ def log(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @capture_stdout(strip=True, listify=True)
+ @git_cmd
+ def ls_files(self, *argv, listify=False, **kwargs):
+ stdout = self.run_cmd(*argv, **kwargs)
+ return stdout
+
+ @capture_stdout(strip=True)
+ @git_cmd
+ def rev_parse(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @capture_stdout(strip=True)
+ @git_cmd
+ def status(self, *argv, **kwargs):
+ return self.run_cmd(*argv, **kwargs)
+
+ @capture_stdout(strip=True)
+ def head(self, **kwargs):
+ """ Return commit pointed by HEAD. """
+ return self.rev_parse("HEAD", **kwargs)
+
+ @capture_stdout(strip=True)
+ def current_branch(self, **kwargs):
+ return self.rev_parse("--abbrev-ref", "HEAD", **kwargs)
+
+ def repository_root(self, git_dir=None, **kwargs):
+ """ Locates the repository's root path from a subdirectory. """
+ stdout = self.rev_parse("--show-toplevel", git_dir=git_dir, **kwargs)
+ return stdout.decode('utf-8')
+
+
+git = Git()
diff --git a/src/arrow/dev/archery/archery/utils/lint.py b/src/arrow/dev/archery/archery/utils/lint.py
new file mode 100644
index 000000000..d95bfeea3
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/lint.py
@@ -0,0 +1,429 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import gzip
+import os
+from pathlib import Path
+
+import click
+
+from .command import Bash, Command, default_bin
+from .cmake import CMake
+from .git import git
+from .logger import logger
+from ..lang.cpp import CppCMakeDefinition, CppConfiguration
+from ..lang.python import Autopep8, Flake8, NumpyDoc
+from .rat import Rat, exclusion_from_globs
+from .tmpdir import tmpdir
+
+
+_archery_install_msg = (
+ "Please install archery using: `pip install -e dev/archery[lint]`. "
+)
+
+
+class LintValidationException(Exception):
+ pass
+
+
+class LintResult:
+ def __init__(self, success, reason=None):
+ self.success = success
+
+ def ok(self):
+ if not self.success:
+ raise LintValidationException
+
+ @staticmethod
+ def from_cmd(command_result):
+ return LintResult(command_result.returncode == 0)
+
+
+def cpp_linter(src, build_dir, clang_format=True, cpplint=True,
+ clang_tidy=False, iwyu=False, iwyu_all=False,
+ fix=False):
+ """ Run clang-format, cpplint and clang-tidy on cpp/ codebase. """
+ logger.info("Running C++ linters")
+
+ cmake = CMake()
+ if not cmake.available:
+ logger.error("cpp linter requested but cmake binary not found.")
+ return
+
+ # A cmake build directory is required to populate `compile_commands.json`
+ # which in turn is required by clang-tidy. It also provides a convenient
+ # way to hide clang-format/clang-tidy invocation via the Generate
+ # (ninja/make) targets.
+
+ # ARROW_LINT_ONLY exits early but ignore building compile_command.json
+ lint_only = not (iwyu or clang_tidy)
+ cmake_args = {"with_python": False, "with_lint_only": lint_only}
+ cmake_def = CppCMakeDefinition(src.cpp, CppConfiguration(**cmake_args))
+
+ build = cmake_def.build(build_dir)
+ if clang_format:
+ target = "format" if fix else "check-format"
+ yield LintResult.from_cmd(build.run(target, check=False))
+
+ if cpplint:
+ yield LintResult.from_cmd(build.run("lint", check=False))
+ yield LintResult.from_cmd(build.run("lint_cpp_cli", check=False))
+
+ if clang_tidy:
+ yield LintResult.from_cmd(build.run("check-clang-tidy", check=False))
+
+ if iwyu:
+ if iwyu_all:
+ iwyu_cmd = "iwyu-all"
+ else:
+ iwyu_cmd = "iwyu"
+ yield LintResult.from_cmd(build.run(iwyu_cmd, check=False))
+
+
+class CMakeFormat(Command):
+
+ def __init__(self, paths, cmake_format_bin=None):
+ self.check_version()
+ self.bin = default_bin(cmake_format_bin, "cmake-format")
+ self.paths = paths
+
+ @classmethod
+ def from_patterns(cls, base_path, include_patterns, exclude_patterns):
+ paths = {
+ str(path.as_posix())
+ for pattern in include_patterns
+ for path in base_path.glob(pattern)
+ }
+ for pattern in exclude_patterns:
+ pattern = (base_path / pattern).as_posix()
+ paths -= set(fnmatch.filter(paths, str(pattern)))
+ return cls(paths)
+
+ @staticmethod
+ def check_version():
+ try:
+ # cmake_format is part of the cmakelang package
+ import cmakelang
+ except ImportError:
+ raise ImportError(
+
+ )
+ # pin a specific version of cmake_format, must be updated in setup.py
+ if cmakelang.__version__ != "0.6.13":
+ raise LintValidationException(
+ f"Wrong version of cmake_format is detected. "
+ f"{_archery_install_msg}"
+ )
+
+ def check(self):
+ return self.run("-l", "error", "--check", *self.paths, check=False)
+
+ def fix(self):
+ return self.run("--in-place", *self.paths, check=False)
+
+
+def cmake_linter(src, fix=False):
+ """
+ Run cmake-format on all CMakeFiles.txt
+ """
+ logger.info("Running cmake-format linters")
+
+ cmake_format = CMakeFormat.from_patterns(
+ src.path,
+ include_patterns=[
+ 'ci/**/*.cmake',
+ 'cpp/CMakeLists.txt',
+ 'cpp/src/**/CMakeLists.txt',
+ 'cpp/cmake_modules/*.cmake',
+ 'go/**/CMakeLists.txt',
+ 'java/**/CMakeLists.txt',
+ 'matlab/**/CMakeLists.txt',
+ 'python/CMakeLists.txt',
+ ],
+ exclude_patterns=[
+ 'cpp/cmake_modules/FindNumPy.cmake',
+ 'cpp/cmake_modules/FindPythonLibsNew.cmake',
+ 'cpp/cmake_modules/UseCython.cmake',
+ 'cpp/src/arrow/util/config.h.cmake',
+ ]
+ )
+ method = cmake_format.fix if fix else cmake_format.check
+
+ yield LintResult.from_cmd(method())
+
+
+def python_linter(src, fix=False):
+ """Run Python linters on python/pyarrow, python/examples, setup.py
+ and dev/. """
+ setup_py = os.path.join(src.python, "setup.py")
+ setup_cfg = os.path.join(src.python, "setup.cfg")
+
+ logger.info("Running Python formatter (autopep8)")
+
+ autopep8 = Autopep8()
+ if not autopep8.available:
+ logger.error(
+ "Python formatter requested but autopep8 binary not found. "
+ f"{_archery_install_msg}")
+ return
+
+ # Gather files for autopep8
+ patterns = ["python/pyarrow/**/*.py",
+ "python/pyarrow/**/*.pyx",
+ "python/pyarrow/**/*.pxd",
+ "python/pyarrow/**/*.pxi",
+ "python/examples/**/*.py",
+ "dev/archery/**/*.py"]
+ files = [setup_py]
+ for pattern in patterns:
+ files += list(map(str, Path(src.path).glob(pattern)))
+
+ args = ['--global-config', setup_cfg, '--ignore-local-config']
+ if fix:
+ args += ['-j0', '--in-place']
+ args += sorted(files)
+ yield LintResult.from_cmd(autopep8(*args))
+ else:
+ # XXX `-j0` doesn't work well with `--exit-code`, so instead
+ # we capture the diff and check whether it's empty
+ # (https://github.com/hhatto/autopep8/issues/543)
+ args += ['-j0', '--diff']
+ args += sorted(files)
+ diff = autopep8.run_captured(*args)
+ if diff:
+ print(diff.decode('utf8'))
+ yield LintResult(success=False)
+ else:
+ yield LintResult(success=True)
+
+ # Run flake8 after autopep8 (the latter may have modified some files)
+ logger.info("Running Python linter (flake8)")
+
+ flake8 = Flake8()
+ if not flake8.available:
+ logger.error(
+ "Python linter requested but flake8 binary not found. "
+ f"{_archery_install_msg}")
+ return
+
+ flake8_exclude = ['.venv*']
+
+ yield LintResult.from_cmd(
+ flake8("--extend-exclude=" + ','.join(flake8_exclude),
+ setup_py, src.pyarrow, os.path.join(src.python, "examples"),
+ src.dev, check=False))
+ config = os.path.join(src.python, ".flake8.cython")
+ yield LintResult.from_cmd(
+ flake8("--config=" + config, src.pyarrow, check=False))
+
+
+def python_numpydoc(symbols=None, allow_rules=None, disallow_rules=None):
+ """Run numpydoc linter on python.
+
+ Pyarrow must be available for import.
+ """
+ logger.info("Running Python docstring linters")
+ # by default try to run on all pyarrow package
+ symbols = symbols or {
+ 'pyarrow',
+ 'pyarrow.compute',
+ 'pyarrow.csv',
+ 'pyarrow.dataset',
+ 'pyarrow.feather',
+ 'pyarrow.flight',
+ 'pyarrow.fs',
+ 'pyarrow.gandiva',
+ 'pyarrow.ipc',
+ 'pyarrow.json',
+ 'pyarrow.orc',
+ 'pyarrow.parquet',
+ 'pyarrow.plasma',
+ 'pyarrow.types',
+ }
+ try:
+ numpydoc = NumpyDoc(symbols)
+ except RuntimeError as e:
+ logger.error(str(e))
+ yield LintResult(success=False)
+ return
+
+ results = numpydoc.validate(
+ # limit the validation scope to the pyarrow package
+ from_package='pyarrow',
+ allow_rules=allow_rules,
+ disallow_rules=disallow_rules
+ )
+
+ if len(results) == 0:
+ yield LintResult(success=True)
+ return
+
+ number_of_violations = 0
+ for obj, result in results:
+ errors = result['errors']
+
+ # inspect doesn't play nice with cython generated source code,
+ # to use a hacky way to represent a proper __qualname__
+ doc = getattr(obj, '__doc__', '')
+ name = getattr(obj, '__name__', '')
+ qualname = getattr(obj, '__qualname__', '')
+ module = getattr(obj, '__module__', '')
+ instance = getattr(obj, '__self__', '')
+ if instance:
+ klass = instance.__class__.__name__
+ else:
+ klass = ''
+
+ try:
+ cython_signature = doc.splitlines()[0]
+ except Exception:
+ cython_signature = ''
+
+ desc = '.'.join(filter(None, [module, klass, qualname or name]))
+
+ click.echo()
+ click.echo(click.style(desc, bold=True, fg='yellow'))
+ if cython_signature:
+ qualname_with_signature = '.'.join([module, cython_signature])
+ click.echo(
+ click.style(
+ '-> {}'.format(qualname_with_signature),
+ fg='yellow'
+ )
+ )
+
+ for error in errors:
+ number_of_violations += 1
+ click.echo('{}: {}'.format(*error))
+
+ msg = 'Total number of docstring violations: {}'.format(
+ number_of_violations
+ )
+ click.echo()
+ click.echo(click.style(msg, fg='red'))
+
+ yield LintResult(success=False)
+
+
+def rat_linter(src, root):
+ """Run apache-rat license linter."""
+ logger.info("Running apache-rat linter")
+
+ if src.git_dirty:
+ logger.warn("Due to the usage of git-archive, uncommitted files will"
+ " not be checked for rat violations. ")
+
+ exclusion = exclusion_from_globs(
+ os.path.join(src.dev, "release", "rat_exclude_files.txt"))
+
+ # Creates a git-archive of ArrowSources, apache-rat expects a gzip
+ # compressed tar archive.
+ archive_path = os.path.join(root, "apache-arrow.tar.gz")
+ src.archive(archive_path, compressor=gzip.compress)
+ report = Rat().report(archive_path)
+
+ violations = list(report.validate(exclusion=exclusion))
+ for violation in violations:
+ print("apache-rat license violation: {}".format(violation))
+
+ yield LintResult(len(violations) == 0)
+
+
+def r_linter(src):
+ """Run R linter."""
+ logger.info("Running R linter")
+ r_lint_sh = os.path.join(src.r, "lint.sh")
+ yield LintResult.from_cmd(Bash().run(r_lint_sh, check=False))
+
+
+class Hadolint(Command):
+ def __init__(self, hadolint_bin=None):
+ self.bin = default_bin(hadolint_bin, "hadolint")
+
+
+def is_docker_image(path):
+ dirname = os.path.dirname(path)
+ filename = os.path.basename(path)
+
+ excluded = dirname.startswith(
+ "dev") or dirname.startswith("python/manylinux")
+
+ return filename.startswith("Dockerfile") and not excluded
+
+
+def docker_linter(src):
+ """Run Hadolint docker linter."""
+ logger.info("Running Docker linter")
+
+ hadolint = Hadolint()
+
+ if not hadolint.available:
+ logger.error(
+ "hadolint linter requested but hadolint binary not found.")
+ return
+
+ for path in git.ls_files(git_dir=src.path):
+ if is_docker_image(path):
+ yield LintResult.from_cmd(hadolint.run(path, check=False,
+ cwd=src.path))
+
+
+def linter(src, fix=False, *, clang_format=False, cpplint=False,
+ clang_tidy=False, iwyu=False, iwyu_all=False,
+ python=False, numpydoc=False, cmake_format=False, rat=False,
+ r=False, docker=False):
+ """Run all linters."""
+ with tmpdir(prefix="arrow-lint-") as root:
+ build_dir = os.path.join(root, "cpp-build")
+
+ # Linters yield LintResult without raising exceptions on failure.
+ # This allows running all linters in one pass and exposing all
+ # errors to the user.
+ results = []
+
+ if clang_format or cpplint or clang_tidy or iwyu:
+ results.extend(cpp_linter(src, build_dir,
+ clang_format=clang_format,
+ cpplint=cpplint,
+ clang_tidy=clang_tidy,
+ iwyu=iwyu,
+ iwyu_all=iwyu_all,
+ fix=fix))
+
+ if python:
+ results.extend(python_linter(src, fix=fix))
+
+ if numpydoc:
+ results.extend(python_numpydoc())
+
+ if cmake_format:
+ results.extend(cmake_linter(src, fix=fix))
+
+ if rat:
+ results.extend(rat_linter(src, root))
+
+ if r:
+ results.extend(r_linter(src))
+
+ if docker:
+ results.extend(docker_linter(src))
+
+ # Raise error if one linter failed, ensuring calling code can exit with
+ # non-zero.
+ for result in results:
+ result.ok()
diff --git a/src/arrow/dev/archery/archery/utils/logger.py b/src/arrow/dev/archery/archery/utils/logger.py
new file mode 100644
index 000000000..9d0feda88
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/logger.py
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+""" Global logger. """
+logger = logging.getLogger("archery")
+
+
+class LoggingContext:
+ def __init__(self, quiet=False):
+ self.quiet = quiet
+
+
+ctx = LoggingContext()
diff --git a/src/arrow/dev/archery/archery/utils/maven.py b/src/arrow/dev/archery/archery/utils/maven.py
new file mode 100644
index 000000000..96a3bf5bd
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/maven.py
@@ -0,0 +1,204 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from .command import Command, default_bin
+
+
+class Maven(Command):
+ def __init__(self, maven_bin=None):
+ self.bin = default_bin(maven_bin, "mvn")
+
+
+maven = Maven()
+
+
+class MavenDefinition:
+ """ MavenDefinition captures the maven invocation arguments.
+
+ It allows creating build directories with the same definition, e.g.
+ ```
+ build_1 = maven_def.build("/tmp/build-1")
+ build_2 = maven_def.build("/tmp/build-2")
+
+ ...
+
+ build1.install()
+ build2.install()
+ """
+
+ def __init__(self, source, build_definitions=None,
+ benchmark_definitions=None, env=None):
+ """ Initialize a MavenDefinition
+
+ Parameters
+ ----------
+ source : str
+ Source directory where the top-level pom.xml is
+ located. This is usually the root of the project.
+ build_definitions: list(str), optional
+ benchmark_definitions: list(str), optional
+ """
+ self.source = os.path.abspath(source)
+ self.build_definitions = build_definitions if build_definitions else []
+ self.benchmark_definitions =\
+ benchmark_definitions if benchmark_definitions else []
+ self.env = env
+
+ @property
+ def build_arguments(self):
+ """" Return the arguments to maven invocation for build. """
+ arguments = self.build_definitions + [
+ "-B", "-DskipTests", "-Drat.skip=true",
+ "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer."
+ "Slf4jMavenTransferListener=warn",
+ "-T", "2C", "install"
+ ]
+ return arguments
+
+ def build(self, build_dir, force=False, cmd_kwargs=None, **kwargs):
+ """ Invoke maven into a build directory.
+
+ Parameters
+ ----------
+ build_dir : str
+ Directory in which the Maven build will be instantiated.
+ force : bool
+ not used now
+ """
+ if os.path.exists(build_dir):
+ # Extra safety to ensure we're deleting a build folder.
+ if not MavenBuild.is_build_dir(build_dir):
+ raise FileExistsError(
+ "{} is not a maven build".format(build_dir)
+ )
+
+ cmd_kwargs = cmd_kwargs if cmd_kwargs else {}
+ assert MavenBuild.is_build_dir(build_dir)
+ maven(*self.build_arguments, cwd=build_dir, env=self.env, **cmd_kwargs)
+ return MavenBuild(build_dir, definition=self, **kwargs)
+
+ @property
+ def list_arguments(self):
+ """" Return the arguments to maven invocation for list """
+ arguments = [
+ "-Dskip.perf.benchmarks=false", "-Dbenchmark.list=-lp", "install"
+ ]
+ return arguments
+
+ @property
+ def benchmark_arguments(self):
+ """" Return the arguments to maven invocation for benchmark """
+ arguments = self.benchmark_definitions + [
+ "-Dskip.perf.benchmarks=false", "-Dbenchmark.fork=1",
+ "-Dbenchmark.jvmargs=\"-Darrow.enable_null_check_for_get=false "
+ "-Darrow.enable_unsafe_memory_access=true\"",
+ "install"
+ ]
+ return arguments
+
+ def __repr__(self):
+ return "MavenDefinition[source={}]".format(self.source)
+
+
+class MavenBuild(Maven):
+ """ MavenBuild represents a build directory initialized by maven.
+
+ The build instance can be used to build/test/install. It alleviates the
+ user to know which generator is used.
+ """
+
+ def __init__(self, build_dir, definition=None):
+ """ Initialize a MavenBuild.
+
+ The caller must ensure that maven was invoked in the build directory.
+
+ Parameters
+ ----------
+ definition : MavenDefinition
+ The definition to build from.
+ build_dir : str
+ The build directory to setup into.
+ """
+ assert MavenBuild.is_build_dir(build_dir)
+ super().__init__()
+ self.build_dir = os.path.abspath(build_dir)
+ self.definition = definition
+
+ @property
+ def binaries_dir(self):
+ return self.build_dir
+
+ def run(self, *argv, verbose=False, cwd=None, **kwargs):
+ extra = []
+ if verbose:
+ extra.append("-X")
+ if cwd is None:
+ cwd = self.build_dir
+ # Commands must be ran under the directory where pom.xml exists
+ return super().run(*extra, *argv, **kwargs, cwd=cwd)
+
+ def build(self, *argv, verbose=False, **kwargs):
+ definition_args = self.definition.build_arguments
+ cwd = self.binaries_dir
+ return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd,
+ env=self.definition.env, **kwargs)
+
+ def list(self, *argv, verbose=False, **kwargs):
+ definition_args = self.definition.list_arguments
+ cwd = self.binaries_dir + "/performance"
+ return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd,
+ env=self.definition.env, **kwargs)
+
+ def benchmark(self, *argv, verbose=False, **kwargs):
+ definition_args = self.definition.benchmark_arguments
+ cwd = self.binaries_dir + "/performance"
+ return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd,
+ env=self.definition.env, **kwargs)
+
+ @staticmethod
+ def is_build_dir(path):
+ """ Indicate if a path is Maven top directory.
+
+ This method only checks for the existence of paths and does not do any
+ validation whatsoever.
+ """
+ pom_xml = os.path.join(path, "pom.xml")
+ performance_dir = os.path.join(path, "performance")
+ return os.path.exists(pom_xml) and os.path.isdir(performance_dir)
+
+ @staticmethod
+ def from_path(path):
+ """ Instantiate a Maven from a path.
+
+ This is used to recover from an existing physical directory (created
+ with or without Maven).
+
+ Note that this method is not idempotent as the original definition will
+ be lost.
+ """
+ if not MavenBuild.is_build_dir(path):
+ raise ValueError("Not a valid MavenBuild path: {}".format(path))
+
+ return MavenBuild(path, definition=None)
+
+ def __repr__(self):
+ return ("MavenBuild["
+ "build = {},"
+ "definition = {}]".format(self.build_dir,
+ self.definition))
diff --git a/src/arrow/dev/archery/archery/utils/rat.py b/src/arrow/dev/archery/archery/utils/rat.py
new file mode 100644
index 000000000..e7fe19a7e
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/rat.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import re
+from xml.etree import ElementTree
+
+from ..lang.java import Jar
+from .cache import Cache
+from .command import capture_stdout
+
+RAT_VERSION = 0.13
+RAT_JAR_FILENAME = "apache-rat-{}.jar".format(RAT_VERSION)
+RAT_URL_ = "https://repo1.maven.org/maven2/org/apache/rat/apache-rat"
+RAT_URL = "/".join([RAT_URL_, str(RAT_VERSION), RAT_JAR_FILENAME])
+
+
+class Rat(Jar):
+ def __init__(self):
+ jar = Cache().get_or_insert_from_url(RAT_JAR_FILENAME, RAT_URL)
+ Jar.__init__(self, jar)
+
+ @capture_stdout(strip=False)
+ def run_report(self, archive_path, **kwargs):
+ return self.run("--xml", archive_path, **kwargs)
+
+ def report(self, archive_path, **kwargs):
+ return RatReport(self.run_report(archive_path, **kwargs))
+
+
+def exclusion_from_globs(exclusions_path):
+ with open(exclusions_path, 'r') as exclusions_fd:
+ exclusions = [e.strip() for e in exclusions_fd]
+ return lambda path: any([fnmatch.fnmatch(path, e) for e in exclusions])
+
+
+class RatReport:
+ def __init__(self, xml):
+ self.xml = xml
+ self.tree = ElementTree.fromstring(xml)
+
+ def __repr__(self):
+ return "RatReport({})".format(self.xml)
+
+ def validate(self, exclusion=None):
+ for r in self.tree.findall('resource'):
+ approvals = r.findall('license-approval')
+ if not approvals or approvals[0].attrib['name'] == 'true':
+ continue
+
+ clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+
+ if exclusion and exclusion(clean_name):
+ continue
+
+ yield clean_name
diff --git a/src/arrow/dev/archery/archery/utils/report.py b/src/arrow/dev/archery/archery/utils/report.py
new file mode 100644
index 000000000..6c7587ddd
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/report.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABCMeta, abstractmethod
+import datetime
+
+import jinja2
+
+
+def markdown_escape(s):
+ for char in ('*', '#', '_', '~', '`', '>'):
+ s = s.replace(char, '\\' + char)
+ return s
+
+
+class Report(metaclass=ABCMeta):
+
+ def __init__(self, **kwargs):
+ for field in self.fields:
+ if field not in kwargs:
+ raise ValueError('Missing keyword argument {}'.format(field))
+ self._data = kwargs
+
+ def __getattr__(self, key):
+ return self._data[key]
+
+ @abstractmethod
+ def fields(self):
+ pass
+
+ @property
+ @abstractmethod
+ def templates(self):
+ pass
+
+
+class JinjaReport(Report):
+
+ def __init__(self, **kwargs):
+ self.env = jinja2.Environment(
+ loader=jinja2.PackageLoader('archery', 'templates')
+ )
+ self.env.filters['md'] = markdown_escape
+ self.env.globals['today'] = datetime.date.today
+ super().__init__(**kwargs)
+
+ def render(self, template_name):
+ template_path = self.templates[template_name]
+ template = self.env.get_template(template_path)
+ return template.render(**self._data)
diff --git a/src/arrow/dev/archery/archery/utils/source.py b/src/arrow/dev/archery/archery/utils/source.py
new file mode 100644
index 000000000..1080cb75d
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/source.py
@@ -0,0 +1,211 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from pathlib import Path
+import subprocess
+
+from .git import git
+
+
+class InvalidArrowSource(Exception):
+ pass
+
+
+class ArrowSources:
+ """ ArrowSources is a companion class representing a directory containing
+ Apache Arrow's sources.
+ """
+ # Note that WORKSPACE is a reserved git revision name by this module to
+ # reference the current git workspace. In other words, this indicates to
+ # ArrowSources.at_revision that no cloning/checkout is required.
+ WORKSPACE = "WORKSPACE"
+
+ def __init__(self, path):
+ """ Initialize an ArrowSources
+
+ The caller must ensure that path is valid arrow source directory (can
+ be checked with ArrowSources.valid)
+
+ Parameters
+ ----------
+ path : src
+ """
+ path = Path(path)
+ # validate by checking a specific path in the arrow source tree
+ if not (path / 'cpp' / 'CMakeLists.txt').exists():
+ raise InvalidArrowSource(
+ "No Arrow C++ sources found in {}.".format(path)
+ )
+ self.path = path
+
+ @property
+ def archery(self):
+ """ Returns the archery directory of an Arrow sources. """
+ return self.dev / "archery"
+
+ @property
+ def cpp(self):
+ """ Returns the cpp directory of an Arrow sources. """
+ return self.path / "cpp"
+
+ @property
+ def dev(self):
+ """ Returns the dev directory of an Arrow sources. """
+ return self.path / "dev"
+
+ @property
+ def java(self):
+ """ Returns the java directory of an Arrow sources. """
+ return self.path / "java"
+
+ @property
+ def python(self):
+ """ Returns the python directory of an Arrow sources. """
+ return self.path / "python"
+
+ @property
+ def pyarrow(self):
+ """ Returns the python/pyarrow directory of an Arrow sources. """
+ return self.python / "pyarrow"
+
+ @property
+ def r(self):
+ """ Returns the r directory of an Arrow sources. """
+ return self.path / "r"
+
+ @property
+ def git_backed(self):
+ """ Indicate if the sources are backed by git. """
+ return (self.path / ".git").exists()
+
+ @property
+ def git_dirty(self):
+ """ Indicate if the sources is a dirty git directory. """
+ return self.git_backed and git.dirty(git_dir=self.path)
+
+ def archive(self, path, dereference=False, compressor=None, revision=None):
+ """ Saves a git archive at path. """
+ if not self.git_backed:
+ raise ValueError("{} is not backed by git".format(self))
+
+ rev = revision if revision else "HEAD"
+ archive = git.archive("--prefix=apache-arrow/", rev,
+ git_dir=self.path)
+
+ # TODO(fsaintjacques): fix dereference for
+
+ if compressor:
+ archive = compressor(archive)
+
+ with open(path, "wb") as archive_fd:
+ archive_fd.write(archive)
+
+ def at_revision(self, revision, clone_dir):
+ """ Return a copy of the current sources for a specified git revision.
+
+ This method may return the current object if no checkout is required.
+ The caller is responsible to remove the cloned repository directory.
+
+ The user can use the special WORKSPACE token to mean the current git
+ workspace (no checkout performed).
+
+ The second value of the returned tuple indicates if a clone was
+ performed.
+
+ Parameters
+ ----------
+ revision : str
+ Revision to checkout sources at.
+ clone_dir : str
+ Path to checkout the local clone.
+ """
+ if not self.git_backed:
+ raise ValueError("{} is not backed by git".format(self))
+
+ if revision == ArrowSources.WORKSPACE:
+ return self, False
+
+ # A local clone is required to leave the current sources intact such
+ # that builds depending on said sources are not invalidated (or worse
+ # slightly affected when re-invoking the generator).
+ # "--local" only works when dest dir is on same volume of source dir.
+ # "--shared" works even if dest dir is on different volume.
+ git.clone("--shared", self.path, clone_dir)
+
+ # Revision can reference "origin/" (or any remotes) that are not found
+ # in the local clone. Thus, revisions are dereferenced in the source
+ # repository.
+ original_revision = git.rev_parse(revision)
+
+ git.checkout(original_revision, git_dir=clone_dir)
+
+ return ArrowSources(clone_dir), True
+
+ @staticmethod
+ def find(path=None):
+ """ Infer Arrow sources directory from various method.
+
+ The following guesses are done in order until a valid match is found:
+
+ 1. Checks the given optional parameter.
+
+ 2. Checks if the environment variable `ARROW_SRC` is defined and use
+ this.
+
+ 3. Checks if the current working directory (cwd) is an Arrow source
+ directory.
+
+ 4. Checks if this file (cli.py) is still in the original source
+ repository. If so, returns the relative path to the source
+ directory.
+ """
+
+ # Explicit via environment
+ env = os.environ.get("ARROW_SRC")
+
+ # Implicit via cwd
+ cwd = Path.cwd()
+
+ # Implicit via current file
+ try:
+ this = Path(__file__).parents[4]
+ except IndexError:
+ this = None
+
+ # Implicit via git repository (if archery is installed system wide)
+ try:
+ repo = git.repository_root(git_dir=cwd)
+ except subprocess.CalledProcessError:
+ # We're not inside a git repository.
+ repo = None
+
+ paths = list(filter(None, [path, env, cwd, this, repo]))
+ for p in paths:
+ try:
+ return ArrowSources(p)
+ except InvalidArrowSource:
+ pass
+
+ searched_paths = "\n".join([" - {}".format(p) for p in paths])
+ raise InvalidArrowSource(
+ "Unable to locate Arrow's source directory. "
+ "Searched paths are:\n{}".format(searched_paths)
+ )
+
+ def __repr__(self):
+ return self.path
diff --git a/src/arrow/dev/archery/archery/utils/tmpdir.py b/src/arrow/dev/archery/archery/utils/tmpdir.py
new file mode 100644
index 000000000..07d7355c8
--- /dev/null
+++ b/src/arrow/dev/archery/archery/utils/tmpdir.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from contextlib import contextmanager
+from tempfile import mkdtemp, TemporaryDirectory
+
+
+@contextmanager
+def tmpdir(preserve=False, prefix="arrow-archery-"):
+ if preserve:
+ yield mkdtemp(prefix=prefix)
+ else:
+ with TemporaryDirectory(prefix=prefix) as tmp:
+ yield tmp
diff --git a/src/arrow/dev/archery/conftest.py b/src/arrow/dev/archery/conftest.py
new file mode 100644
index 000000000..06a643bea
--- /dev/null
+++ b/src/arrow/dev/archery/conftest.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pathlib
+
+import pytest
+
+
+def pytest_addoption(parser):
+ parser.addoption(
+ "--enable-integration",
+ action="store_true",
+ default=False,
+ help="run slow tests"
+ )
+
+
+def pytest_configure(config):
+ config.addinivalue_line(
+ "markers",
+ (
+ "integration: mark test as integration tests involving more "
+ "extensive setup (only used for crossbow at the moment)"
+ )
+ )
+
+
+def pytest_collection_modifyitems(config, items):
+ if config.getoption("--enable-integration"):
+ return
+ marker = pytest.mark.skip(reason="need --enable-integration option to run")
+ for item in items:
+ if "integration" in item.keywords:
+ item.add_marker(marker)
+
+
+@pytest.fixture
+def load_fixture(request):
+ current_test_directory = pathlib.Path(request.node.fspath).parent
+
+ def decoder(path):
+ with path.open('r') as fp:
+ if path.suffix == '.json':
+ import json
+ return json.load(fp)
+ elif path.suffix == '.yaml':
+ import yaml
+ return yaml.load(fp)
+ else:
+ return fp.read()
+
+ def loader(name, decoder=decoder):
+ path = current_test_directory / 'fixtures' / name
+ return decoder(path)
+
+ return loader
diff --git a/src/arrow/dev/archery/generate_files_for_endian_test.sh b/src/arrow/dev/archery/generate_files_for_endian_test.sh
new file mode 100755
index 000000000..ba3ce9f16
--- /dev/null
+++ b/src/arrow/dev/archery/generate_files_for_endian_test.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script generates json and arrow files of each type (e.g. primitive) for integration endian test
+# Usage: generate_files_for_endian_test.sh
+# ARROW_CPP_EXE_PATH : where Arrow C++ binaries can be found
+# TMP_DIR : where files will be generated
+
+set -e
+
+: ${ARROW_CPP_EXE_PATH:=/arrow/cpp/build/debug/}
+: ${TMP_DIR:=/tmp/arrow}
+
+json_dir=$TMP_DIR/arrow.$$
+mkdir -p $json_dir
+
+archery integration --stop-on-error --with-cpp=1 --tempdir=$json_dir
+
+for f in $json_dir/*.json ; do
+ $ARROW_CPP_EXE_PATH/arrow-json-integration-test -mode JSON_TO_ARROW -json $f -arrow ${f%.*}.arrow_file -integration true ;
+done
+for f in $json_dir/*.arrow_file ; do
+ $ARROW_CPP_EXE_PATH/arrow-file-to-stream $f > ${f%.*}.stream;
+done
+for f in $json_dir/*.json ; do
+ gzip $f ;
+done
+echo "The files are under $json_dir"
diff --git a/src/arrow/dev/archery/requirements.txt b/src/arrow/dev/archery/requirements.txt
new file mode 100644
index 000000000..0e1258adb
--- /dev/null
+++ b/src/arrow/dev/archery/requirements.txt
@@ -0,0 +1,4 @@
+click
+pygithub
+python-dotenv
+ruamel.yaml
diff --git a/src/arrow/dev/archery/setup.py b/src/arrow/dev/archery/setup.py
new file mode 100755
index 000000000..664807375
--- /dev/null
+++ b/src/arrow/dev/archery/setup.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import functools
+import operator
+import sys
+from setuptools import setup, find_packages
+
+if sys.version_info < (3, 6):
+ sys.exit('Python < 3.6 is not supported')
+
+# For pathlib.Path compatibility
+jinja_req = 'jinja2>=2.11'
+
+extras = {
+ 'lint': ['numpydoc==1.1.0', 'autopep8', 'flake8', 'cmake_format==0.6.13'],
+ 'benchmark': ['pandas'],
+ 'docker': ['ruamel.yaml', 'python-dotenv'],
+ 'release': [jinja_req, 'jira', 'semver', 'gitpython'],
+ 'crossbow': ['github3.py', jinja_req, 'pygit2>=1.6.0', 'ruamel.yaml',
+ 'setuptools_scm'],
+ 'crossbow-upload': ['github3.py', jinja_req, 'ruamel.yaml',
+ 'setuptools_scm'],
+}
+extras['bot'] = extras['crossbow'] + ['pygithub', 'jira']
+extras['all'] = list(set(functools.reduce(operator.add, extras.values())))
+
+setup(
+ name='archery',
+ version="0.1.0",
+ description='Apache Arrow Developers Tools',
+ url='http://github.com/apache/arrow',
+ maintainer='Arrow Developers',
+ maintainer_email='dev@arrow.apache.org',
+ packages=find_packages(),
+ include_package_data=True,
+ install_requires=['click>=7'],
+ tests_require=['pytest', 'responses'],
+ extras_require=extras,
+ entry_points='''
+ [console_scripts]
+ archery=archery.cli:archery
+ '''
+)