diff options
Diffstat (limited to '')
87 files changed, 16009 insertions, 0 deletions
diff --git a/src/arrow/dev/archery/MANIFEST.in b/src/arrow/dev/archery/MANIFEST.in new file mode 100644 index 000000000..90fe034c2 --- /dev/null +++ b/src/arrow/dev/archery/MANIFEST.in @@ -0,0 +1,4 @@ +include ../../LICENSE.txt +include ../../NOTICE.txt + +include archery/reports/* diff --git a/src/arrow/dev/archery/README.md b/src/arrow/dev/archery/README.md new file mode 100644 index 000000000..eff654416 --- /dev/null +++ b/src/arrow/dev/archery/README.md @@ -0,0 +1,49 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +# Developing with Archery + +Archery is documented on the Arrow website: + +* [Daily development using Archery](https://arrow.apache.org/docs/developers/archery.html) +* [Using Archery and Crossbow](https://arrow.apache.org/docs/developers/crossbow.html) +* [Using Archer and Docker](https://arrow.apache.org/docs/developers/docker.html) + +# Installing Archery + +See the pages linked aboved for more details. As a general overview, Archery +comes in a number of subpackages, each needing to be installed if you want +to use the functionality of it: + +* lint – lint (and in some cases auto-format) code in the Arrow repo + To install: `pip install -e "arrow/dev/archery[lint]"` +* benchmark – to run Arrow benchmarks using Archery + To install: `pip install -e "arrow/dev/archery[benchmark]"` +* docker – to run docker-compose based tasks more easily + To install: `pip install -e "arrow/dev/archery[docker]"` +* release – release related helpers + To install: `pip install -e "arrow/dev/archery[release]"` +* crossbow – to trigger + interact with the crossbow build system + To install: `pip install -e "arrow/dev/archery[crossbow]"` +* crossbow-upload + To install: `pip install -e "arrow/dev/archery[crossbow-upload]"` + +Additionally, if you would prefer to install everything at once, +`pip install -e "arrow/dev/archery[all]"` is an alias for all of +the above subpackages.
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/__init__.py b/src/arrow/dev/archery/archery/__init__.py new file mode 100644 index 000000000..13a83393a --- /dev/null +++ b/src/arrow/dev/archery/archery/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/arrow/dev/archery/archery/benchmark/__init__.py b/src/arrow/dev/archery/archery/benchmark/__init__.py new file mode 100644 index 000000000..13a83393a --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/arrow/dev/archery/archery/benchmark/codec.py b/src/arrow/dev/archery/archery/benchmark/codec.py new file mode 100644 index 000000000..4157890d1 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/codec.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import json + +from ..benchmark.core import Benchmark, BenchmarkSuite +from ..benchmark.runner import BenchmarkRunner, StaticBenchmarkRunner +from ..benchmark.compare import BenchmarkComparator + + +class JsonEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, Benchmark): + return BenchmarkCodec.encode(o) + + if isinstance(o, BenchmarkSuite): + return BenchmarkSuiteCodec.encode(o) + + if isinstance(o, BenchmarkRunner): + return BenchmarkRunnerCodec.encode(o) + + if isinstance(o, BenchmarkComparator): + return BenchmarkComparatorCodec.encode(o) + + return json.JSONEncoder.default(self, o) + + +class BenchmarkCodec: + @staticmethod + def encode(b): + return { + "name": b.name, + "unit": b.unit, + "less_is_better": b.less_is_better, + "values": b.values, + "time_unit": b.time_unit, + "times": b.times, + "counters": b.counters, + } + + @staticmethod + def decode(dct, **kwargs): + return Benchmark(**dct, **kwargs) + + +class BenchmarkSuiteCodec: + @staticmethod + def encode(bs): + return { + "name": bs.name, + "benchmarks": [BenchmarkCodec.encode(b) for b in bs.benchmarks] + } + + @staticmethod + def decode(dct, **kwargs): + benchmarks = [BenchmarkCodec.decode(b) + for b in dct.pop("benchmarks", [])] + return BenchmarkSuite(benchmarks=benchmarks, **dct, **kwargs) + + +class BenchmarkRunnerCodec: + @staticmethod + def encode(br): + return {"suites": [BenchmarkSuiteCodec.encode(s) for s in br.suites]} + + @staticmethod + def decode(dct, **kwargs): + suites = [BenchmarkSuiteCodec.decode(s) + for s in dct.pop("suites", [])] + return StaticBenchmarkRunner(suites=suites, **dct, **kwargs) + + +class BenchmarkComparatorCodec: + @staticmethod + def encode(bc): + comparator = bc.formatted + + suite_name = bc.suite_name + if suite_name: + comparator["suite"] = suite_name + + return comparator diff --git a/src/arrow/dev/archery/archery/benchmark/compare.py b/src/arrow/dev/archery/archery/benchmark/compare.py new file mode 100644 index 000000000..622b80179 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/compare.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# Define a global regression threshold as 5%. This is purely subjective and +# flawed. This does not track cumulative regression. +DEFAULT_THRESHOLD = 0.05 + + +def items_per_seconds_fmt(value): + if value < 1000: + return "{} items/sec".format(value) + if value < 1000**2: + return "{:.3f}K items/sec".format(value / 1000) + if value < 1000**3: + return "{:.3f}M items/sec".format(value / 1000**2) + else: + return "{:.3f}G items/sec".format(value / 1000**3) + + +def bytes_per_seconds_fmt(value): + if value < 1024: + return "{} bytes/sec".format(value) + if value < 1024**2: + return "{:.3f} KiB/sec".format(value / 1024) + if value < 1024**3: + return "{:.3f} MiB/sec".format(value / 1024**2) + if value < 1024**4: + return "{:.3f} GiB/sec".format(value / 1024**3) + else: + return "{:.3f} TiB/sec".format(value / 1024**4) + + +def change_fmt(value): + return "{:.3%}".format(value) + + +def formatter_for_unit(unit): + if unit == "bytes_per_second": + return bytes_per_seconds_fmt + elif unit == "items_per_second": + return items_per_seconds_fmt + else: + return lambda x: x + + +class BenchmarkComparator: + """ Compares two benchmarks. + + Encodes the logic of comparing two benchmarks and taking a decision on + if it induce a regression. + """ + + def __init__(self, contender, baseline, threshold=DEFAULT_THRESHOLD, + suite_name=None): + self.contender = contender + self.baseline = baseline + self.threshold = threshold + self.suite_name = suite_name + + @property + def name(self): + return self.baseline.name + + @property + def less_is_better(self): + return self.baseline.less_is_better + + @property + def unit(self): + return self.baseline.unit + + @property + def change(self): + new = self.contender.value + old = self.baseline.value + + if old == 0 and new == 0: + return 0.0 + if old == 0: + return 0.0 + + return float(new - old) / abs(old) + + @property + def confidence(self): + """ Indicate if a comparison of benchmarks should be trusted. """ + return True + + @property + def regression(self): + change = self.change + adjusted_change = change if self.less_is_better else -change + return (self.confidence and adjusted_change > self.threshold) + + @property + def formatted(self): + fmt = formatter_for_unit(self.unit) + return { + "benchmark": self.name, + "change": change_fmt(self.change), + "regression": self.regression, + "baseline": fmt(self.baseline.value), + "contender": fmt(self.contender.value), + "unit": self.unit, + "less_is_better": self.less_is_better, + "counters": str(self.baseline.counters) + } + + def compare(self, comparator=None): + return { + "benchmark": self.name, + "change": self.change, + "regression": self.regression, + "baseline": self.baseline.value, + "contender": self.contender.value, + "unit": self.unit, + "less_is_better": self.less_is_better, + "counters": self.baseline.counters + } + + def __call__(self, **kwargs): + return self.compare(**kwargs) + + +def pairwise_compare(contender, baseline): + dict_contender = {e.name: e for e in contender} + dict_baseline = {e.name: e for e in baseline} + + for name in (dict_contender.keys() & dict_baseline.keys()): + yield name, (dict_contender[name], dict_baseline[name]) + + +class RunnerComparator: + """ Compares suites/benchmarks from runners. + + It is up to the caller that ensure that runners are compatible (both from + the same language implementation). + """ + + def __init__(self, contender, baseline, threshold=DEFAULT_THRESHOLD): + self.contender = contender + self.baseline = baseline + self.threshold = threshold + + @property + def comparisons(self): + contender = self.contender.suites + baseline = self.baseline.suites + suites = pairwise_compare(contender, baseline) + + for suite_name, (suite_cont, suite_base) in suites: + benchmarks = pairwise_compare( + suite_cont.benchmarks, suite_base.benchmarks) + + for _, (bench_cont, bench_base) in benchmarks: + yield BenchmarkComparator(bench_cont, bench_base, + threshold=self.threshold, + suite_name=suite_name) diff --git a/src/arrow/dev/archery/archery/benchmark/core.py b/src/arrow/dev/archery/archery/benchmark/core.py new file mode 100644 index 000000000..5a92271a3 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/core.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def median(values): + n = len(values) + if n == 0: + raise ValueError("median requires at least one value") + elif n % 2 == 0: + return (values[(n // 2) - 1] + values[n // 2]) / 2 + else: + return values[n // 2] + + +class Benchmark: + def __init__(self, name, unit, less_is_better, values, time_unit, + times, counters=None): + self.name = name + self.unit = unit + self.less_is_better = less_is_better + self.values = sorted(values) + self.time_unit = time_unit + self.times = sorted(times) + self.median = median(self.values) + self.counters = counters or {} + + @property + def value(self): + return self.median + + def __repr__(self): + return "Benchmark[name={},value={}]".format(self.name, self.value) + + +class BenchmarkSuite: + def __init__(self, name, benchmarks): + self.name = name + self.benchmarks = benchmarks + + def __repr__(self): + return "BenchmarkSuite[name={}, benchmarks={}]".format( + self.name, self.benchmarks + ) diff --git a/src/arrow/dev/archery/archery/benchmark/google.py b/src/arrow/dev/archery/archery/benchmark/google.py new file mode 100644 index 000000000..ebcc52636 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/google.py @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from itertools import filterfalse, groupby, tee +import json +import subprocess +from tempfile import NamedTemporaryFile + +from .core import Benchmark +from ..utils.command import Command + + +def partition(pred, iterable): + # adapted from python's examples + t1, t2 = tee(iterable) + return list(filter(pred, t1)), list(filterfalse(pred, t2)) + + +class GoogleBenchmarkCommand(Command): + """ Run a google benchmark binary. + + This assumes the binary supports the standard command line options, + notably `--benchmark_filter`, `--benchmark_format`, etc... + """ + + def __init__(self, benchmark_bin, benchmark_filter=None): + self.bin = benchmark_bin + self.benchmark_filter = benchmark_filter + + def list_benchmarks(self): + argv = ["--benchmark_list_tests"] + if self.benchmark_filter: + argv.append("--benchmark_filter={}".format(self.benchmark_filter)) + result = self.run(*argv, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return str.splitlines(result.stdout.decode("utf-8")) + + def results(self, repetitions=1): + with NamedTemporaryFile() as out: + argv = ["--benchmark_repetitions={}".format(repetitions), + "--benchmark_out={}".format(out.name), + "--benchmark_out_format=json"] + + if self.benchmark_filter: + argv.append( + "--benchmark_filter={}".format(self.benchmark_filter) + ) + + self.run(*argv, check=True) + return json.load(out) + + +class GoogleBenchmarkObservation: + """ Represents one run of a single (google c++) benchmark. + + Aggregates are reported by Google Benchmark executables alongside + other observations whenever repetitions are specified (with + `--benchmark_repetitions` on the bare benchmark, or with the + archery option `--repetitions`). Aggregate observations are not + included in `GoogleBenchmark.runs`. + + RegressionSumKernel/32768/0 1 us 1 us 25.8077GB/s + RegressionSumKernel/32768/0 1 us 1 us 25.7066GB/s + RegressionSumKernel/32768/0 1 us 1 us 25.1481GB/s + RegressionSumKernel/32768/0 1 us 1 us 25.846GB/s + RegressionSumKernel/32768/0 1 us 1 us 25.6453GB/s + RegressionSumKernel/32768/0_mean 1 us 1 us 25.6307GB/s + RegressionSumKernel/32768/0_median 1 us 1 us 25.7066GB/s + RegressionSumKernel/32768/0_stddev 0 us 0 us 288.046MB/s + """ + + def __init__(self, name, real_time, cpu_time, time_unit, run_type, + size=None, bytes_per_second=None, items_per_second=None, + **counters): + self._name = name + self.real_time = real_time + self.cpu_time = cpu_time + self.time_unit = time_unit + self.run_type = run_type + self.size = size + self.bytes_per_second = bytes_per_second + self.items_per_second = items_per_second + self.counters = counters + + @property + def is_aggregate(self): + """ Indicate if the observation is a run or an aggregate. """ + return self.run_type == "aggregate" + + @property + def is_realtime(self): + """ Indicate if the preferred value is realtime instead of cputime. """ + return self.name.find("/real_time") != -1 + + @property + def name(self): + name = self._name + return name.rsplit("_", maxsplit=1)[0] if self.is_aggregate else name + + @property + def time(self): + return self.real_time if self.is_realtime else self.cpu_time + + @property + def value(self): + """ Return the benchmark value.""" + return self.bytes_per_second or self.items_per_second or self.time + + @property + def unit(self): + if self.bytes_per_second: + return "bytes_per_second" + elif self.items_per_second: + return "items_per_second" + else: + return self.time_unit + + def __repr__(self): + return str(self.value) + + +class GoogleBenchmark(Benchmark): + """ A set of GoogleBenchmarkObservations. """ + + def __init__(self, name, runs): + """ Initialize a GoogleBenchmark. + + Parameters + ---------- + name: str + Name of the benchmark + runs: list(GoogleBenchmarkObservation) + Repetitions of GoogleBenchmarkObservation run. + + """ + self.name = name + # exclude google benchmark aggregate artifacts + _, runs = partition(lambda b: b.is_aggregate, runs) + self.runs = sorted(runs, key=lambda b: b.value) + unit = self.runs[0].unit + time_unit = self.runs[0].time_unit + less_is_better = not unit.endswith("per_second") + values = [b.value for b in self.runs] + times = [b.real_time for b in self.runs] + # Slight kludge to extract the UserCounters for each benchmark + counters = self.runs[0].counters + super().__init__(name, unit, less_is_better, values, time_unit, times, + counters) + + def __repr__(self): + return "GoogleBenchmark[name={},runs={}]".format(self.names, self.runs) + + @classmethod + def from_json(cls, payload): + def group_key(x): + return x.name + + benchmarks = map(lambda x: GoogleBenchmarkObservation(**x), payload) + groups = groupby(sorted(benchmarks, key=group_key), group_key) + return [cls(k, list(bs)) for k, bs in groups] diff --git a/src/arrow/dev/archery/archery/benchmark/jmh.py b/src/arrow/dev/archery/archery/benchmark/jmh.py new file mode 100644 index 000000000..f531b6de1 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/jmh.py @@ -0,0 +1,201 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from itertools import filterfalse, groupby, tee +import json +import subprocess +from tempfile import NamedTemporaryFile + +from .core import Benchmark +from ..utils.command import Command +from ..utils.maven import Maven + + +def partition(pred, iterable): + # adapted from python's examples + t1, t2 = tee(iterable) + return list(filter(pred, t1)), list(filterfalse(pred, t2)) + + +class JavaMicrobenchmarkHarnessCommand(Command): + """ Run a Java Micro Benchmark Harness + + This assumes the binary supports the standard command line options, + notably `-Dbenchmark_filter` + """ + + def __init__(self, build, benchmark_filter=None): + self.benchmark_filter = benchmark_filter + self.build = build + self.maven = Maven() + + """ Extract benchmark names from output between "Benchmarks:" and "[INFO]". + Assume the following output: + ... + Benchmarks: + org.apache.arrow.vector.IntBenchmarks.setIntDirectly + ... + org.apache.arrow.vector.IntBenchmarks.setWithValueHolder + org.apache.arrow.vector.IntBenchmarks.setWithWriter + ... + [INFO] + """ + + def list_benchmarks(self): + argv = [] + if self.benchmark_filter: + argv.append("-Dbenchmark.filter={}".format(self.benchmark_filter)) + result = self.build.list( + *argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + lists = [] + benchmarks = False + for line in str.splitlines(result.stdout.decode("utf-8")): + if not benchmarks: + if line.startswith("Benchmarks:"): + benchmarks = True + else: + if line.startswith("org.apache.arrow"): + lists.append(line) + if line.startswith("[INFO]"): + break + return lists + + def results(self, repetitions): + with NamedTemporaryFile(suffix=".json") as out: + argv = ["-Dbenchmark.runs={}".format(repetitions), + "-Dbenchmark.resultfile={}".format(out.name), + "-Dbenchmark.resultformat=json"] + if self.benchmark_filter: + argv.append( + "-Dbenchmark.filter={}".format(self.benchmark_filter) + ) + + self.build.benchmark(*argv, check=True) + return json.load(out) + + +class JavaMicrobenchmarkHarnessObservation: + """ Represents one run of a single Java Microbenchmark Harness + """ + + def __init__(self, benchmark, primaryMetric, + forks, warmupIterations, measurementIterations, **counters): + self.name = benchmark + self.primaryMetric = primaryMetric + self.score = primaryMetric["score"] + self.score_unit = primaryMetric["scoreUnit"] + self.forks = forks + self.warmups = warmupIterations + self.runs = measurementIterations + self.counters = { + "mode": counters["mode"], + "threads": counters["threads"], + "warmups": warmupIterations, + "warmupTime": counters["warmupTime"], + "measurements": measurementIterations, + "measurementTime": counters["measurementTime"], + "jvmArgs": counters["jvmArgs"] + } + self.reciprocal_value = True if self.score_unit.endswith( + "/op") else False + if self.score_unit.startswith("ops/"): + idx = self.score_unit.find("/") + self.normalizePerSec(self.score_unit[idx+1:]) + elif self.score_unit.endswith("/op"): + idx = self.score_unit.find("/") + self.normalizePerSec(self.score_unit[:idx]) + else: + self.normalizeFactor = 1 + + @property + def value(self): + """ Return the benchmark value.""" + val = 1 / self.score if self.reciprocal_value else self.score + return val * self.normalizeFactor + + def normalizePerSec(self, unit): + if unit == "ns": + self.normalizeFactor = 1000 * 1000 * 1000 + elif unit == "us": + self.normalizeFactor = 1000 * 1000 + elif unit == "ms": + self.normalizeFactor = 1000 + elif unit == "min": + self.normalizeFactor = 1 / 60 + elif unit == "hr": + self.normalizeFactor = 1 / (60 * 60) + elif unit == "day": + self.normalizeFactor = 1 / (60 * 60 * 24) + else: + self.normalizeFactor = 1 + + @property + def unit(self): + if self.score_unit.startswith("ops/"): + return "items_per_second" + elif self.score_unit.endswith("/op"): + return "items_per_second" + else: + return "?" + + def __repr__(self): + return str(self.value) + + +class JavaMicrobenchmarkHarness(Benchmark): + """ A set of JavaMicrobenchmarkHarnessObservations. """ + + def __init__(self, name, runs): + """ Initialize a JavaMicrobenchmarkHarness. + + Parameters + ---------- + name: str + Name of the benchmark + forks: int + warmups: int + runs: int + runs: list(JavaMicrobenchmarkHarnessObservation) + Repetitions of JavaMicrobenchmarkHarnessObservation run. + + """ + self.name = name + self.runs = sorted(runs, key=lambda b: b.value) + unit = self.runs[0].unit + time_unit = "N/A" + less_is_better = not unit.endswith("per_second") + values = [b.value for b in self.runs] + times = [] + # Slight kludge to extract the UserCounters for each benchmark + counters = self.runs[0].counters + super().__init__(name, unit, less_is_better, values, time_unit, times, + counters) + + def __repr__(self): + return "JavaMicrobenchmark[name={},runs={}]".format( + self.name, self.runs) + + @classmethod + def from_json(cls, payload): + def group_key(x): + return x.name + + benchmarks = map( + lambda x: JavaMicrobenchmarkHarnessObservation(**x), payload) + groups = groupby(sorted(benchmarks, key=group_key), group_key) + return [cls(k, list(bs)) for k, bs in groups] diff --git a/src/arrow/dev/archery/archery/benchmark/runner.py b/src/arrow/dev/archery/archery/benchmark/runner.py new file mode 100644 index 000000000..fc6d354b1 --- /dev/null +++ b/src/arrow/dev/archery/archery/benchmark/runner.py @@ -0,0 +1,313 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import glob +import json +import os +import re + +from .core import BenchmarkSuite +from .google import GoogleBenchmarkCommand, GoogleBenchmark +from .jmh import JavaMicrobenchmarkHarnessCommand, JavaMicrobenchmarkHarness +from ..lang.cpp import CppCMakeDefinition, CppConfiguration +from ..lang.java import JavaMavenDefinition, JavaConfiguration +from ..utils.cmake import CMakeBuild +from ..utils.maven import MavenBuild +from ..utils.logger import logger + + +def regex_filter(re_expr): + if re_expr is None: + return lambda s: True + re_comp = re.compile(re_expr) + return lambda s: re_comp.search(s) + + +DEFAULT_REPETITIONS = 1 + + +class BenchmarkRunner: + def __init__(self, suite_filter=None, benchmark_filter=None, + repetitions=DEFAULT_REPETITIONS): + self.suite_filter = suite_filter + self.benchmark_filter = benchmark_filter + self.repetitions = repetitions + + @property + def suites(self): + raise NotImplementedError("BenchmarkRunner must implement suites") + + @staticmethod + def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs): + raise NotImplementedError( + "BenchmarkRunner must implement from_rev_or_path") + + +class StaticBenchmarkRunner(BenchmarkRunner): + """ Run suites from a (static) set of suites. """ + + def __init__(self, suites, **kwargs): + self._suites = suites + super().__init__(**kwargs) + + @property + def list_benchmarks(self): + for suite in self._suites: + for benchmark in suite.benchmarks: + yield "{}.{}".format(suite.name, benchmark.name) + + @property + def suites(self): + suite_fn = regex_filter(self.suite_filter) + benchmark_fn = regex_filter(self.benchmark_filter) + + for suite in (s for s in self._suites if suite_fn(s.name)): + benchmarks = [b for b in suite.benchmarks if benchmark_fn(b.name)] + yield BenchmarkSuite(suite.name, benchmarks) + + @classmethod + def is_json_result(cls, path_or_str): + builder = None + try: + builder = cls.from_json(path_or_str) + except BaseException: + pass + + return builder is not None + + @staticmethod + def from_json(path_or_str, **kwargs): + # .codec imported here to break recursive imports + from .codec import BenchmarkRunnerCodec + if os.path.isfile(path_or_str): + with open(path_or_str) as f: + loaded = json.load(f) + else: + loaded = json.loads(path_or_str) + return BenchmarkRunnerCodec.decode(loaded, **kwargs) + + def __repr__(self): + return "BenchmarkRunner[suites={}]".format(list(self.suites)) + + +class CppBenchmarkRunner(BenchmarkRunner): + """ Run suites from a CMakeBuild. """ + + def __init__(self, build, **kwargs): + """ Initialize a CppBenchmarkRunner. """ + self.build = build + super().__init__(**kwargs) + + @staticmethod + def default_configuration(**kwargs): + """ Returns the default benchmark configuration. """ + return CppConfiguration( + build_type="release", with_tests=False, with_benchmarks=True, + with_compute=True, + with_csv=True, + with_dataset=True, + with_json=True, + with_parquet=True, + with_python=False, + with_brotli=True, + with_bz2=True, + with_lz4=True, + with_snappy=True, + with_zlib=True, + with_zstd=True, + **kwargs) + + @property + def suites_binaries(self): + """ Returns a list of benchmark binaries for this build. """ + # Ensure build is up-to-date to run benchmarks + self.build() + # Not the best method, but works for now + glob_expr = os.path.join(self.build.binaries_dir, "*-benchmark") + return {os.path.basename(b): b for b in glob.glob(glob_expr)} + + def suite(self, name, suite_bin): + """ Returns the resulting benchmarks for a given suite. """ + suite_cmd = GoogleBenchmarkCommand(suite_bin, self.benchmark_filter) + + # Ensure there will be data + benchmark_names = suite_cmd.list_benchmarks() + if not benchmark_names: + return None + + results = suite_cmd.results(repetitions=self.repetitions) + benchmarks = GoogleBenchmark.from_json(results.get("benchmarks")) + return BenchmarkSuite(name, benchmarks) + + @property + def list_benchmarks(self): + for suite_name, suite_bin in self.suites_binaries.items(): + suite_cmd = GoogleBenchmarkCommand(suite_bin) + for benchmark_name in suite_cmd.list_benchmarks(): + yield "{}.{}".format(suite_name, benchmark_name) + + @property + def suites(self): + """ Returns all suite for a runner. """ + suite_matcher = regex_filter(self.suite_filter) + + suite_and_binaries = self.suites_binaries + for suite_name in suite_and_binaries: + if not suite_matcher(suite_name): + logger.debug("Ignoring suite {}".format(suite_name)) + continue + + suite_bin = suite_and_binaries[suite_name] + suite = self.suite(suite_name, suite_bin) + + # Filter may exclude all benchmarks + if not suite: + logger.debug("Suite {} executed but no results" + .format(suite_name)) + continue + + yield suite + + @staticmethod + def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs): + """ Returns a BenchmarkRunner from a path or a git revision. + + First, it checks if `rev_or_path` is a valid path (or string) of a json + object that can deserialize to a BenchmarkRunner. If so, it initialize + a StaticBenchmarkRunner from it. This allows memoizing the result of a + run in a file or a string. + + Second, it checks if `rev_or_path` points to a valid CMake build + directory. If so, it creates a CppBenchmarkRunner with this existing + CMakeBuild. + + Otherwise, it assumes `rev_or_path` is a revision and clone/checkout + the given revision and create a fresh CMakeBuild. + """ + build = None + if StaticBenchmarkRunner.is_json_result(rev_or_path): + return StaticBenchmarkRunner.from_json(rev_or_path, **kwargs) + elif CMakeBuild.is_build_dir(rev_or_path): + build = CMakeBuild.from_path(rev_or_path) + return CppBenchmarkRunner(build, **kwargs) + else: + # Revisions can references remote via the `/` character, ensure + # that the revision is path friendly + path_rev = rev_or_path.replace("/", "_") + root_rev = os.path.join(root, path_rev) + os.mkdir(root_rev) + + clone_dir = os.path.join(root_rev, "arrow") + # Possibly checkout the sources at given revision, no need to + # perform cleanup on cloned repository as root_rev is reclaimed. + src_rev, _ = src.at_revision(rev_or_path, clone_dir) + cmake_def = CppCMakeDefinition(src_rev.cpp, cmake_conf) + build_dir = os.path.join(root_rev, "build") + return CppBenchmarkRunner(cmake_def.build(build_dir), **kwargs) + + +class JavaBenchmarkRunner(BenchmarkRunner): + """ Run suites for Java. """ + + # default repetitions is 5 for Java microbenchmark harness + def __init__(self, build, **kwargs): + """ Initialize a JavaBenchmarkRunner. """ + self.build = build + super().__init__(**kwargs) + + @staticmethod + def default_configuration(**kwargs): + """ Returns the default benchmark configuration. """ + return JavaConfiguration(**kwargs) + + def suite(self, name): + """ Returns the resulting benchmarks for a given suite. """ + # update .m2 directory, which installs target jars + self.build.build() + + suite_cmd = JavaMicrobenchmarkHarnessCommand( + self.build, self.benchmark_filter) + + # Ensure there will be data + benchmark_names = suite_cmd.list_benchmarks() + if not benchmark_names: + return None + + results = suite_cmd.results(repetitions=self.repetitions) + benchmarks = JavaMicrobenchmarkHarness.from_json(results) + return BenchmarkSuite(name, benchmarks) + + @property + def list_benchmarks(self): + """ Returns all suite names """ + # Ensure build is up-to-date to run benchmarks + self.build.build() + + suite_cmd = JavaMicrobenchmarkHarnessCommand(self.build) + benchmark_names = suite_cmd.list_benchmarks() + for benchmark_name in benchmark_names: + yield "{}".format(benchmark_name) + + @property + def suites(self): + """ Returns all suite for a runner. """ + suite_name = "JavaBenchmark" + suite = self.suite(suite_name) + + # Filter may exclude all benchmarks + if not suite: + logger.debug("Suite {} executed but no results" + .format(suite_name)) + return + + yield suite + + @staticmethod + def from_rev_or_path(src, root, rev_or_path, maven_conf, **kwargs): + """ Returns a BenchmarkRunner from a path or a git revision. + + First, it checks if `rev_or_path` is a valid path (or string) of a json + object that can deserialize to a BenchmarkRunner. If so, it initialize + a StaticBenchmarkRunner from it. This allows memoizing the result of a + run in a file or a string. + + Second, it checks if `rev_or_path` points to a valid Maven build + directory. If so, it creates a JavaBenchmarkRunner with this existing + MavenBuild. + + Otherwise, it assumes `rev_or_path` is a revision and clone/checkout + the given revision and create a fresh MavenBuild. + """ + if StaticBenchmarkRunner.is_json_result(rev_or_path): + return StaticBenchmarkRunner.from_json(rev_or_path, **kwargs) + elif MavenBuild.is_build_dir(rev_or_path): + maven_def = JavaMavenDefinition(rev_or_path, maven_conf) + return JavaBenchmarkRunner(maven_def.build(rev_or_path), **kwargs) + else: + # Revisions can references remote via the `/` character, ensure + # that the revision is path friendly + path_rev = rev_or_path.replace("/", "_") + root_rev = os.path.join(root, path_rev) + os.mkdir(root_rev) + + clone_dir = os.path.join(root_rev, "arrow") + # Possibly checkout the sources at given revision, no need to + # perform cleanup on cloned repository as root_rev is reclaimed. + src_rev, _ = src.at_revision(rev_or_path, clone_dir) + maven_def = JavaMavenDefinition(src_rev.java, maven_conf) + build_dir = os.path.join(root_rev, "arrow/java") + return JavaBenchmarkRunner(maven_def.build(build_dir), **kwargs) diff --git a/src/arrow/dev/archery/archery/bot.py b/src/arrow/dev/archery/archery/bot.py new file mode 100644 index 000000000..e8fbbdd04 --- /dev/null +++ b/src/arrow/dev/archery/archery/bot.py @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import shlex +from pathlib import Path +from functools import partial +import tempfile + +import click +import github + +from .utils.git import git +from .utils.logger import logger +from .crossbow import Repo, Queue, Config, Target, Job, CommentReport + + +class EventError(Exception): + pass + + +class CommandError(Exception): + + def __init__(self, message): + self.message = message + + +class _CommandMixin: + + def get_help_option(self, ctx): + def show_help(ctx, param, value): + if value and not ctx.resilient_parsing: + raise click.UsageError(ctx.get_help()) + option = super().get_help_option(ctx) + option.callback = show_help + return option + + def __call__(self, message, **kwargs): + args = shlex.split(message) + try: + with self.make_context(self.name, args=args, obj=kwargs) as ctx: + return self.invoke(ctx) + except click.ClickException as e: + raise CommandError(e.format_message()) + + +class Command(_CommandMixin, click.Command): + pass + + +class Group(_CommandMixin, click.Group): + + def command(self, *args, **kwargs): + kwargs.setdefault('cls', Command) + return super().command(*args, **kwargs) + + def group(self, *args, **kwargs): + kwargs.setdefault('cls', Group) + return super().group(*args, **kwargs) + + def parse_args(self, ctx, args): + if not args and self.no_args_is_help and not ctx.resilient_parsing: + raise click.UsageError(ctx.get_help()) + return super().parse_args(ctx, args) + + +command = partial(click.command, cls=Command) +group = partial(click.group, cls=Group) + + +class CommentBot: + + def __init__(self, name, handler, token=None): + # TODO(kszucs): validate + assert isinstance(name, str) + assert callable(handler) + self.name = name + self.handler = handler + self.github = github.Github(token) + + def parse_command(self, payload): + # only allow users of apache org to submit commands, for more see + # https://developer.github.com/v4/enum/commentauthorassociation/ + allowed_roles = {'OWNER', 'MEMBER', 'CONTRIBUTOR'} + mention = '@{}'.format(self.name) + comment = payload['comment'] + + if payload['sender']['login'] == self.name: + raise EventError("Don't respond to itself") + elif payload['action'] not in {'created', 'edited'}: + raise EventError("Don't respond to comment deletion") + elif comment['author_association'] not in allowed_roles: + raise EventError( + "Don't respond to comments from non-authorized users" + ) + elif not comment['body'].lstrip().startswith(mention): + raise EventError("The bot is not mentioned") + + # Parse the comment, removing the bot mentioned (and everything + # before it) + command = payload['comment']['body'].split(mention)[-1] + + # then split on newlines and keep only the first line + # (ignoring all other lines) + return command.split("\n")[0].strip() + + def handle(self, event, payload): + try: + command = self.parse_command(payload) + except EventError as e: + logger.error(e) + # see the possible reasons in the validate method + return + + if event == 'issue_comment': + return self.handle_issue_comment(command, payload) + elif event == 'pull_request_review_comment': + return self.handle_review_comment(command, payload) + else: + raise ValueError("Unexpected event type {}".format(event)) + + def handle_issue_comment(self, command, payload): + repo = self.github.get_repo(payload['repository']['id'], lazy=True) + issue = repo.get_issue(payload['issue']['number']) + + try: + pull = issue.as_pull_request() + except github.GithubException: + return issue.create_comment( + "The comment bot only listens to pull request comments!" + ) + + comment = pull.get_issue_comment(payload['comment']['id']) + try: + self.handler(command, issue=issue, pull_request=pull, + comment=comment) + except CommandError as e: + logger.error(e) + pull.create_issue_comment("```\n{}\n```".format(e.message)) + except Exception as e: + logger.exception(e) + comment.create_reaction('-1') + else: + comment.create_reaction('+1') + + def handle_review_comment(self, payload): + raise NotImplementedError() + + +@group(name='@github-actions') +@click.pass_context +def actions(ctx): + """Ursabot""" + ctx.ensure_object(dict) + + +@actions.group() +@click.option('--crossbow', '-c', default='ursacomputing/crossbow', + help='Crossbow repository on github to use') +@click.pass_obj +def crossbow(obj, crossbow): + """ + Trigger crossbow builds for this pull request + """ + obj['crossbow_repo'] = crossbow + + +def _clone_arrow_and_crossbow(dest, crossbow_repo, pull_request): + """ + Clone the repositories and initialize crossbow objects. + + Parameters + ---------- + dest : Path + Filesystem path to clone the repositories to. + crossbow_repo : str + Github repository name, like kszucs/crossbow. + pull_request : pygithub.PullRequest + Object containing information about the pull request the comment bot + was triggered from. + """ + arrow_path = dest / 'arrow' + queue_path = dest / 'crossbow' + + # clone arrow and checkout the pull request's branch + pull_request_ref = 'pull/{}/head:{}'.format( + pull_request.number, pull_request.head.ref + ) + git.clone(pull_request.base.repo.clone_url, str(arrow_path)) + git.fetch('origin', pull_request_ref, git_dir=arrow_path) + git.checkout(pull_request.head.ref, git_dir=arrow_path) + + # clone crossbow repository + crossbow_url = 'https://github.com/{}'.format(crossbow_repo) + git.clone(crossbow_url, str(queue_path)) + + # initialize crossbow objects + github_token = os.environ['CROSSBOW_GITHUB_TOKEN'] + arrow = Repo(arrow_path) + queue = Queue(queue_path, github_token=github_token, require_https=True) + + return (arrow, queue) + + +@crossbow.command() +@click.argument('tasks', nargs=-1, required=False) +@click.option('--group', '-g', 'groups', multiple=True, + help='Submit task groups as defined in tests.yml') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') +@click.pass_obj +def submit(obj, tasks, groups, params, arrow_version): + """ + Submit crossbow testing tasks. + + See groups defined in arrow/dev/tasks/tasks.yml + """ + crossbow_repo = obj['crossbow_repo'] + pull_request = obj['pull_request'] + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = Path(tmpdir) + arrow, queue = _clone_arrow_and_crossbow( + dest=Path(tmpdir), + crossbow_repo=crossbow_repo, + pull_request=pull_request, + ) + # load available tasks configuration and groups from yaml + config = Config.load_yaml(arrow.path / "dev" / "tasks" / "tasks.yml") + config.validate() + + # initialize the crossbow build's target repository + target = Target.from_repo(arrow, version=arrow_version, + remote=pull_request.head.repo.clone_url, + branch=pull_request.head.ref) + + # parse additional job parameters + params = dict([p.split("=") for p in params]) + + # instantiate the job object + job = Job.from_config(config=config, target=target, tasks=tasks, + groups=groups, params=params) + + # add the job to the crossbow queue and push to the remote repository + queue.put(job, prefix="actions") + queue.push() + + # render the response comment's content + report = CommentReport(job, crossbow_repo=crossbow_repo) + + # send the response + pull_request.create_issue_comment(report.show()) diff --git a/src/arrow/dev/archery/archery/cli.py b/src/arrow/dev/archery/archery/cli.py new file mode 100644 index 000000000..d408be3cc --- /dev/null +++ b/src/arrow/dev/archery/archery/cli.py @@ -0,0 +1,943 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import namedtuple +from io import StringIO +import click +import errno +import json +import logging +import os +import pathlib +import sys + +from .benchmark.codec import JsonEncoder +from .benchmark.compare import RunnerComparator, DEFAULT_THRESHOLD +from .benchmark.runner import CppBenchmarkRunner, JavaBenchmarkRunner +from .compat import _import_pandas +from .lang.cpp import CppCMakeDefinition, CppConfiguration +from .utils.cli import ArrowBool, validate_arrow_sources, add_optional_command +from .utils.lint import linter, python_numpydoc, LintValidationException +from .utils.logger import logger, ctx as log_ctx +from .utils.source import ArrowSources +from .utils.tmpdir import tmpdir + +# Set default logging to INFO in command line. +logging.basicConfig(level=logging.INFO) + + +BOOL = ArrowBool() + + +@click.group() +@click.option("--debug", type=BOOL, is_flag=True, default=False, + help="Increase logging with debugging output.") +@click.option("--pdb", type=BOOL, is_flag=True, default=False, + help="Invoke pdb on uncaught exception.") +@click.option("-q", "--quiet", type=BOOL, is_flag=True, default=False, + help="Silence executed commands.") +@click.pass_context +def archery(ctx, debug, pdb, quiet): + """ Apache Arrow developer utilities. + + See sub-commands help with `archery <cmd> --help`. + + """ + # Ensure ctx.obj exists + ctx.ensure_object(dict) + + log_ctx.quiet = quiet + if debug: + logger.setLevel(logging.DEBUG) + + ctx.debug = debug + + if pdb: + import pdb + sys.excepthook = lambda t, v, e: pdb.pm() + + +build_dir_type = click.Path(dir_okay=True, file_okay=False, resolve_path=True) +# Supported build types +build_type = click.Choice(["debug", "relwithdebinfo", "release"], + case_sensitive=False) +# Supported warn levels +warn_level_type = click.Choice(["everything", "checkin", "production"], + case_sensitive=False) + +simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"], + case_sensitive=True) + + +def cpp_toolchain_options(cmd): + options = [ + click.option("--cc", metavar="<compiler>", help="C compiler."), + click.option("--cxx", metavar="<compiler>", help="C++ compiler."), + click.option("--cxx-flags", help="C++ compiler flags."), + click.option("--cpp-package-prefix", + help=("Value to pass for ARROW_PACKAGE_PREFIX and " + "use ARROW_DEPENDENCY_SOURCE=SYSTEM")) + ] + return _apply_options(cmd, options) + + +def java_toolchain_options(cmd): + options = [ + click.option("--java-home", metavar="<java_home>", + help="Path to Java Developers Kit."), + click.option("--java-options", help="java compiler options."), + ] + return _apply_options(cmd, options) + + +def _apply_options(cmd, options): + for option in options: + cmd = option(cmd) + return cmd + + +@archery.command(short_help="Initialize an Arrow C++ build") +@click.option("--src", metavar="<arrow_src>", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory") +# toolchain +@cpp_toolchain_options +@click.option("--build-type", default=None, type=build_type, + help="CMake's CMAKE_BUILD_TYPE") +@click.option("--warn-level", default="production", type=warn_level_type, + help="Controls compiler warnings -W(no-)error.") +@click.option("--use-gold-linker", default=True, type=BOOL, + help="Toggles ARROW_USE_LD_GOLD option.") +@click.option("--simd-level", default="SSE4_2", type=simd_level, + help="Toggles ARROW_SIMD_LEVEL option.") +# Tests and benchmarks +@click.option("--with-tests", default=True, type=BOOL, + help="Build with tests.") +@click.option("--with-benchmarks", default=None, type=BOOL, + help="Build with benchmarks.") +@click.option("--with-examples", default=None, type=BOOL, + help="Build with examples.") +@click.option("--with-integration", default=None, type=BOOL, + help="Build with integration test executables.") +# Static checks +@click.option("--use-asan", default=None, type=BOOL, + help="Toggle ARROW_USE_ASAN sanitizer.") +@click.option("--use-tsan", default=None, type=BOOL, + help="Toggle ARROW_USE_TSAN sanitizer.") +@click.option("--use-ubsan", default=None, type=BOOL, + help="Toggle ARROW_USE_UBSAN sanitizer.") +@click.option("--with-fuzzing", default=None, type=BOOL, + help="Toggle ARROW_FUZZING.") +# Components +@click.option("--with-compute", default=None, type=BOOL, + help="Build the Arrow compute module.") +@click.option("--with-csv", default=None, type=BOOL, + help="Build the Arrow CSV parser module.") +@click.option("--with-cuda", default=None, type=BOOL, + help="Build the Arrow CUDA extensions.") +@click.option("--with-dataset", default=None, type=BOOL, + help="Build the Arrow dataset module.") +@click.option("--with-filesystem", default=None, type=BOOL, + help="Build the Arrow filesystem layer.") +@click.option("--with-flight", default=None, type=BOOL, + help="Build with Flight rpc support.") +@click.option("--with-gandiva", default=None, type=BOOL, + help="Build with Gandiva expression compiler support.") +@click.option("--with-hdfs", default=None, type=BOOL, + help="Build the Arrow HDFS bridge.") +@click.option("--with-hiveserver2", default=None, type=BOOL, + help="Build the HiveServer2 client and arrow adapater.") +@click.option("--with-ipc", default=None, type=BOOL, + help="Build the Arrow IPC extensions.") +@click.option("--with-json", default=None, type=BOOL, + help="Build the Arrow JSON parser module.") +@click.option("--with-jni", default=None, type=BOOL, + help="Build the Arrow JNI lib.") +@click.option("--with-mimalloc", default=None, type=BOOL, + help="Build the Arrow mimalloc based allocator.") +@click.option("--with-parquet", default=None, type=BOOL, + help="Build with Parquet file support.") +@click.option("--with-plasma", default=None, type=BOOL, + help="Build with Plasma object store support.") +@click.option("--with-python", default=None, type=BOOL, + help="Build the Arrow CPython extesions.") +@click.option("--with-r", default=None, type=BOOL, + help="Build the Arrow R extensions. This is not a CMake option, " + "it will toggle required options") +@click.option("--with-s3", default=None, type=BOOL, + help="Build Arrow with S3 support.") +# Compressions +@click.option("--with-brotli", default=None, type=BOOL, + help="Build Arrow with brotli compression.") +@click.option("--with-bz2", default=None, type=BOOL, + help="Build Arrow with bz2 compression.") +@click.option("--with-lz4", default=None, type=BOOL, + help="Build Arrow with lz4 compression.") +@click.option("--with-snappy", default=None, type=BOOL, + help="Build Arrow with snappy compression.") +@click.option("--with-zlib", default=None, type=BOOL, + help="Build Arrow with zlib compression.") +@click.option("--with-zstd", default=None, type=BOOL, + help="Build Arrow with zstd compression.") +# CMake extra feature +@click.option("--cmake-extras", type=str, multiple=True, + help="Extra flags/options to pass to cmake invocation. " + "Can be stacked") +@click.option("--install-prefix", type=str, + help="Destination directory where files are installed. Expand to" + "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the" + "variable exists.") +# misc +@click.option("-f", "--force", type=BOOL, is_flag=True, default=False, + help="Delete existing build directory if found.") +@click.option("--targets", type=str, multiple=True, + help="Generator targets to run. Can be stacked.") +@click.argument("build_dir", type=build_dir_type) +@click.pass_context +def build(ctx, src, build_dir, force, targets, **kwargs): + """ Initialize a C++ build directory. + + The build command creates a directory initialized with Arrow's cpp source + cmake and configuration. It can also optionally invoke the generator to + test the build (and used in scripts). + + Note that archery will carry the caller environment. It will also not touch + an existing directory, one must use the `--force` option to remove the + existing directory. + + Examples: + + \b + # Initialize build with clang8 and avx2 support in directory `clang8-build` + \b + archery build --cc=clang-8 --cxx=clang++-8 --cxx-flags=-mavx2 clang8-build + + \b + # Builds and run test + archery build --targets=all --targets=test build + """ + # Arrow's cpp cmake configuration + conf = CppConfiguration(**kwargs) + # This is a closure around cmake invocation, e.g. calling `def.build()` + # yields a directory ready to be run with the generator + cmake_def = CppCMakeDefinition(src.cpp, conf) + # Create build directory + build = cmake_def.build(build_dir, force=force) + + for target in targets: + build.run(target) + + +LintCheck = namedtuple('LintCheck', ('option_name', 'help')) + +lint_checks = [ + LintCheck('clang-format', "Format C++ files with clang-format."), + LintCheck('clang-tidy', "Lint C++ files with clang-tidy."), + LintCheck('cpplint', "Lint C++ files with cpplint."), + LintCheck('iwyu', "Lint changed C++ files with Include-What-You-Use."), + LintCheck('python', + "Format and lint Python files with autopep8 and flake8."), + LintCheck('numpydoc', "Lint Python files with numpydoc."), + LintCheck('cmake-format', "Format CMake files with cmake-format.py."), + LintCheck('rat', + "Check all sources files for license texts via Apache RAT."), + LintCheck('r', "Lint R files."), + LintCheck('docker', "Lint Dockerfiles with hadolint."), +] + + +def decorate_lint_command(cmd): + """ + Decorate the lint() command function to add individual per-check options. + """ + for check in lint_checks: + option = click.option("--{0}/--no-{0}".format(check.option_name), + default=None, help=check.help) + cmd = option(cmd) + return cmd + + +@archery.command(short_help="Check Arrow source tree for errors") +@click.option("--src", metavar="<arrow_src>", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory") +@click.option("--fix", is_flag=True, type=BOOL, default=False, + help="Toggle fixing the lint errors if the linter supports it.") +@click.option("--iwyu_all", is_flag=True, type=BOOL, default=False, + help="Run IWYU on all C++ files if enabled") +@click.option("-a", "--all", is_flag=True, default=False, + help="Enable all checks.") +@decorate_lint_command +@click.pass_context +def lint(ctx, src, fix, iwyu_all, **checks): + if checks.pop('all'): + # "--all" is given => enable all non-selected checks + for k, v in checks.items(): + if v is None: + checks[k] = True + if not any(checks.values()): + raise click.UsageError( + "Need to enable at least one lint check (try --help)") + try: + linter(src, fix, iwyu_all=iwyu_all, **checks) + except LintValidationException: + sys.exit(1) + + +@archery.command(short_help="Lint python docstring with NumpyDoc") +@click.argument('symbols', nargs=-1) +@click.option("--src", metavar="<arrow_src>", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory") +@click.option("--allow-rule", "-a", multiple=True, + help="Allow only these rules") +@click.option("--disallow-rule", "-d", multiple=True, + help="Disallow these rules") +def numpydoc(src, symbols, allow_rule, disallow_rule): + """ + Pass list of modules or symbols as arguments to restrict the validation. + + By default all modules of pyarrow are tried to be validated. + + Examples + -------- + archery numpydoc pyarrow.dataset + archery numpydoc pyarrow.csv pyarrow.json pyarrow.parquet + archery numpydoc pyarrow.array + """ + disallow_rule = disallow_rule or {'GL01', 'SA01', 'EX01', 'ES01'} + try: + results = python_numpydoc(symbols, allow_rules=allow_rule, + disallow_rules=disallow_rule) + for result in results: + result.ok() + except LintValidationException: + sys.exit(1) + + +@archery.group() +@click.pass_context +def benchmark(ctx): + """ Arrow benchmarking. + + Use the diff sub-command to benchmark revisions, and/or build directories. + """ + pass + + +def benchmark_common_options(cmd): + def check_language(ctx, param, value): + if value not in {"cpp", "java"}: + raise click.BadParameter("cpp or java is supported now") + return value + + options = [ + click.option("--src", metavar="<arrow_src>", show_default=True, + default=None, callback=validate_arrow_sources, + help="Specify Arrow source directory"), + click.option("--preserve", type=BOOL, default=False, show_default=True, + is_flag=True, + help="Preserve workspace for investigation."), + click.option("--output", metavar="<output>", + type=click.File("w", encoding="utf8"), default="-", + help="Capture output result into file."), + click.option("--language", metavar="<lang>", type=str, default="cpp", + show_default=True, callback=check_language, + help="Specify target language for the benchmark"), + click.option("--build-extras", type=str, multiple=True, + help="Extra flags/options to pass to mvn build. " + "Can be stacked. For language=java"), + click.option("--benchmark-extras", type=str, multiple=True, + help="Extra flags/options to pass to mvn benchmark. " + "Can be stacked. For language=java"), + click.option("--cmake-extras", type=str, multiple=True, + help="Extra flags/options to pass to cmake invocation. " + "Can be stacked. For language=cpp") + ] + + cmd = java_toolchain_options(cmd) + cmd = cpp_toolchain_options(cmd) + return _apply_options(cmd, options) + + +def benchmark_filter_options(cmd): + options = [ + click.option("--suite-filter", metavar="<regex>", show_default=True, + type=str, default=None, + help="Regex filtering benchmark suites."), + click.option("--benchmark-filter", metavar="<regex>", + show_default=True, type=str, default=None, + help="Regex filtering benchmarks.") + ] + return _apply_options(cmd, options) + + +@benchmark.command(name="list", short_help="List benchmark suite") +@click.argument("rev_or_path", metavar="[<rev_or_path>]", + default="WORKSPACE", required=False) +@benchmark_common_options +@click.pass_context +def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras, + java_home, java_options, build_extras, benchmark_extras, + language, **kwargs): + """ List benchmark suite. + """ + with tmpdir(preserve=preserve) as root: + logger.debug("Running benchmark {}".format(rev_or_path)) + + if language == "cpp": + conf = CppBenchmarkRunner.default_configuration( + cmake_extras=cmake_extras, **kwargs) + + runner_base = CppBenchmarkRunner.from_rev_or_path( + src, root, rev_or_path, conf) + + elif language == "java": + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + del kwargs[key] + conf = JavaBenchmarkRunner.default_configuration( + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) + + runner_base = JavaBenchmarkRunner.from_rev_or_path( + src, root, rev_or_path, conf) + + for b in runner_base.list_benchmarks: + click.echo(b, file=output) + + +@benchmark.command(name="run", short_help="Run benchmark suite") +@click.argument("rev_or_path", metavar="[<rev_or_path>]", + default="WORKSPACE", required=False) +@benchmark_common_options +@benchmark_filter_options +@click.option("--repetitions", type=int, default=-1, + help=("Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java")) +@click.pass_context +def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras, + java_home, java_options, build_extras, benchmark_extras, + language, suite_filter, benchmark_filter, repetitions, + **kwargs): + """ Run benchmark suite. + + This command will run the benchmark suite for a single build. This is + used to capture (and/or publish) the results. + + The caller can optionally specify a target which is either a git revision + (commit, tag, special values like HEAD) or a cmake build directory. + + When a commit is referenced, a local clone of the arrow sources (specified + via --src) is performed and the proper branch is created. This is done in + a temporary directory which can be left intact with the `--preserve` flag. + + The special token "WORKSPACE" is reserved to specify the current git + workspace. This imply that no clone will be performed. + + Examples: + + \b + # Run the benchmarks on current git workspace + \b + archery benchmark run + + \b + # Run the benchmarks on current previous commit + \b + archery benchmark run HEAD~1 + + \b + # Run the benchmarks on current previous commit + \b + archery benchmark run --output=run.json + """ + with tmpdir(preserve=preserve) as root: + logger.debug("Running benchmark {}".format(rev_or_path)) + + if language == "cpp": + conf = CppBenchmarkRunner.default_configuration( + cmake_extras=cmake_extras, **kwargs) + + repetitions = repetitions if repetitions != -1 else 1 + runner_base = CppBenchmarkRunner.from_rev_or_path( + src, root, rev_or_path, conf, + repetitions=repetitions, + suite_filter=suite_filter, benchmark_filter=benchmark_filter) + + elif language == "java": + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + del kwargs[key] + conf = JavaBenchmarkRunner.default_configuration( + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) + + repetitions = repetitions if repetitions != -1 else 5 + runner_base = JavaBenchmarkRunner.from_rev_or_path( + src, root, rev_or_path, conf, + repetitions=repetitions, + benchmark_filter=benchmark_filter) + + json.dump(runner_base, output, cls=JsonEncoder) + + +@benchmark.command(name="diff", short_help="Compare benchmark suites") +@benchmark_common_options +@benchmark_filter_options +@click.option("--threshold", type=float, default=DEFAULT_THRESHOLD, + show_default=True, + help="Regression failure threshold in percentage.") +@click.option("--repetitions", type=int, default=1, show_default=True, + help=("Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java")) +@click.option("--no-counters", type=BOOL, default=False, is_flag=True, + help="Hide counters field in diff report.") +@click.argument("contender", metavar="[<contender>", + default=ArrowSources.WORKSPACE, required=False) +@click.argument("baseline", metavar="[<baseline>]]", default="origin/master", + required=False) +@click.pass_context +def benchmark_diff(ctx, src, preserve, output, language, cmake_extras, + suite_filter, benchmark_filter, repetitions, no_counters, + java_home, java_options, build_extras, benchmark_extras, + threshold, contender, baseline, **kwargs): + """Compare (diff) benchmark runs. + + This command acts like git-diff but for benchmark results. + + The caller can optionally specify both the contender and the baseline. If + unspecified, the contender will default to the current workspace (like git) + and the baseline will default to master. + + Each target (contender or baseline) can either be a git revision + (commit, tag, special values like HEAD) or a cmake build directory. This + allow comparing git commits, and/or different compilers and/or compiler + flags. + + When a commit is referenced, a local clone of the arrow sources (specified + via --src) is performed and the proper branch is created. This is done in + a temporary directory which can be left intact with the `--preserve` flag. + + The special token "WORKSPACE" is reserved to specify the current git + workspace. This imply that no clone will be performed. + + Examples: + + \b + # Compare workspace (contender) with master (baseline) + \b + archery benchmark diff + + \b + # Compare master (contender) with latest version (baseline) + \b + export LAST=$(git tag -l "apache-arrow-[0-9]*" | sort -rV | head -1) + \b + archery benchmark diff master "$LAST" + + \b + # Compare g++7 (contender) with clang++-8 (baseline) builds + \b + archery build --with-benchmarks=true \\ + --cxx-flags=-ftree-vectorize \\ + --cc=gcc-7 --cxx=g++-7 gcc7-build + \b + archery build --with-benchmarks=true \\ + --cxx-flags=-flax-vector-conversions \\ + --cc=clang-8 --cxx=clang++-8 clang8-build + \b + archery benchmark diff gcc7-build clang8-build + + \b + # Compare default targets but scoped to the suites matching + # `^arrow-compute-aggregate` and benchmarks matching `(Sum|Mean)Kernel`. + \b + archery benchmark diff --suite-filter="^arrow-compute-aggregate" \\ + --benchmark-filter="(Sum|Mean)Kernel" + + \b + # Capture result in file `result.json` + \b + archery benchmark diff --output=result.json + \b + # Equivalently with no stdout clutter. + archery --quiet benchmark diff > result.json + + \b + # Comparing with a cached results from `archery benchmark run` + \b + archery benchmark run --output=run.json HEAD~1 + \b + # This should not recompute the benchmark from run.json + archery --quiet benchmark diff WORKSPACE run.json > result.json + """ + with tmpdir(preserve=preserve) as root: + logger.debug("Comparing {} (contender) with {} (baseline)" + .format(contender, baseline)) + + if language == "cpp": + conf = CppBenchmarkRunner.default_configuration( + cmake_extras=cmake_extras, **kwargs) + + repetitions = repetitions if repetitions != -1 else 1 + runner_cont = CppBenchmarkRunner.from_rev_or_path( + src, root, contender, conf, + repetitions=repetitions, + suite_filter=suite_filter, + benchmark_filter=benchmark_filter) + runner_base = CppBenchmarkRunner.from_rev_or_path( + src, root, baseline, conf, + repetitions=repetitions, + suite_filter=suite_filter, + benchmark_filter=benchmark_filter) + + elif language == "java": + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + del kwargs[key] + conf = JavaBenchmarkRunner.default_configuration( + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) + + repetitions = repetitions if repetitions != -1 else 5 + runner_cont = JavaBenchmarkRunner.from_rev_or_path( + src, root, contender, conf, + repetitions=repetitions, + benchmark_filter=benchmark_filter) + runner_base = JavaBenchmarkRunner.from_rev_or_path( + src, root, baseline, conf, + repetitions=repetitions, + benchmark_filter=benchmark_filter) + + runner_comp = RunnerComparator(runner_cont, runner_base, threshold) + + # TODO(kszucs): test that the output is properly formatted jsonlines + comparisons_json = _get_comparisons_as_json(runner_comp.comparisons) + ren_counters = language == "java" + formatted = _format_comparisons_with_pandas(comparisons_json, + no_counters, ren_counters) + output.write(formatted) + output.write('\n') + + +def _get_comparisons_as_json(comparisons): + buf = StringIO() + for comparator in comparisons: + json.dump(comparator, buf, cls=JsonEncoder) + buf.write("\n") + + return buf.getvalue() + + +def _format_comparisons_with_pandas(comparisons_json, no_counters, + ren_counters): + pd = _import_pandas() + df = pd.read_json(StringIO(comparisons_json), lines=True) + # parse change % so we can sort by it + df['change %'] = df.pop('change').str[:-1].map(float) + first_regression = len(df) - df['regression'].sum() + + fields = ['benchmark', 'baseline', 'contender', 'change %'] + if not no_counters: + fields += ['counters'] + + df = df[fields] + if ren_counters: + df = df.rename(columns={'counters': 'configurations'}) + df = df.sort_values(by='change %', ascending=False) + + def labelled(title, df): + if len(df) == 0: + return '' + title += ': ({})'.format(len(df)) + df_str = df.to_string(index=False) + bar = '-' * df_str.index('\n') + return '\n'.join([bar, title, bar, df_str]) + + return '\n\n'.join([labelled('Non-regressions', df[:first_regression]), + labelled('Regressions', df[first_regression:])]) + + +# ---------------------------------------------------------------------- +# Integration testing + +def _set_default(opt, default): + if opt is None: + return default + return opt + + +@archery.command(short_help="Execute protocol and Flight integration tests") +@click.option('--with-all', is_flag=True, default=False, + help=('Include all known languages by default ' + 'in integration tests')) +@click.option('--random-seed', type=int, default=12345, + help="Seed for PRNG when generating test data") +@click.option('--with-cpp', type=bool, default=False, + help='Include C++ in integration tests') +@click.option('--with-csharp', type=bool, default=False, + help='Include C# in integration tests') +@click.option('--with-java', type=bool, default=False, + help='Include Java in integration tests') +@click.option('--with-js', type=bool, default=False, + help='Include JavaScript in integration tests') +@click.option('--with-go', type=bool, default=False, + help='Include Go in integration tests') +@click.option('--with-rust', type=bool, default=False, + help='Include Rust in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_RUST") +@click.option('--write_generated_json', default=False, + help='Generate test JSON to indicated path') +@click.option('--run-flight', is_flag=True, default=False, + help='Run Flight integration tests') +@click.option('--debug', is_flag=True, default=False, + help='Run executables in debug mode as relevant') +@click.option('--serial', is_flag=True, default=False, + help='Run tests serially, rather than in parallel') +@click.option('--tempdir', default=None, + help=('Directory to use for writing ' + 'integration test temporary files')) +@click.option('stop_on_error', '-x', '--stop-on-error', + is_flag=True, default=False, + help='Stop on first error') +@click.option('--gold-dirs', multiple=True, + help="gold integration test file paths") +@click.option('-k', '--match', + help=("Substring for test names to include in run, " + "e.g. -k primitive")) +def integration(with_all=False, random_seed=12345, **args): + from .integration.runner import write_js_test_json, run_all_tests + import numpy as np + + # FIXME(bkietz) Include help strings for individual testers. + # For example, CPPTester's ARROW_CPP_EXE_PATH environment variable. + + # Make runs involving data generation deterministic + np.random.seed(random_seed) + + gen_path = args['write_generated_json'] + + languages = ['cpp', 'csharp', 'java', 'js', 'go', 'rust'] + + enabled_languages = 0 + for lang in languages: + param = 'with_{}'.format(lang) + if with_all: + args[param] = with_all + + if args[param]: + enabled_languages += 1 + + if gen_path: + try: + os.makedirs(gen_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + write_js_test_json(gen_path) + else: + if enabled_languages == 0: + raise Exception("Must enable at least 1 language to test") + run_all_tests(**args) + + +@archery.command() +@click.option('--event-name', '-n', required=True) +@click.option('--event-payload', '-p', type=click.File('r', encoding='utf8'), + default='-', required=True) +@click.option('--arrow-token', envvar='ARROW_GITHUB_TOKEN', + help='OAuth token for responding comment in the arrow repo') +def trigger_bot(event_name, event_payload, arrow_token): + from .bot import CommentBot, actions + + event_payload = json.loads(event_payload.read()) + + bot = CommentBot(name='github-actions', handler=actions, token=arrow_token) + bot.handle(event_name, event_payload) + + +@archery.group('release') +@click.option("--src", metavar="<arrow_src>", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory.") +@click.option("--jira-cache", type=click.Path(), default=None, + help="File path to cache queried JIRA issues per version.") +@click.pass_obj +def release(obj, src, jira_cache): + """Release releated commands.""" + from .release import Jira, CachedJira + + jira = Jira() + if jira_cache is not None: + jira = CachedJira(jira_cache, jira=jira) + + obj['jira'] = jira + obj['repo'] = src.path + + +@release.command('curate') +@click.argument('version') +@click.pass_obj +def release_curate(obj, version): + """Release curation.""" + from .release import Release + + release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo']) + curation = release.curate() + + click.echo(curation.render('console')) + + +@release.group('changelog') +def release_changelog(): + """Release changelog.""" + pass + + +@release_changelog.command('add') +@click.argument('version') +@click.pass_obj +def release_changelog_add(obj, version): + """Prepend the changelog with the current release""" + from .release import Release + + jira, repo = obj['jira'], obj['repo'] + + # just handle the current version + release = Release.from_jira(version, jira=jira, repo=repo) + if release.is_released: + raise ValueError('This version has been already released!') + + changelog = release.changelog() + changelog_path = pathlib.Path(repo) / 'CHANGELOG.md' + + current_content = changelog_path.read_text() + new_content = changelog.render('markdown') + current_content + + changelog_path.write_text(new_content) + click.echo("CHANGELOG.md is updated!") + + +@release_changelog.command('generate') +@click.argument('version') +@click.argument('output', type=click.File('w', encoding='utf8'), default='-') +@click.pass_obj +def release_changelog_generate(obj, version, output): + """Generate the changelog of a specific release.""" + from .release import Release + + jira, repo = obj['jira'], obj['repo'] + + # just handle the current version + release = Release.from_jira(version, jira=jira, repo=repo) + + changelog = release.changelog() + output.write(changelog.render('markdown')) + + +@release_changelog.command('regenerate') +@click.pass_obj +def release_changelog_regenerate(obj): + """Regeneretate the whole CHANGELOG.md file""" + from .release import Release + + jira, repo = obj['jira'], obj['repo'] + changelogs = [] + + for version in jira.project_versions('ARROW'): + if not version.released: + continue + release = Release.from_jira(version, jira=jira, repo=repo) + click.echo('Querying changelog for version: {}'.format(version)) + changelogs.append(release.changelog()) + + click.echo('Rendering new CHANGELOG.md file...') + changelog_path = pathlib.Path(repo) / 'CHANGELOG.md' + with changelog_path.open('w') as fp: + for cl in changelogs: + fp.write(cl.render('markdown')) + + +@release.command('cherry-pick') +@click.argument('version') +@click.option('--dry-run/--execute', default=True, + help="Display the git commands instead of executing them.") +@click.option('--recreate/--continue', default=True, + help="Recreate the maintenance branch or only apply unapplied " + "patches.") +@click.pass_obj +def release_cherry_pick(obj, version, dry_run, recreate): + """ + Cherry pick commits. + """ + from .release import Release, MinorRelease, PatchRelease + + release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo']) + if not isinstance(release, (MinorRelease, PatchRelease)): + raise click.UsageError('Cherry-pick command only supported for minor ' + 'and patch releases') + + if not dry_run: + release.cherry_pick_commits(recreate_branch=recreate) + click.echo('Executed the following commands:\n') + + click.echo( + 'git checkout {} -b {}'.format(release.previous.tag, release.branch) + ) + for commit in release.commits_to_pick(): + click.echo('git cherry-pick {}'.format(commit.hexsha)) + + +@archery.group("linking") +@click.pass_obj +def linking(obj): + """ + Quick and dirty utilities for checking library linkage. + """ + pass + + +@linking.command("check-dependencies") +@click.argument("paths", nargs=-1) +@click.option("--allow", "-a", "allowed", multiple=True, + help="Name of the allowed libraries") +@click.option("--disallow", "-d", "disallowed", multiple=True, + help="Name of the disallowed libraries") +@click.pass_obj +def linking_check_dependencies(obj, allowed, disallowed, paths): + from .linking import check_dynamic_library_dependencies, DependencyError + + allowed, disallowed = set(allowed), set(disallowed) + try: + for path in map(pathlib.Path, paths): + check_dynamic_library_dependencies(path, allowed=allowed, + disallowed=disallowed) + except DependencyError as e: + raise click.ClickException(str(e)) + + +add_optional_command("docker", module=".docker.cli", function="docker", + parent=archery) +add_optional_command("crossbow", module=".crossbow.cli", function="crossbow", + parent=archery) + + +if __name__ == "__main__": + archery(obj={}) diff --git a/src/arrow/dev/archery/archery/compat.py b/src/arrow/dev/archery/archery/compat.py new file mode 100644 index 000000000..bb0b15428 --- /dev/null +++ b/src/arrow/dev/archery/archery/compat.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + + +def _is_path_like(path): + # PEP519 filesystem path protocol is available from python 3.6, so pathlib + # doesn't implement __fspath__ for earlier versions + return (isinstance(path, str) or + hasattr(path, '__fspath__') or + isinstance(path, pathlib.Path)) + + +def _ensure_path(path): + if isinstance(path, pathlib.Path): + return path + else: + return pathlib.Path(_stringify_path(path)) + + +def _stringify_path(path): + """ + Convert *path* to a string or unicode path if possible. + """ + if isinstance(path, str): + return path + + # checking whether path implements the filesystem protocol + try: + return path.__fspath__() # new in python 3.6 + except AttributeError: + # fallback pathlib ckeck for earlier python versions than 3.6 + if isinstance(path, pathlib.Path): + return str(path) + + raise TypeError("not a path-like object") + + +def _import_pandas(): + # ARROW-13425: avoid importing PyArrow from Pandas + sys.modules['pyarrow'] = None + import pandas as pd + return pd diff --git a/src/arrow/dev/archery/archery/crossbow/__init__.py b/src/arrow/dev/archery/archery/crossbow/__init__.py new file mode 100644 index 000000000..bc72e81f0 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .core import Config, Repo, Queue, Target, Job # noqa +from .reports import CommentReport, ConsoleReport, EmailReport # noqa diff --git a/src/arrow/dev/archery/archery/crossbow/cli.py b/src/arrow/dev/archery/archery/crossbow/cli.py new file mode 100644 index 000000000..1d0610343 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/cli.py @@ -0,0 +1,365 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path + +import click + +from .core import Config, Repo, Queue, Target, Job, CrossbowError +from .reports import EmailReport, ConsoleReport +from ..utils.source import ArrowSources + + +_default_arrow_path = ArrowSources.find().path +_default_queue_path = _default_arrow_path.parent / "crossbow" +_default_config_path = _default_arrow_path / "dev" / "tasks" / "tasks.yml" + + +@click.group() +@click.option('--github-token', '-t', default=None, + envvar="CROSSBOW_GITHUB_TOKEN", + help='OAuth token for GitHub authentication') +@click.option('--arrow-path', '-a', + type=click.Path(), default=_default_arrow_path, + help='Arrow\'s repository path. Defaults to the repository of ' + 'this script') +@click.option('--queue-path', '-q', + type=click.Path(), default=_default_queue_path, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@click.option('--queue-remote', '-qr', default=None, + help='Force to use this remote URL for the Queue repository') +@click.option('--output-file', metavar='<output>', + type=click.File('w', encoding='utf8'), default='-', + help='Capture output result into file.') +@click.pass_context +def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote, + output_file): + """ + Schedule packaging tasks or nightly builds on CI services. + """ + ctx.ensure_object(dict) + ctx.obj['output'] = output_file + ctx.obj['arrow'] = Repo(arrow_path) + ctx.obj['queue'] = Queue(queue_path, remote_url=queue_remote, + github_token=github_token, require_https=True) + + +@crossbow.command() +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.pass_obj +def check_config(obj, config_path): + # load available tasks configuration and groups from yaml + config = Config.load_yaml(config_path) + config.validate() + + output = obj['output'] + config.show(output) + + +@crossbow.command() +@click.argument('tasks', nargs=-1, required=False) +@click.option('--group', '-g', 'groups', multiple=True, + help='Submit task groups as defined in task.yml') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') +@click.option('--job-prefix', default='build', + help='Arbitrary prefix for branch names, e.g. nightly') +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') +@click.option('--arrow-remote', '-r', default=None, + help='Set GitHub remote explicitly, which is going to be cloned ' + 'on the CI services. Note, that no validation happens ' + 'locally. Examples: https://github.com/apache/arrow or ' + 'https://github.com/kszucs/arrow.') +@click.option('--arrow-branch', '-b', default=None, + help='Give the branch name explicitly, e.g. master, ARROW-1949.') +@click.option('--arrow-sha', '-t', default=None, + help='Set commit SHA or Tag name explicitly, e.g. f67a515, ' + 'apache-arrow-0.11.1.') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.option('--dry-run/--commit', default=False, + help='Just display the rendered CI configurations without ' + 'committing them') +@click.option('--no-push/--push', default=False, + help='Don\'t push the changes') +@click.pass_obj +def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version, + arrow_remote, arrow_branch, arrow_sha, fetch, dry_run, no_push): + output = obj['output'] + queue, arrow = obj['queue'], obj['arrow'] + + # load available tasks configuration and groups from yaml + config = Config.load_yaml(config_path) + try: + config.validate() + except CrossbowError as e: + raise click.ClickException(str(e)) + + # Override the detected repo url / remote, branch and sha - this aims to + # make release procedure a bit simpler. + # Note, that the target resivion's crossbow templates must be + # compatible with the locally checked out version of crossbow (which is + # in case of the release procedure), because the templates still + # contain some business logic (dependency installation, deployments) + # which will be reduced to a single command in the future. + target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, + head=arrow_sha, version=arrow_version) + + # parse additional job parameters + params = dict([p.split("=") for p in params]) + + # instantiate the job object + try: + job = Job.from_config(config=config, target=target, tasks=tasks, + groups=groups, params=params) + except CrossbowError as e: + raise click.ClickException(str(e)) + + job.show(output) + if dry_run: + return + + if fetch: + queue.fetch() + queue.put(job, prefix=job_prefix) + + if no_push: + click.echo('Branches and commits created but not pushed: `{}`' + .format(job.branch)) + else: + queue.push() + click.echo('Pushed job identifier is: `{}`'.format(job.branch)) + + +@crossbow.command() +@click.argument('task', required=True) +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') +@click.option('--arrow-remote', '-r', default=None, + help='Set GitHub remote explicitly, which is going to be cloned ' + 'on the CI services. Note, that no validation happens ' + 'locally. Examples: https://github.com/apache/arrow or ' + 'https://github.com/kszucs/arrow.') +@click.option('--arrow-branch', '-b', default=None, + help='Give the branch name explicitly, e.g. master, ARROW-1949.') +@click.option('--arrow-sha', '-t', default=None, + help='Set commit SHA or Tag name explicitly, e.g. f67a515, ' + 'apache-arrow-0.11.1.') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') +@click.pass_obj +def render(obj, task, config_path, arrow_version, arrow_remote, arrow_branch, + arrow_sha, params): + """ + Utility command to check the rendered CI templates. + """ + from .core import _flatten + + def highlight(code): + try: + from pygments import highlight + from pygments.lexers import YamlLexer + from pygments.formatters import TerminalFormatter + return highlight(code, YamlLexer(), TerminalFormatter()) + except ImportError: + return code + + arrow = obj['arrow'] + + target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, + head=arrow_sha, version=arrow_version) + config = Config.load_yaml(config_path) + params = dict([p.split("=") for p in params]) + params["queue_remote_url"] = "https://github.com/org/crossbow" + job = Job.from_config(config=config, target=target, tasks=[task], + params=params) + + for task_name, rendered_files in job.render_tasks().items(): + for path, content in _flatten(rendered_files).items(): + click.echo('#' * 80) + click.echo('### {:^72} ###'.format("/".join(path))) + click.echo('#' * 80) + click.echo(highlight(content)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.option('--task-filter', '-f', 'task_filters', multiple=True, + help='Glob pattern for filtering relevant tasks') +@click.pass_obj +def status(obj, job_name, fetch, task_filters): + output = obj['output'] + queue = obj['queue'] + if fetch: + queue.fetch() + job = queue.get(job_name) + + report = ConsoleReport(job, task_filters=task_filters) + report.show(output) + + +@crossbow.command() +@click.argument('prefix', required=True) +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def latest_prefix(obj, prefix, fetch): + queue = obj['queue'] + if fetch: + queue.fetch() + latest = queue.latest_for_prefix(prefix) + click.echo(latest.branch) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--sender-name', '-n', + help='Name to use for report e-mail.') +@click.option('--sender-email', '-e', + help='E-mail to use for report e-mail.') +@click.option('--recipient-email', '-r', + help='Where to send the e-mail report') +@click.option('--smtp-user', '-u', + help='E-mail address to use for SMTP login') +@click.option('--smtp-password', '-P', + help='SMTP password to use for report e-mail.') +@click.option('--smtp-server', '-s', default='smtp.gmail.com', + help='SMTP server to use for report e-mail.') +@click.option('--smtp-port', '-p', default=465, + help='SMTP port to use for report e-mail.') +@click.option('--poll/--no-poll', default=False, + help='Wait for completion if there are tasks pending') +@click.option('--poll-max-minutes', default=180, + help='Maximum amount of time waiting for job completion') +@click.option('--poll-interval-minutes', default=10, + help='Number of minutes to wait to check job status again') +@click.option('--send/--dry-run', default=False, + help='Just display the report, don\'t send it') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def report(obj, job_name, sender_name, sender_email, recipient_email, + smtp_user, smtp_password, smtp_server, smtp_port, poll, + poll_max_minutes, poll_interval_minutes, send, fetch): + """ + Send an e-mail report showing success/failure of tasks in a Crossbow run + """ + output = obj['output'] + queue = obj['queue'] + if fetch: + queue.fetch() + + job = queue.get(job_name) + report = EmailReport( + job=job, + sender_name=sender_name, + sender_email=sender_email, + recipient_email=recipient_email + ) + + if poll: + job.wait_until_finished( + poll_max_minutes=poll_max_minutes, + poll_interval_minutes=poll_interval_minutes + ) + + if send: + report.send( + smtp_user=smtp_user, + smtp_password=smtp_password, + smtp_server=smtp_server, + smtp_port=smtp_port + ) + else: + report.show(output) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('-t', '--target-dir', + default=_default_arrow_path / 'packages', + type=click.Path(file_okay=False, dir_okay=True), + help='Directory to download the build artifacts') +@click.option('--dry-run/--execute', default=False, + help='Just display process, don\'t download anything') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.option('--task-filter', '-f', 'task_filters', multiple=True, + help='Glob pattern for filtering relevant tasks') +@click.option('--validate-patterns/--skip-pattern-validation', default=True, + help='Whether to validate artifact name patterns or not') +@click.pass_obj +def download_artifacts(obj, job_name, target_dir, dry_run, fetch, + validate_patterns, task_filters): + """Download build artifacts from GitHub releases""" + output = obj['output'] + + # fetch the queue repository + queue = obj['queue'] + if fetch: + queue.fetch() + + # query the job's artifacts + job = queue.get(job_name) + + # create directory to download the assets to + target_dir = Path(target_dir).absolute() / job_name + target_dir.mkdir(parents=True, exist_ok=True) + + # download the assets while showing the job status + def asset_callback(task_name, task, asset): + if asset is not None: + path = target_dir / task_name / asset.name + path.parent.mkdir(exist_ok=True) + if not dry_run: + asset.download(path) + + click.echo('Downloading {}\'s artifacts.'.format(job_name)) + click.echo('Destination directory is {}'.format(target_dir)) + click.echo() + + report = ConsoleReport(job, task_filters=task_filters) + report.show( + output, + asset_callback=asset_callback, + validate_patterns=validate_patterns + ) + + +@crossbow.command() +@click.argument('patterns', nargs=-1, required=True) +@click.option('--sha', required=True, help='Target committish') +@click.option('--tag', required=True, help='Target tag') +@click.option('--method', default='curl', help='Use cURL to upload') +@click.pass_obj +def upload_artifacts(obj, tag, sha, patterns, method): + queue = obj['queue'] + queue.github_overwrite_release_assets( + tag_name=tag, target_commitish=sha, method=method, patterns=patterns + ) diff --git a/src/arrow/dev/archery/archery/crossbow/core.py b/src/arrow/dev/archery/archery/crossbow/core.py new file mode 100644 index 000000000..0f2309e47 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/core.py @@ -0,0 +1,1172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import fnmatch +import glob +import time +import logging +import mimetypes +import subprocess +import textwrap +from io import StringIO +from pathlib import Path +from datetime import date + +import jinja2 +from ruamel.yaml import YAML + +try: + import github3 + _have_github3 = True +except ImportError: + github3 = object + _have_github3 = False + +try: + import pygit2 +except ImportError: + PygitRemoteCallbacks = object +else: + PygitRemoteCallbacks = pygit2.RemoteCallbacks + +from ..utils.source import ArrowSources + + +for pkg in ["requests", "urllib3", "github3"]: + logging.getLogger(pkg).setLevel(logging.WARNING) + +logger = logging.getLogger("crossbow") + + +class CrossbowError(Exception): + pass + + +def _flatten(mapping): + """Converts a hierarchical mapping to a flat dictionary""" + result = {} + for k, v in mapping.items(): + if isinstance(v, dict): + for ik, iv in _flatten(v).items(): + ik = ik if isinstance(ik, tuple) else (ik,) + result[(k,) + ik] = iv + elif isinstance(v, list): + for ik, iv in enumerate(_flatten(v)): + ik = ik if isinstance(ik, tuple) else (ik,) + result[(k,) + ik] = iv + else: + result[(k,)] = v + return result + + +def _unflatten(mapping): + """Converts a flat tuple => object mapping to hierarchical one""" + result = {} + for path, value in mapping.items(): + parents, leaf = path[:-1], path[-1] + # create the hierarchy until we reach the leaf value + temp = result + for parent in parents: + temp.setdefault(parent, {}) + temp = temp[parent] + # set the leaf value + temp[leaf] = value + + return result + + +def _unflatten_tree(files): + """Converts a flat path => object mapping to a hierarchical directories + + Input: + { + 'path/to/file.a': a_content, + 'path/to/file.b': b_content, + 'path/file.c': c_content + } + Output: + { + 'path': { + 'to': { + 'file.a': a_content, + 'file.b': b_content + }, + 'file.c': c_content + } + } + """ + files = {tuple(k.split('/')): v for k, v in files.items()} + return _unflatten(files) + + +def _render_jinja_template(searchpath, template, params): + def format_all(items, pattern): + return [pattern.format(item) for item in items] + + loader = jinja2.FileSystemLoader(searchpath) + env = jinja2.Environment(loader=loader, trim_blocks=True, + lstrip_blocks=True, + undefined=jinja2.StrictUndefined) + env.filters['format_all'] = format_all + template = env.get_template(template) + return template.render(**params) + + +# configurations for setting up branch skipping +# - appveyor has a feature to skip builds without an appveyor.yml +# - travis reads from the master branch and applies the rules +# - circle requires the configuration to be present on all branch, even ones +# that are configured to be skipped +# - azure skips branches without azure-pipelines.yml by default +# - github skips branches without .github/workflows/ by default + +_default_travis_yml = """ +branches: + only: + - master + - /.*-travis-.*/ + +os: linux +dist: trusty +language: generic +""" + +_default_circle_yml = """ +version: 2 + +jobs: + build: + machine: true + +workflows: + version: 2 + build: + jobs: + - build: + filters: + branches: + only: + - /.*-circle-.*/ +""" + +_default_tree = { + '.travis.yml': _default_travis_yml, + '.circleci/config.yml': _default_circle_yml +} + + +class GitRemoteCallbacks(PygitRemoteCallbacks): + + def __init__(self, token): + self.token = token + self.attempts = 0 + super().__init__() + + def push_update_reference(self, refname, message): + pass + + def update_tips(self, refname, old, new): + pass + + def credentials(self, url, username_from_url, allowed_types): + # its a libgit2 bug, that it infinitely retries the authentication + self.attempts += 1 + + if self.attempts >= 5: + # pygit2 doesn't propagate the exception properly + msg = 'Wrong oauth personal access token' + print(msg) + raise CrossbowError(msg) + + if (allowed_types & + pygit2.credentials.GIT_CREDENTIAL_USERPASS_PLAINTEXT): + return pygit2.UserPass(self.token, 'x-oauth-basic') + else: + return None + + +def _git_ssh_to_https(url): + return url.replace('git@github.com:', 'https://github.com/') + + +class Repo: + """ + Base class for interaction with local git repositories + + A high level wrapper used for both reading revision information from + arrow's repository and pushing continuous integration tasks to the queue + repository. + + Parameters + ---------- + require_https : boolean, default False + Raise exception for SSH origin URLs + """ + + def __init__(self, path, github_token=None, remote_url=None, + require_https=False): + self.path = Path(path) + self.github_token = github_token + self.require_https = require_https + self._remote_url = remote_url + self._pygit_repo = None + self._github_repo = None # set by as_github_repo() + self._updated_refs = [] + + def __str__(self): + tpl = textwrap.dedent(''' + Repo: {remote}@{branch} + Commit: {head} + ''') + return tpl.format( + remote=self.remote_url, + branch=self.branch.branch_name, + head=self.head + ) + + @property + def repo(self): + if self._pygit_repo is None: + self._pygit_repo = pygit2.Repository(str(self.path)) + return self._pygit_repo + + @property + def origin(self): + remote = self.repo.remotes['origin'] + if self.require_https and remote.url.startswith('git@github.com'): + raise CrossbowError("Change SSH origin URL to HTTPS to use " + "Crossbow: {}".format(remote.url)) + return remote + + def fetch(self): + refspec = '+refs/heads/*:refs/remotes/origin/*' + self.origin.fetch([refspec]) + + def push(self, refs=None, github_token=None): + github_token = github_token or self.github_token + if github_token is None: + raise RuntimeError( + 'Could not determine GitHub token. Please set the ' + 'CROSSBOW_GITHUB_TOKEN environment variable to a ' + 'valid GitHub access token or pass one to --github-token.' + ) + callbacks = GitRemoteCallbacks(github_token) + refs = refs or [] + try: + self.origin.push(refs + self._updated_refs, callbacks=callbacks) + except pygit2.GitError: + raise RuntimeError('Failed to push updated references, ' + 'potentially because of credential issues: {}' + .format(self._updated_refs)) + else: + self.updated_refs = [] + + @property + def head(self): + """Currently checked out commit's sha""" + return self.repo.head + + @property + def branch(self): + """Currently checked out branch""" + try: + return self.repo.branches[self.repo.head.shorthand] + except KeyError: + return None # detached + + @property + def remote(self): + """Currently checked out branch's remote counterpart""" + try: + return self.repo.remotes[self.branch.upstream.remote_name] + except (AttributeError, KeyError): + return None # cannot detect + + @property + def remote_url(self): + """Currently checked out branch's remote counterpart URL + + If an SSH github url is set, it will be replaced by the https + equivalent usable with GitHub OAuth token. + """ + try: + return self._remote_url or _git_ssh_to_https(self.remote.url) + except AttributeError: + return None + + @property + def user_name(self): + try: + return next(self.repo.config.get_multivar('user.name')) + except StopIteration: + return os.environ.get('GIT_COMMITTER_NAME', 'unknown') + + @property + def user_email(self): + try: + return next(self.repo.config.get_multivar('user.email')) + except StopIteration: + return os.environ.get('GIT_COMMITTER_EMAIL', 'unknown') + + @property + def signature(self): + return pygit2.Signature(self.user_name, self.user_email, + int(time.time())) + + def create_tree(self, files): + builder = self.repo.TreeBuilder() + + for filename, content in files.items(): + if isinstance(content, dict): + # create a subtree + tree_id = self.create_tree(content) + builder.insert(filename, tree_id, pygit2.GIT_FILEMODE_TREE) + else: + # create a file + blob_id = self.repo.create_blob(content) + builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB) + + tree_id = builder.write() + return tree_id + + def create_commit(self, files, parents=None, message='', + reference_name=None): + if parents is None: + # by default use the main branch as the base of the new branch + # required to reuse github actions cache across crossbow tasks + commit, _ = self.repo.resolve_refish("master") + parents = [commit.id] + tree_id = self.create_tree(files) + + author = committer = self.signature + commit_id = self.repo.create_commit(reference_name, author, committer, + message, tree_id, parents) + return self.repo[commit_id] + + def create_branch(self, branch_name, files, parents=None, message='', + signature=None): + # create commit with the passed tree + commit = self.create_commit(files, parents=parents, message=message) + + # create branch pointing to the previously created commit + branch = self.repo.create_branch(branch_name, commit) + + # append to the pushable references + self._updated_refs.append('refs/heads/{}'.format(branch_name)) + + return branch + + def create_tag(self, tag_name, commit_id, message=''): + tag_id = self.repo.create_tag(tag_name, commit_id, + pygit2.GIT_OBJ_COMMIT, self.signature, + message) + + # append to the pushable references + self._updated_refs.append('refs/tags/{}'.format(tag_name)) + + return self.repo[tag_id] + + def file_contents(self, commit_id, file): + commit = self.repo[commit_id] + entry = commit.tree[file] + blob = self.repo[entry.id] + return blob.data + + def _parse_github_user_repo(self): + m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url) + if m is None: + raise CrossbowError( + "Unable to parse the github owner and repository from the " + "repository's remote url '{}'".format(self.remote_url) + ) + user, repo = m.group(1), m.group(2) + return user, repo + + def as_github_repo(self, github_token=None): + """Converts it to a repository object which wraps the GitHub API""" + if self._github_repo is None: + if not _have_github3: + raise ImportError('Must install github3.py') + github_token = github_token or self.github_token + username, reponame = self._parse_github_user_repo() + session = github3.session.GitHubSession( + default_connect_timeout=10, + default_read_timeout=30 + ) + github = github3.GitHub(session=session) + github.login(token=github_token) + self._github_repo = github.repository(username, reponame) + return self._github_repo + + def github_commit(self, sha): + repo = self.as_github_repo() + return repo.commit(sha) + + def github_release(self, tag): + repo = self.as_github_repo() + try: + return repo.release_from_tag(tag) + except github3.exceptions.NotFoundError: + return None + + def github_upload_asset_requests(self, release, path, name, mime, + max_retries=None, retry_backoff=None): + if max_retries is None: + max_retries = int(os.environ.get('CROSSBOW_MAX_RETRIES', 8)) + if retry_backoff is None: + retry_backoff = int(os.environ.get('CROSSBOW_RETRY_BACKOFF', 5)) + + for i in range(max_retries): + try: + with open(path, 'rb') as fp: + result = release.upload_asset(name=name, asset=fp, + content_type=mime) + except github3.exceptions.ResponseError as e: + logger.error('Attempt {} has failed with message: {}.' + .format(i + 1, str(e))) + logger.error('Error message {}'.format(e.msg)) + logger.error('List of errors provided by Github:') + for err in e.errors: + logger.error(' - {}'.format(err)) + + if e.code == 422: + # 422 Validation Failed, probably raised because + # ReleaseAsset already exists, so try to remove it before + # reattempting the asset upload + for asset in release.assets(): + if asset.name == name: + logger.info('Release asset {} already exists, ' + 'removing it...'.format(name)) + asset.delete() + logger.info('Asset {} removed.'.format(name)) + break + except github3.exceptions.ConnectionError as e: + logger.error('Attempt {} has failed with message: {}.' + .format(i + 1, str(e))) + else: + logger.info('Attempt {} has finished.'.format(i + 1)) + return result + + time.sleep(retry_backoff) + + raise RuntimeError('Github asset uploading has failed!') + + def github_upload_asset_curl(self, release, path, name, mime): + upload_url, _ = release.upload_url.split('{?') + upload_url += '?name={}'.format(name) + + command = [ + 'curl', + '--fail', + '-H', "Authorization: token {}".format(self.github_token), + '-H', "Content-Type: {}".format(mime), + '--data-binary', '@{}'.format(path), + upload_url + ] + return subprocess.run(command, shell=False, check=True) + + def github_overwrite_release_assets(self, tag_name, target_commitish, + patterns, method='requests'): + # Since github has changed something the asset uploading via requests + # got instable, so prefer the cURL alternative. + # Potential cause: + # sigmavirus24/github3.py/issues/779#issuecomment-379470626 + repo = self.as_github_repo() + if not tag_name: + raise CrossbowError('Empty tag name') + if not target_commitish: + raise CrossbowError('Empty target commit for the release tag') + + # remove the whole release if it already exists + try: + release = repo.release_from_tag(tag_name) + except github3.exceptions.NotFoundError: + pass + else: + release.delete() + + release = repo.create_release(tag_name, target_commitish) + for pattern in patterns: + for path in glob.glob(pattern, recursive=True): + name = os.path.basename(path) + size = os.path.getsize(path) + mime = mimetypes.guess_type(name)[0] or 'application/zip' + + logger.info( + 'Uploading asset `{}` with mimetype {} and size {}...' + .format(name, mime, size) + ) + + if method == 'requests': + self.github_upload_asset_requests(release, path, name=name, + mime=mime) + elif method == 'curl': + self.github_upload_asset_curl(release, path, name=name, + mime=mime) + else: + raise CrossbowError( + 'Unsupported upload method {}'.format(method) + ) + + +class Queue(Repo): + + def _latest_prefix_id(self, prefix): + pattern = re.compile(r'[\w\/-]*{}-(\d+)'.format(prefix)) + matches = list(filter(None, map(pattern.match, self.repo.branches))) + if matches: + latest = max(int(m.group(1)) for m in matches) + else: + latest = -1 + return latest + + def _next_job_id(self, prefix): + """Auto increments the branch's identifier based on the prefix""" + latest_id = self._latest_prefix_id(prefix) + return '{}-{}'.format(prefix, latest_id + 1) + + def latest_for_prefix(self, prefix): + latest_id = self._latest_prefix_id(prefix) + if latest_id < 0: + raise RuntimeError( + 'No job has been submitted with prefix {} yet'.format(prefix) + ) + job_name = '{}-{}'.format(prefix, latest_id) + return self.get(job_name) + + def date_of(self, job): + # it'd be better to bound to the queue repository on deserialization + # and reorganize these methods to Job + branch_name = 'origin/{}'.format(job.branch) + branch = self.repo.branches[branch_name] + commit = self.repo[branch.target] + return date.fromtimestamp(commit.commit_time) + + def jobs(self, pattern): + """Return jobs sorted by its identifier in reverse order""" + job_names = [] + for name in self.repo.branches.remote: + origin, name = name.split('/', 1) + result = re.match(pattern, name) + if result: + job_names.append(name) + + for name in sorted(job_names, reverse=True): + yield self.get(name) + + def get(self, job_name): + branch_name = 'origin/{}'.format(job_name) + branch = self.repo.branches[branch_name] + try: + content = self.file_contents(branch.target, 'job.yml') + except KeyError: + raise CrossbowError( + 'No job is found with name: {}'.format(job_name) + ) + + buffer = StringIO(content.decode('utf-8')) + job = yaml.load(buffer) + job.queue = self + return job + + def put(self, job, prefix='build'): + if not isinstance(job, Job): + raise CrossbowError('`job` must be an instance of Job') + if job.branch is not None: + raise CrossbowError('`job.branch` is automatically generated, ' + 'thus it must be blank') + + if job.target.remote is None: + raise CrossbowError( + 'Cannot determine git remote for the Arrow repository to ' + 'clone or push to, try to push the `{}` branch first to have ' + 'a remote tracking counterpart.'.format(job.target.branch) + ) + if job.target.branch is None: + raise CrossbowError( + 'Cannot determine the current branch of the Arrow repository ' + 'to clone or push to, perhaps it is in detached HEAD state. ' + 'Please checkout a branch.' + ) + + # auto increment and set next job id, e.g. build-85 + job._queue = self + job.branch = self._next_job_id(prefix) + + # create tasks' branches + for task_name, task in job.tasks.items(): + # adding CI's name to the end of the branch in order to use skip + # patterns on travis and circleci + task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name) + params = { + **job.params, + "arrow": job.target, + "queue_remote_url": self.remote_url + } + files = task.render_files(job.template_searchpath, params=params) + branch = self.create_branch(task.branch, files=files) + self.create_tag(task.tag, branch.target) + task.commit = str(branch.target) + + # create job's branch with its description + return self.create_branch(job.branch, files=job.render_files()) + + +def get_version(root, **kwargs): + """ + Parse function for setuptools_scm that ignores tags for non-C++ + subprojects, e.g. apache-arrow-js-XXX tags. + """ + from setuptools_scm.git import parse as parse_git_version + + # query the calculated version based on the git tags + kwargs['describe_command'] = ( + 'git describe --dirty --tags --long --match "apache-arrow-[0-9].*"' + ) + version = parse_git_version(root, **kwargs) + tag = str(version.tag) + + # We may get a development tag for the next version, such as "5.0.0.dev0", + # or the tag of an already released version, such as "4.0.0". + # In the latter case, we need to increment the version so that the computed + # version comes after any patch release (the next feature version after + # 4.0.0 is 5.0.0). + pattern = r"^(\d+)\.(\d+)\.(\d+)" + match = re.match(pattern, tag) + major, minor, patch = map(int, match.groups()) + if 'dev' not in tag: + major += 1 + + return "{}.{}.{}.dev{}".format(major, minor, patch, version.distance) + + +class Serializable: + + @classmethod + def to_yaml(cls, representer, data): + tag = '!{}'.format(cls.__name__) + dct = {k: v for k, v in data.__dict__.items() if not k.startswith('_')} + return representer.represent_mapping(tag, dct) + + +class Target(Serializable): + """ + Describes target repository and revision the builds run against + + This serializable data container holding information about arrow's + git remote, branch, sha and version number as well as some metadata + (currently only an email address where the notification should be sent). + """ + + def __init__(self, head, branch, remote, version, email=None): + self.head = head + self.email = email + self.branch = branch + self.remote = remote + self.version = version + self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html + # + # > A pre-release version number MAY be denoted by appending an + # > arbitrary string immediately following the patch version and a + # > dash. The string MUST be comprised of only alphanumerics plus + # > dash [0-9A-Za-z-]. + # + # Example: + # + # '0.16.1.dev10' -> + # '0.16.1-dev10' + self.no_rc_semver_version = \ + re.sub(r'\.(dev\d+)\Z', r'-\1', self.no_rc_version) + + @classmethod + def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, + email=None): + """Initialize from a repository + + Optionally override detected remote, branch, head, and/or version. + """ + assert isinstance(repo, Repo) + + if head is None: + head = str(repo.head.target) + if branch is None: + branch = repo.branch.branch_name + if remote is None: + remote = repo.remote_url + if version is None: + version = get_version(repo.path) + if email is None: + email = repo.user_email + + return cls(head=head, email=email, branch=branch, remote=remote, + version=version) + + +class Task(Serializable): + """ + Describes a build task and metadata required to render CI templates + + A task is represented as a single git commit and branch containing jinja2 + rendered files (currently appveyor.yml or .travis.yml configurations). + + A task can't be directly submitted to a queue, must belong to a job. + Each task's unique identifier is its branch name, which is generated after + submitting the job to a queue. + """ + + def __init__(self, ci, template, artifacts=None, params=None): + assert ci in { + 'circle', + 'travis', + 'appveyor', + 'azure', + 'github', + 'drone', + } + self.ci = ci + self.template = template + self.artifacts = artifacts or [] + self.params = params or {} + self.branch = None # filled after adding to a queue + self.commit = None # filled after adding to a queue + self._queue = None # set by the queue object after put or get + self._status = None # status cache + self._assets = None # assets cache + + def render_files(self, searchpath, params=None): + params = {**self.params, **(params or {}), "task": self} + try: + rendered = _render_jinja_template(searchpath, self.template, + params=params) + except jinja2.TemplateError as e: + raise RuntimeError( + 'Failed to render template `{}` with {}: {}'.format( + self.template, e.__class__.__name__, str(e) + ) + ) + + tree = {**_default_tree, self.filename: rendered} + return _unflatten_tree(tree) + + @property + def tag(self): + return self.branch + + @property + def filename(self): + config_files = { + 'circle': '.circleci/config.yml', + 'travis': '.travis.yml', + 'appveyor': 'appveyor.yml', + 'azure': 'azure-pipelines.yml', + 'github': '.github/workflows/crossbow.yml', + 'drone': '.drone.yml', + } + return config_files[self.ci] + + def status(self, force_query=False): + _status = getattr(self, '_status', None) + if force_query or _status is None: + github_commit = self._queue.github_commit(self.commit) + self._status = TaskStatus(github_commit) + return self._status + + def assets(self, force_query=False, validate_patterns=True): + _assets = getattr(self, '_assets', None) + if force_query or _assets is None: + github_release = self._queue.github_release(self.tag) + self._assets = TaskAssets(github_release, + artifact_patterns=self.artifacts, + validate_patterns=validate_patterns) + return self._assets + + +class TaskStatus: + """ + Combine the results from status and checks API to a single state. + + Azure pipelines uses checks API which doesn't provide a combined + interface like status API does, so we need to manually combine + both the commit statuses and the commit checks coming from + different API endpoint + + Status.state: error, failure, pending or success, default pending + CheckRun.status: queued, in_progress or completed, default: queued + CheckRun.conclusion: success, failure, neutral, cancelled, timed_out + or action_required, only set if + CheckRun.status == 'completed' + + 1. Convert CheckRun's status and conclusion to one of Status.state + 2. Merge the states based on the following rules: + - failure if any of the contexts report as error or failure + - pending if there are no statuses or a context is pending + - success if the latest status for all contexts is success + error otherwise. + + Parameters + ---------- + commit : github3.Commit + Commit to query the combined status for. + + Returns + ------- + TaskStatus( + combined_state='error|failure|pending|success', + github_status='original github status object', + github_check_runs='github checks associated with the commit', + total_count='number of statuses and checks' + ) + """ + + def __init__(self, commit): + status = commit.status() + check_runs = list(commit.check_runs()) + states = [s.state for s in status.statuses] + + for check in check_runs: + if check.status == 'completed': + if check.conclusion in {'success', 'failure'}: + states.append(check.conclusion) + elif check.conclusion in {'cancelled', 'timed_out', + 'action_required'}: + states.append('error') + # omit `neutral` conclusion + else: + states.append('pending') + + # it could be more effective, but the following is more descriptive + combined_state = 'error' + if len(states): + if any(state in {'error', 'failure'} for state in states): + combined_state = 'failure' + elif any(state == 'pending' for state in states): + combined_state = 'pending' + elif all(state == 'success' for state in states): + combined_state = 'success' + + # show link to the actual build, some of the CI providers implement + # the statuses API others implement the checks API, so display both + build_links = [s.target_url for s in status.statuses] + build_links += [c.html_url for c in check_runs] + + self.combined_state = combined_state + self.github_status = status + self.github_check_runs = check_runs + self.total_count = len(states) + self.build_links = build_links + + +class TaskAssets(dict): + + def __init__(self, github_release, artifact_patterns, + validate_patterns=True): + # HACK(kszucs): don't expect uploaded assets of no atifacts were + # defiened for the tasks in order to spare a bit of github rate limit + if not artifact_patterns: + return + + if github_release is None: + github_assets = {} # no assets have been uploaded for the task + else: + github_assets = {a.name: a for a in github_release.assets()} + + if not validate_patterns: + # shortcut to avoid pattern validation and just set all artifacts + return self.update(github_assets) + + for pattern in artifact_patterns: + # artifact can be a regex pattern + compiled = re.compile(f"^{pattern}$") + matches = list( + filter(None, map(compiled.match, github_assets.keys())) + ) + num_matches = len(matches) + + # validate artifact pattern matches single asset + if num_matches == 0: + self[pattern] = None + elif num_matches == 1: + self[pattern] = github_assets[matches[0].group(0)] + else: + raise CrossbowError( + 'Only a single asset should match pattern `{}`, there are ' + 'multiple ones: {}'.format(pattern, ', '.join(matches)) + ) + + def missing_patterns(self): + return [pattern for pattern, asset in self.items() if asset is None] + + def uploaded_assets(self): + return [asset for asset in self.values() if asset is not None] + + +class Job(Serializable): + """Describes multiple tasks against a single target repository""" + + def __init__(self, target, tasks, params=None, template_searchpath=None): + if not tasks: + raise ValueError('no tasks were provided for the job') + if not all(isinstance(task, Task) for task in tasks.values()): + raise ValueError('each `tasks` mus be an instance of Task') + if not isinstance(target, Target): + raise ValueError('`target` must be an instance of Target') + if not isinstance(target, Target): + raise ValueError('`target` must be an instance of Target') + if not isinstance(params, dict): + raise ValueError('`params` must be an instance of dict') + + self.target = target + self.tasks = tasks + self.params = params or {} # additional parameters for the tasks + self.branch = None # filled after adding to a queue + self._queue = None # set by the queue object after put or get + if template_searchpath is None: + self._template_searchpath = ArrowSources.find().path + else: + self._template_searchpath = template_searchpath + + def render_files(self): + with StringIO() as buf: + yaml.dump(self, buf) + content = buf.getvalue() + tree = {**_default_tree, "job.yml": content} + return _unflatten_tree(tree) + + def render_tasks(self, params=None): + result = {} + params = { + **self.params, + "arrow": self.target, + **(params or {}) + } + for task_name, task in self.tasks.items(): + files = task.render_files(self._template_searchpath, params) + result[task_name] = files + return result + + @property + def template_searchpath(self): + return self._template_searchpath + + @property + def queue(self): + assert isinstance(self._queue, Queue) + return self._queue + + @queue.setter + def queue(self, queue): + assert isinstance(queue, Queue) + self._queue = queue + for task in self.tasks.values(): + task._queue = queue + + @property + def email(self): + return os.environ.get('CROSSBOW_EMAIL', self.target.email) + + @property + def date(self): + return self.queue.date_of(self) + + def show(self, stream=None): + return yaml.dump(self, stream=stream) + + @classmethod + def from_config(cls, config, target, tasks=None, groups=None, params=None): + """ + Intantiate a job from based on a config. + + Parameters + ---------- + config : dict + Deserialized content of tasks.yml + target : Target + Describes target repository and revision the builds run against. + tasks : Optional[List[str]], default None + List of glob patterns for matching task names. + groups : Optional[List[str]], default None + List of exact group names matching predefined task sets in the + config. + params : Optional[Dict[str, str]], default None + Additional rendering parameters for the task templates. + + Returns + ------- + Job + + Raises + ------ + Exception: + If invalid groups or tasks has been passed. + """ + task_definitions = config.select(tasks, groups=groups) + + # instantiate the tasks + tasks = {} + versions = {'version': target.version, + 'no_rc_version': target.no_rc_version, + 'no_rc_semver_version': target.no_rc_semver_version} + for task_name, task in task_definitions.items(): + artifacts = task.pop('artifacts', None) or [] # because of yaml + artifacts = [fn.format(**versions) for fn in artifacts] + tasks[task_name] = Task(artifacts=artifacts, **task) + + return cls(target=target, tasks=tasks, params=params, + template_searchpath=config.template_searchpath) + + def is_finished(self): + for task in self.tasks.values(): + status = task.status(force_query=True) + if status.combined_state == 'pending': + return False + return True + + def wait_until_finished(self, poll_max_minutes=120, + poll_interval_minutes=10): + started_at = time.time() + while True: + if self.is_finished(): + break + + waited_for_minutes = (time.time() - started_at) / 60 + if waited_for_minutes > poll_max_minutes: + msg = ('Exceeded the maximum amount of time waiting for job ' + 'to finish, waited for {} minutes.') + raise RuntimeError(msg.format(waited_for_minutes)) + + logger.info('Waiting {} minutes and then checking again' + .format(poll_interval_minutes)) + time.sleep(poll_interval_minutes * 60) + + +class Config(dict): + + def __init__(self, tasks, template_searchpath): + super().__init__(tasks) + self.template_searchpath = template_searchpath + + @classmethod + def load_yaml(cls, path): + path = Path(path) + searchpath = path.parent + rendered = _render_jinja_template(searchpath, template=path.name, + params={}) + config = yaml.load(rendered) + return cls(config, template_searchpath=searchpath) + + def show(self, stream=None): + return yaml.dump(dict(self), stream=stream) + + def select(self, tasks=None, groups=None): + config_groups = dict(self['groups']) + config_tasks = dict(self['tasks']) + valid_groups = set(config_groups.keys()) + valid_tasks = set(config_tasks.keys()) + group_whitelist = list(groups or []) + task_whitelist = list(tasks or []) + + # validate that the passed groups are defined in the config + requested_groups = set(group_whitelist) + invalid_groups = requested_groups - valid_groups + if invalid_groups: + msg = 'Invalid group(s) {!r}. Must be one of {!r}'.format( + invalid_groups, valid_groups + ) + raise CrossbowError(msg) + + # merge the tasks defined in the selected groups + task_patterns = [list(config_groups[name]) for name in group_whitelist] + task_patterns = set(sum(task_patterns, task_whitelist)) + + # treat the task names as glob patterns to select tasks more easily + requested_tasks = set() + for pattern in task_patterns: + matches = fnmatch.filter(valid_tasks, pattern) + if len(matches): + requested_tasks.update(matches) + else: + raise CrossbowError( + "Unable to match any tasks for `{}`".format(pattern) + ) + + # validate that the passed and matched tasks are defined in the config + invalid_tasks = requested_tasks - valid_tasks + if invalid_tasks: + msg = 'Invalid task(s) {!r}. Must be one of {!r}'.format( + invalid_tasks, valid_tasks + ) + raise CrossbowError(msg) + + return { + task_name: config_tasks[task_name] for task_name in requested_tasks + } + + def validate(self): + # validate that the task groups are properly referening the tasks + for group_name, group in self['groups'].items(): + for pattern in group: + tasks = self.select(tasks=[pattern]) + if not tasks: + raise CrossbowError( + "The pattern `{}` defined for task group `{}` is not " + "matching any of the tasks defined in the " + "configuration file.".format(pattern, group_name) + ) + + # validate that the tasks are constructible + for task_name, task in self['tasks'].items(): + try: + Task(**task) + except Exception as e: + raise CrossbowError( + 'Unable to construct a task object from the ' + 'definition of task `{}`. The original error message ' + 'is: `{}`'.format(task_name, str(e)) + ) + + # validate that the defined tasks are renderable, in order to to that + # define the required object with dummy data + target = Target( + head='e279a7e06e61c14868ca7d71dea795420aea6539', + branch='master', + remote='https://github.com/apache/arrow', + version='1.0.0dev123', + email='dummy@example.ltd' + ) + + for task_name, task in self['tasks'].items(): + task = Task(**task) + files = task.render_files( + self.template_searchpath, + params=dict( + arrow=target, + queue_remote_url='https://github.com/org/crossbow' + ) + ) + if not files: + raise CrossbowError('No files have been rendered for task `{}`' + .format(task_name)) + + +# configure yaml serializer +yaml = YAML() +yaml.register_class(Job) +yaml.register_class(Task) +yaml.register_class(Target) diff --git a/src/arrow/dev/archery/archery/crossbow/reports.py b/src/arrow/dev/archery/archery/crossbow/reports.py new file mode 100644 index 000000000..f86a67a74 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/reports.py @@ -0,0 +1,315 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import click +import collections +import operator +import fnmatch +import functools +from io import StringIO +import textwrap + + +# TODO(kszucs): use archery.report.JinjaReport instead +class Report: + + def __init__(self, job, task_filters=None): + self.job = job + + tasks = sorted(job.tasks.items()) + if task_filters: + filtered = set() + for pattern in task_filters: + filtered |= set(fnmatch.filter(job.tasks.keys(), pattern)) + + tasks = [(name, task) for name, task in tasks if name in filtered] + + self._tasks = dict(tasks) + + @property + def tasks(self): + return self._tasks + + def show(self): + raise NotImplementedError() + + +class ConsoleReport(Report): + """Report the status of a Job to the console using click""" + + # output table's header template + HEADER = '[{state:>7}] {branch:<52} {content:>16}' + DETAILS = ' └ {url}' + + # output table's row template for assets + ARTIFACT_NAME = '{artifact:>69} ' + ARTIFACT_STATE = '[{state:>7}]' + + # state color mapping to highlight console output + COLORS = { + # from CombinedStatus + 'error': 'red', + 'failure': 'red', + 'pending': 'yellow', + 'success': 'green', + # custom state messages + 'ok': 'green', + 'missing': 'red' + } + + def lead(self, state, branch, n_uploaded, n_expected): + line = self.HEADER.format( + state=state.upper(), + branch=branch, + content='uploaded {} / {}'.format(n_uploaded, n_expected) + ) + return click.style(line, fg=self.COLORS[state.lower()]) + + def header(self): + header = self.HEADER.format( + state='state', + branch='Task / Branch', + content='Artifacts' + ) + delimiter = '-' * len(header) + return '{}\n{}'.format(header, delimiter) + + def artifact(self, state, pattern, asset): + if asset is None: + artifact = pattern + state = 'pending' if state == 'pending' else 'missing' + else: + artifact = asset.name + state = 'ok' + + name_ = self.ARTIFACT_NAME.format(artifact=artifact) + state_ = click.style( + self.ARTIFACT_STATE.format(state=state.upper()), + self.COLORS[state] + ) + return name_ + state_ + + def show(self, outstream, asset_callback=None, validate_patterns=True): + echo = functools.partial(click.echo, file=outstream) + + # write table's header + echo(self.header()) + + # write table's body + for task_name, task in self.tasks.items(): + # write summary of the uploaded vs total assets + status = task.status() + assets = task.assets(validate_patterns=validate_patterns) + + # mapping of artifact pattern to asset or None of not uploaded + n_expected = len(task.artifacts) + n_uploaded = len(assets.uploaded_assets()) + echo(self.lead(status.combined_state, task_name, n_uploaded, + n_expected)) + + # show link to the actual build, some of the CI providers implement + # the statuses API others implement the checks API, so display both + for link in status.build_links: + echo(self.DETAILS.format(url=link)) + + # write per asset status + for artifact_pattern, asset in assets.items(): + if asset_callback is not None: + asset_callback(task_name, task, asset) + echo(self.artifact(status.combined_state, artifact_pattern, + asset)) + + +class EmailReport(Report): + + HEADER = textwrap.dedent(""" + Arrow Build Report for Job {job_name} + + All tasks: {all_tasks_url} + """) + + TASK = textwrap.dedent(""" + - {name}: + URL: {url} + """).strip() + + EMAIL = textwrap.dedent(""" + From: {sender_name} <{sender_email}> + To: {recipient_email} + Subject: {subject} + + {body} + """).strip() + + STATUS_HEADERS = { + # from CombinedStatus + 'error': 'Errored Tasks:', + 'failure': 'Failed Tasks:', + 'pending': 'Pending Tasks:', + 'success': 'Succeeded Tasks:', + } + + def __init__(self, job, sender_name, sender_email, recipient_email): + self.sender_name = sender_name + self.sender_email = sender_email + self.recipient_email = recipient_email + super().__init__(job) + + def url(self, query): + repo_url = self.job.queue.remote_url.strip('.git') + return '{}/branches/all?query={}'.format(repo_url, query) + + def listing(self, tasks): + return '\n'.join( + sorted( + self.TASK.format(name=task_name, url=self.url(task.branch)) + for task_name, task in tasks.items() + ) + ) + + def header(self): + url = self.url(self.job.branch) + return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url) + + def subject(self): + return ( + "[NIGHTLY] Arrow Build Report for Job {}".format(self.job.branch) + ) + + def body(self): + buffer = StringIO() + buffer.write(self.header()) + + tasks_by_state = collections.defaultdict(dict) + for task_name, task in self.job.tasks.items(): + state = task.status().combined_state + tasks_by_state[state][task_name] = task + + for state in ('failure', 'error', 'pending', 'success'): + if state in tasks_by_state: + tasks = tasks_by_state[state] + buffer.write('\n') + buffer.write(self.STATUS_HEADERS[state]) + buffer.write('\n') + buffer.write(self.listing(tasks)) + buffer.write('\n') + + return buffer.getvalue() + + def email(self): + return self.EMAIL.format( + sender_name=self.sender_name, + sender_email=self.sender_email, + recipient_email=self.recipient_email, + subject=self.subject(), + body=self.body() + ) + + def show(self, outstream): + outstream.write(self.email()) + + def send(self, smtp_user, smtp_password, smtp_server, smtp_port): + import smtplib + + email = self.email() + + server = smtplib.SMTP_SSL(smtp_server, smtp_port) + server.ehlo() + server.login(smtp_user, smtp_password) + server.sendmail(smtp_user, self.recipient_email, email) + server.close() + + +class CommentReport(Report): + + _markdown_badge = '[]({url})' + + badges = { + 'github': _markdown_badge.format( + title='Github Actions', + url='https://github.com/{repo}/actions?query=branch:{branch}', + badge=( + 'https://github.com/{repo}/workflows/Crossbow/' + 'badge.svg?branch={branch}' + ), + ), + 'azure': _markdown_badge.format( + title='Azure', + url=( + 'https://dev.azure.com/{repo}/_build/latest' + '?definitionId=1&branchName={branch}' + ), + badge=( + 'https://dev.azure.com/{repo}/_apis/build/status/' + '{repo_dotted}?branchName={branch}' + ) + ), + 'travis': _markdown_badge.format( + title='TravisCI', + url='https://travis-ci.com/{repo}/branches', + badge='https://img.shields.io/travis/{repo}/{branch}.svg' + ), + 'circle': _markdown_badge.format( + title='CircleCI', + url='https://circleci.com/gh/{repo}/tree/{branch}', + badge=( + 'https://img.shields.io/circleci/build/github' + '/{repo}/{branch}.svg' + ) + ), + 'appveyor': _markdown_badge.format( + title='Appveyor', + url='https://ci.appveyor.com/project/{repo}/history', + badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg' + ), + 'drone': _markdown_badge.format( + title='Drone', + url='https://cloud.drone.io/{repo}', + badge='https://img.shields.io/drone/build/{repo}/{branch}.svg' + ), + } + + def __init__(self, job, crossbow_repo): + self.crossbow_repo = crossbow_repo + super().__init__(job) + + def show(self): + url = 'https://github.com/{repo}/branches/all?query={branch}' + sha = self.job.target.head + + msg = 'Revision: {}\n\n'.format(sha) + msg += 'Submitted crossbow builds: [{repo} @ {branch}]' + msg += '({})\n'.format(url) + msg += '\n|Task|Status|\n|----|------|' + + tasks = sorted(self.job.tasks.items(), key=operator.itemgetter(0)) + for key, task in tasks: + branch = task.branch + + try: + template = self.badges[task.ci] + badge = template.format( + repo=self.crossbow_repo, + repo_dotted=self.crossbow_repo.replace('/', '.'), + branch=branch + ) + except KeyError: + badge = 'unsupported CI service `{}`'.format(task.ci) + + msg += '\n|{}|{}|'.format(key, badge) + + return msg.format(repo=self.crossbow_repo, branch=self.job.branch) diff --git a/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml new file mode 100644 index 000000000..c37c7b553 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml @@ -0,0 +1,51 @@ +!Job +target: !Target + head: f766a1d615dd1b7ee706d05102e579195951a61c + email: unkown + branch: refs/pull/4435/merge + remote: https://github.com/apache/arrow + version: 0.13.0.dev306 + no_rc_version: 0.13.0.dev306 +tasks: + docker-cpp-cmake32: !Task + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + artifacts: [] + params: + commands: + - docker-compose build cpp-cmake32 + - docker-compose run cpp-cmake32 + branch: ursabot-1-circle-docker-cpp-cmake32 + commit: a56b077c8d1b891a7935048e5672bf6fc07599ec + wheel-osx-cp37m: !Task + ci: travis + platform: osx + template: python-wheels/travis.osx.yml + artifacts: + - pyarrow-0.13.0.dev306-cp37-cp37m-macosx_10_6_intel.whl + params: + python_version: 3.7 + branch: ursabot-1-travis-wheel-osx-cp37m + commit: a56b077c8d1b891a7935048e5672bf6fc07599ec + wheel-osx-cp36m: !Task + ci: travis + platform: osx + template: python-wheels/travis.osx.yml + artifacts: + - pyarrow-0.13.0.dev306-cp36-cp36m-macosx_10_6_intel.whl + params: + python_version: 3.6 + branch: ursabot-1-travis-wheel-osx-cp36m + commit: a56b077c8d1b891a7935048e5672bf6fc07599ec + wheel-win-cp36m: !Task + ci: appveyor + platform: win + template: python-wheels/appveyor.yml + artifacts: + - pyarrow-0.13.0.dev306-cp36-cp36m-win_amd64.whl + params: + python_version: 3.6 + branch: ursabot-1-appveyor-wheel-win-cp36m + commit: a56b077c8d1b891a7935048e5672bf6fc07599ec +branch: ursabot-1 diff --git a/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md new file mode 100644 index 000000000..15825218c --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md @@ -0,0 +1,10 @@ +Revision: {revision} + +Submitted crossbow builds: [{repo} @ {branch}](https://github.com/{repo}/branches/all?query={branch}) + +|Task|Status| +|----|------| +|docker-cpp-cmake32|[](https://circleci.com/gh/{repo}/tree/{branch}-circle-docker-cpp-cmake32)| +|wheel-osx-cp36m|[](https://travis-ci.com/{repo}/branches)| +|wheel-osx-cp37m|[](https://travis-ci.com/{repo}/branches)| +|wheel-win-cp36m|[](https://ci.appveyor.com/project/{repo}/history)| diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_core.py b/src/arrow/dev/archery/archery/crossbow/tests/test_core.py new file mode 100644 index 000000000..518474236 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/tests/test_core.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from archery.utils.source import ArrowSources +from archery.crossbow import Config + + +def test_config(): + src = ArrowSources.find() + conf = Config.load_yaml(src.dev / "tasks" / "tasks.yml") + conf.validate() diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py b/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py new file mode 100644 index 000000000..ee9ba1ee2 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/tests/test_crossbow_cli.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from click.testing import CliRunner +import pytest + +from archery.crossbow.cli import crossbow +from archery.utils.git import git + + +@pytest.mark.integration +def test_crossbow_submit(tmp_path): + runner = CliRunner() + + def invoke(*args): + return runner.invoke(crossbow, ['--queue-path', str(tmp_path), *args]) + + # initialize an empty crossbow repository + git.run_cmd("init", str(tmp_path)) + git.run_cmd("-C", str(tmp_path), "remote", "add", "origin", + "https://github.com/dummy/repo") + git.run_cmd("-C", str(tmp_path), "commit", "-m", "initial", + "--allow-empty") + + result = invoke('check-config') + assert result.exit_code == 0 + + result = invoke('submit', '--no-fetch', '--no-push', '-g', 'wheel') + assert result.exit_code == 0 diff --git a/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py b/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py new file mode 100644 index 000000000..0df292bb5 --- /dev/null +++ b/src/arrow/dev/archery/archery/crossbow/tests/test_reports.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import textwrap + +from archery.crossbow.core import yaml +from archery.crossbow.reports import CommentReport + + +def test_crossbow_comment_formatter(load_fixture): + msg = load_fixture('crossbow-success-message.md') + job = load_fixture('crossbow-job.yaml', decoder=yaml.load) + + report = CommentReport(job, crossbow_repo='ursa-labs/crossbow') + expected = msg.format( + repo='ursa-labs/crossbow', + branch='ursabot-1', + revision='f766a1d615dd1b7ee706d05102e579195951a61c', + status='has been succeeded.' + ) + assert report.show() == textwrap.dedent(expected).strip() diff --git a/src/arrow/dev/archery/archery/docker.py b/src/arrow/dev/archery/archery/docker.py new file mode 100644 index 000000000..17d4c713a --- /dev/null +++ b/src/arrow/dev/archery/archery/docker.py @@ -0,0 +1,402 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import subprocess +from io import StringIO + +from dotenv import dotenv_values +from ruamel.yaml import YAML + +from .utils.command import Command, default_bin +from .compat import _ensure_path + + +def flatten(node, parents=None): + parents = list(parents or []) + if isinstance(node, str): + yield (node, parents) + elif isinstance(node, list): + for value in node: + yield from flatten(value, parents=parents) + elif isinstance(node, dict): + for key, value in node.items(): + yield (key, parents) + yield from flatten(value, parents=parents + [key]) + else: + raise TypeError(node) + + +def _sanitize_command(cmd): + if isinstance(cmd, list): + cmd = " ".join(cmd) + return re.sub(r"\s+", " ", cmd) + + +class UndefinedImage(Exception): + pass + + +class ComposeConfig: + + def __init__(self, config_path, dotenv_path, compose_bin, params=None): + config_path = _ensure_path(config_path) + if dotenv_path: + dotenv_path = _ensure_path(dotenv_path) + else: + dotenv_path = config_path.parent / '.env' + self._read_env(dotenv_path, params) + self._read_config(config_path, compose_bin) + + def _read_env(self, dotenv_path, params): + """ + Read .env and merge it with explicitly passed parameters. + """ + self.dotenv = dotenv_values(str(dotenv_path)) + if params is None: + self.params = {} + else: + self.params = {k: v for k, v in params.items() if k in self.dotenv} + + # forward the process' environment variables + self.env = os.environ.copy() + # set the defaults from the dotenv files + self.env.update(self.dotenv) + # override the defaults passed as parameters + self.env.update(self.params) + + # translate docker's architecture notation to a more widely used one + arch = self.env.get('ARCH', 'amd64') + arch_aliases = { + 'amd64': 'x86_64', + 'arm64v8': 'aarch64', + 's390x': 's390x' + } + arch_short_aliases = { + 'amd64': 'x64', + 'arm64v8': 'arm64', + 's390x': 's390x' + } + self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch) + self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch) + + def _read_config(self, config_path, compose_bin): + """ + Validate and read the docker-compose.yml + """ + yaml = YAML() + with config_path.open() as fp: + config = yaml.load(fp) + + services = config['services'].keys() + self.hierarchy = dict(flatten(config.get('x-hierarchy', {}))) + self.with_gpus = config.get('x-with-gpus', []) + nodes = self.hierarchy.keys() + errors = [] + + for name in self.with_gpus: + if name not in services: + errors.append( + 'Service `{}` defined in `x-with-gpus` bot not in ' + '`services`'.format(name) + ) + for name in nodes - services: + errors.append( + 'Service `{}` is defined in `x-hierarchy` bot not in ' + '`services`'.format(name) + ) + for name in services - nodes: + errors.append( + 'Service `{}` is defined in `services` but not in ' + '`x-hierarchy`'.format(name) + ) + + # trigger docker-compose's own validation + compose = Command('docker-compose') + args = ['--file', str(config_path), 'config'] + result = compose.run(*args, env=self.env, check=False, + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + if result.returncode != 0: + # strip the intro line of docker-compose errors + errors += result.stderr.decode().splitlines() + + if errors: + msg = '\n'.join([' - {}'.format(msg) for msg in errors]) + raise ValueError( + 'Found errors with docker-compose:\n{}'.format(msg) + ) + + rendered_config = StringIO(result.stdout.decode()) + self.path = config_path + self.config = yaml.load(rendered_config) + + def get(self, service_name): + try: + service = self.config['services'][service_name] + except KeyError: + raise UndefinedImage(service_name) + service['name'] = service_name + service['need_gpu'] = service_name in self.with_gpus + service['ancestors'] = self.hierarchy[service_name] + return service + + def __getitem__(self, service_name): + return self.get(service_name) + + +class Docker(Command): + + def __init__(self, docker_bin=None): + self.bin = default_bin(docker_bin, "docker") + + +class DockerCompose(Command): + + def __init__(self, config_path, dotenv_path=None, compose_bin=None, + params=None): + compose_bin = default_bin(compose_bin, 'docker-compose') + self.config = ComposeConfig(config_path, dotenv_path, compose_bin, + params) + self.bin = compose_bin + self.pull_memory = set() + + def clear_pull_memory(self): + self.pull_memory = set() + + def _execute_compose(self, *args, **kwargs): + # execute as a docker compose command + try: + result = super().run('--file', str(self.config.path), *args, + env=self.config.env, **kwargs) + result.check_returncode() + except subprocess.CalledProcessError as e: + def formatdict(d, template): + return '\n'.join( + template.format(k, v) for k, v in sorted(d.items()) + ) + msg = ( + "`{cmd}` exited with a non-zero exit code {code}, see the " + "process log above.\n\nThe docker-compose command was " + "invoked with the following parameters:\n\nDefaults defined " + "in .env:\n{dotenv}\n\nArchery was called with:\n{params}" + ) + raise RuntimeError( + msg.format( + cmd=' '.join(e.cmd), + code=e.returncode, + dotenv=formatdict(self.config.dotenv, template=' {}: {}'), + params=formatdict( + self.config.params, template=' export {}={}' + ) + ) + ) + + def _execute_docker(self, *args, **kwargs): + # execute as a plain docker cli command + try: + result = Docker().run(*args, **kwargs) + result.check_returncode() + except subprocess.CalledProcessError as e: + raise RuntimeError( + "{} exited with non-zero exit code {}".format( + ' '.join(e.cmd), e.returncode + ) + ) + + def pull(self, service_name, pull_leaf=True, using_docker=False): + def _pull(service): + args = ['pull'] + if service['image'] in self.pull_memory: + return + + if using_docker: + try: + self._execute_docker(*args, service['image']) + except Exception as e: + # better --ignore-pull-failures handling + print(e) + else: + args.append('--ignore-pull-failures') + self._execute_compose(*args, service['name']) + + self.pull_memory.add(service['image']) + + service = self.config.get(service_name) + for ancestor in service['ancestors']: + _pull(self.config.get(ancestor)) + if pull_leaf: + _pull(service) + + def build(self, service_name, use_cache=True, use_leaf_cache=True, + using_docker=False, using_buildx=False): + def _build(service, use_cache): + if 'build' not in service: + # nothing to do + return + + args = [] + cache_from = list(service.get('build', {}).get('cache_from', [])) + if use_cache: + for image in cache_from: + if image not in self.pull_memory: + try: + self._execute_docker('pull', image) + except Exception as e: + print(e) + finally: + self.pull_memory.add(image) + else: + args.append('--no-cache') + + # turn on inline build cache, this is a docker buildx feature + # used to bundle the image build cache to the pushed image manifest + # so the build cache can be reused across hosts, documented at + # https://github.com/docker/buildx#--cache-tonametypetypekeyvalue + if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1': + args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1']) + + if using_buildx: + for k, v in service['build'].get('args', {}).items(): + args.extend(['--build-arg', '{}={}'.format(k, v)]) + + if use_cache: + cache_ref = '{}-cache'.format(service['image']) + cache_from = 'type=registry,ref={}'.format(cache_ref) + cache_to = ( + 'type=registry,ref={},mode=max'.format(cache_ref) + ) + args.extend([ + '--cache-from', cache_from, + '--cache-to', cache_to, + ]) + + args.extend([ + '--output', 'type=docker', + '-f', service['build']['dockerfile'], + '-t', service['image'], + service['build'].get('context', '.') + ]) + self._execute_docker("buildx", "build", *args) + elif using_docker: + # better for caching + for k, v in service['build'].get('args', {}).items(): + args.extend(['--build-arg', '{}={}'.format(k, v)]) + for img in cache_from: + args.append('--cache-from="{}"'.format(img)) + args.extend([ + '-f', service['build']['dockerfile'], + '-t', service['image'], + service['build'].get('context', '.') + ]) + self._execute_docker("build", *args) + else: + self._execute_compose("build", *args, service['name']) + + service = self.config.get(service_name) + # build ancestor services + for ancestor in service['ancestors']: + _build(self.config.get(ancestor), use_cache=use_cache) + # build the leaf/target service + _build(service, use_cache=use_cache and use_leaf_cache) + + def run(self, service_name, command=None, *, env=None, volumes=None, + user=None, using_docker=False): + service = self.config.get(service_name) + + args = [] + if user is not None: + args.extend(['-u', user]) + + if env is not None: + for k, v in env.items(): + args.extend(['-e', '{}={}'.format(k, v)]) + + if volumes is not None: + for volume in volumes: + args.extend(['--volume', volume]) + + if using_docker or service['need_gpu']: + # use gpus, requires docker>=19.03 + if service['need_gpu']: + args.extend(['--gpus', 'all']) + + if service.get('shm_size'): + args.extend(['--shm-size', service['shm_size']]) + + # append env variables from the compose conf + for k, v in service.get('environment', {}).items(): + args.extend(['-e', '{}={}'.format(k, v)]) + + # append volumes from the compose conf + for v in service.get('volumes', []): + if not isinstance(v, str): + # if not the compact string volume definition + v = "{}:{}".format(v['source'], v['target']) + args.extend(['-v', v]) + + # infer whether an interactive shell is desired or not + if command in ['cmd.exe', 'bash', 'sh', 'powershell']: + args.append('-it') + + # get the actual docker image name instead of the compose service + # name which we refer as image in general + args.append(service['image']) + + # add command from compose if it wasn't overridden + if command is not None: + args.append(command) + else: + # replace whitespaces from the preformatted compose command + cmd = _sanitize_command(service.get('command', '')) + if cmd: + args.append(cmd) + + # execute as a plain docker cli command + self._execute_docker('run', '--rm', *args) + else: + # execute as a docker-compose command + args.append(service_name) + if command is not None: + args.append(command) + self._execute_compose('run', '--rm', *args) + + def push(self, service_name, user=None, password=None, using_docker=False): + def _push(service): + if using_docker: + return self._execute_docker('push', service['image']) + else: + return self._execute_compose('push', service['name']) + + if user is not None: + try: + # TODO(kszucs): have an option for a prompt + self._execute_docker('login', '-u', user, '-p', password) + except subprocess.CalledProcessError: + # hide credentials + msg = ('Failed to push `{}`, check the passed credentials' + .format(service_name)) + raise RuntimeError(msg) from None + + service = self.config.get(service_name) + for ancestor in service['ancestors']: + _push(self.config.get(ancestor)) + _push(service) + + def images(self): + return sorted(self.config.hierarchy.keys()) diff --git a/src/arrow/dev/archery/archery/docker/__init__.py b/src/arrow/dev/archery/archery/docker/__init__.py new file mode 100644 index 000000000..6be29c916 --- /dev/null +++ b/src/arrow/dev/archery/archery/docker/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .core import DockerCompose, UndefinedImage # noqa diff --git a/src/arrow/dev/archery/archery/docker/cli.py b/src/arrow/dev/archery/archery/docker/cli.py new file mode 100644 index 000000000..c6b4a6473 --- /dev/null +++ b/src/arrow/dev/archery/archery/docker/cli.py @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +import click + +from ..utils.cli import validate_arrow_sources +from .core import DockerCompose, UndefinedImage + + +def _mock_compose_calls(compose): + from types import MethodType + from subprocess import CompletedProcess + + def _mock(compose, executable): + def _execute(self, *args, **kwargs): + params = ['{}={}'.format(k, v) + for k, v in self.config.params.items()] + command = ' '.join(params + [executable] + list(args)) + click.echo(command) + return CompletedProcess([], 0) + return MethodType(_execute, compose) + + compose._execute_docker = _mock(compose, executable='docker') + compose._execute_compose = _mock(compose, executable='docker-compose') + + +@click.group() +@click.option("--src", metavar="<arrow_src>", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory.") +@click.option('--dry-run/--execute', default=False, + help="Display the docker-compose commands instead of executing " + "them.") +@click.pass_context +def docker(ctx, src, dry_run): + """ + Interact with docker-compose based builds. + """ + ctx.ensure_object(dict) + + config_path = src.path / 'docker-compose.yml' + if not config_path.exists(): + raise click.ClickException( + "Docker compose configuration cannot be found in directory {}, " + "try to pass the arrow source directory explicitly.".format(src) + ) + + # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the + # environment variables to keep the usage similar to docker-compose + compose = DockerCompose(config_path, params=os.environ) + if dry_run: + _mock_compose_calls(compose) + ctx.obj['compose'] = compose + + +@docker.command("check-config") +@click.pass_obj +def check_config(obj): + """ + Validate docker-compose configuration. + """ + # executes the body of the docker function above which does the validation + # during the configuration loading + + +@docker.command('build') +@click.argument('image') +@click.option('--force-pull/--no-pull', default=True, + help="Whether to force pull the image and its ancestor images") +@click.option('--using-docker-cli', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_CLI', + help="Use docker CLI directly for building instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.option('--using-docker-buildx', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_BUILDX', + help="Use buildx with docker CLI directly for building instead " + "of calling docker-compose or the plain docker build " + "command. This option makes the build cache reusable " + "across hosts.") +@click.option('--use-cache/--no-cache', default=True, + help="Whether to use cache when building the image and its " + "ancestor images") +@click.option('--use-leaf-cache/--no-leaf-cache', default=True, + help="Whether to use cache when building only the (leaf) image " + "passed as the argument. To disable caching for both the " + "image and its ancestors use --no-cache option.") +@click.pass_obj +def docker_build(obj, image, *, force_pull, using_docker_cli, + using_docker_buildx, use_cache, use_leaf_cache): + """ + Execute docker-compose builds. + """ + compose = obj['compose'] + + using_docker_cli |= using_docker_buildx + try: + if force_pull: + compose.pull(image, pull_leaf=use_leaf_cache, + using_docker=using_docker_cli) + compose.build(image, use_cache=use_cache, + use_leaf_cache=use_leaf_cache, + using_docker=using_docker_cli, + using_buildx=using_docker_buildx, + pull_parents=force_pull) + except UndefinedImage as e: + raise click.ClickException( + "There is no service/image defined in docker-compose.yml with " + "name: {}".format(str(e)) + ) + except RuntimeError as e: + raise click.ClickException(str(e)) + + +@docker.command('run') +@click.argument('image') +@click.argument('command', required=False, default=None) +@click.option('--env', '-e', multiple=True, + help="Set environment variable within the container") +@click.option('--user', '-u', default=None, + help="Username or UID to run the container with") +@click.option('--force-pull/--no-pull', default=True, + help="Whether to force pull the image and its ancestor images") +@click.option('--force-build/--no-build', default=True, + help="Whether to force build the image and its ancestor images") +@click.option('--build-only', default=False, is_flag=True, + help="Pull and/or build the image, but do not run it") +@click.option('--using-docker-cli', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_CLI', + help="Use docker CLI directly for building instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.option('--using-docker-buildx', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_BUILDX', + help="Use buildx with docker CLI directly for building instead " + "of calling docker-compose or the plain docker build " + "command. This option makes the build cache reusable " + "across hosts.") +@click.option('--use-cache/--no-cache', default=True, + help="Whether to use cache when building the image and its " + "ancestor images") +@click.option('--use-leaf-cache/--no-leaf-cache', default=True, + help="Whether to use cache when building only the (leaf) image " + "passed as the argument. To disable caching for both the " + "image and its ancestors use --no-cache option.") +@click.option('--resource-limit', default=None, + help="A CPU/memory limit preset to mimic CI environments like " + "GitHub Actions. Implies --using-docker-cli. Note that " + "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely " + "required, unless Docker is configured with cgroups v2 " + "(else Docker will silently ignore the limits).") +@click.option('--volume', '-v', multiple=True, + help="Set volume within the container") +@click.pass_obj +def docker_run(obj, image, command, *, env, user, force_pull, force_build, + build_only, using_docker_cli, using_docker_buildx, use_cache, + use_leaf_cache, resource_limit, volume): + """ + Execute docker-compose builds. + + To see the available builds run `archery docker images`. + + Examples: + + # execute a single build + archery docker run conda-python + + # execute the builds but disable the image pulling + archery docker run --no-cache conda-python + + # pass a docker-compose parameter, like the python version + PYTHON=3.8 archery docker run conda-python + + # disable the cache only for the leaf image + PANDAS=master archery docker run --no-leaf-cache conda-python-pandas + + # entirely skip building the image + archery docker run --no-pull --no-build conda-python + + # pass runtime parameters via docker environment variables + archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp + + # set a volume + archery docker run -v $PWD/build:/build ubuntu-cpp + + # starting an interactive bash session for debugging + archery docker run ubuntu-cpp bash + """ + compose = obj['compose'] + using_docker_cli |= using_docker_buildx + + env = dict(kv.split('=', 1) for kv in env) + try: + if force_pull: + compose.pull(image, pull_leaf=use_leaf_cache, + using_docker=using_docker_cli) + if force_build: + compose.build(image, use_cache=use_cache, + use_leaf_cache=use_leaf_cache, + using_docker=using_docker_cli, + using_buildx=using_docker_buildx) + if build_only: + return + compose.run( + image, + command=command, + env=env, + user=user, + using_docker=using_docker_cli, + resource_limit=resource_limit, + volumes=volume + ) + except UndefinedImage as e: + raise click.ClickException( + "There is no service/image defined in docker-compose.yml with " + "name: {}".format(str(e)) + ) + except RuntimeError as e: + raise click.ClickException(str(e)) + + +@docker.command('push') +@click.argument('image') +@click.option('--user', '-u', required=False, envvar='ARCHERY_DOCKER_USER', + help='Docker repository username') +@click.option('--password', '-p', required=False, + envvar='ARCHERY_DOCKER_PASSWORD', + help='Docker repository password') +@click.option('--using-docker-cli', default=False, is_flag=True, + help="Use docker CLI directly for building instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.pass_obj +def docker_compose_push(obj, image, user, password, using_docker_cli): + """Push the generated docker-compose image.""" + compose = obj['compose'] + compose.push(image, user=user, password=password, + using_docker=using_docker_cli) + + +@docker.command('images') +@click.pass_obj +def docker_compose_images(obj): + """List the available docker-compose images.""" + compose = obj['compose'] + click.echo('Available images:') + for image in compose.images(): + click.echo(f' - {image}') diff --git a/src/arrow/dev/archery/archery/docker/core.py b/src/arrow/dev/archery/archery/docker/core.py new file mode 100644 index 000000000..aaf16bdfa --- /dev/null +++ b/src/arrow/dev/archery/archery/docker/core.py @@ -0,0 +1,417 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import subprocess +from io import StringIO + +from dotenv import dotenv_values +from ruamel.yaml import YAML + +from ..utils.command import Command, default_bin +from ..compat import _ensure_path + + +def flatten(node, parents=None): + parents = list(parents or []) + if isinstance(node, str): + yield (node, parents) + elif isinstance(node, list): + for value in node: + yield from flatten(value, parents=parents) + elif isinstance(node, dict): + for key, value in node.items(): + yield (key, parents) + yield from flatten(value, parents=parents + [key]) + else: + raise TypeError(node) + + +def _sanitize_command(cmd): + if isinstance(cmd, list): + cmd = " ".join(cmd) + return re.sub(r"\s+", " ", cmd) + + +class UndefinedImage(Exception): + pass + + +class ComposeConfig: + + def __init__(self, config_path, dotenv_path, compose_bin, params=None): + config_path = _ensure_path(config_path) + if dotenv_path: + dotenv_path = _ensure_path(dotenv_path) + else: + dotenv_path = config_path.parent / '.env' + self._read_env(dotenv_path, params) + self._read_config(config_path, compose_bin) + + def _read_env(self, dotenv_path, params): + """ + Read .env and merge it with explicitly passed parameters. + """ + self.dotenv = dotenv_values(str(dotenv_path)) + if params is None: + self.params = {} + else: + self.params = {k: v for k, v in params.items() if k in self.dotenv} + + # forward the process' environment variables + self.env = os.environ.copy() + # set the defaults from the dotenv files + self.env.update(self.dotenv) + # override the defaults passed as parameters + self.env.update(self.params) + + # translate docker's architecture notation to a more widely used one + arch = self.env.get('ARCH', 'amd64') + arch_aliases = { + 'amd64': 'x86_64', + 'arm64v8': 'aarch64', + 's390x': 's390x' + } + arch_short_aliases = { + 'amd64': 'x64', + 'arm64v8': 'arm64', + 's390x': 's390x' + } + self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch) + self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch) + + def _read_config(self, config_path, compose_bin): + """ + Validate and read the docker-compose.yml + """ + yaml = YAML() + with config_path.open() as fp: + config = yaml.load(fp) + + services = config['services'].keys() + self.hierarchy = dict(flatten(config.get('x-hierarchy', {}))) + self.limit_presets = config.get('x-limit-presets', {}) + self.with_gpus = config.get('x-with-gpus', []) + nodes = self.hierarchy.keys() + errors = [] + + for name in self.with_gpus: + if name not in services: + errors.append( + 'Service `{}` defined in `x-with-gpus` bot not in ' + '`services`'.format(name) + ) + for name in nodes - services: + errors.append( + 'Service `{}` is defined in `x-hierarchy` bot not in ' + '`services`'.format(name) + ) + for name in services - nodes: + errors.append( + 'Service `{}` is defined in `services` but not in ' + '`x-hierarchy`'.format(name) + ) + + # trigger docker-compose's own validation + compose = Command('docker-compose') + args = ['--file', str(config_path), 'config'] + result = compose.run(*args, env=self.env, check=False, + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + if result.returncode != 0: + # strip the intro line of docker-compose errors + errors += result.stderr.decode().splitlines() + + if errors: + msg = '\n'.join([' - {}'.format(msg) for msg in errors]) + raise ValueError( + 'Found errors with docker-compose:\n{}'.format(msg) + ) + + rendered_config = StringIO(result.stdout.decode()) + self.path = config_path + self.config = yaml.load(rendered_config) + + def get(self, service_name): + try: + service = self.config['services'][service_name] + except KeyError: + raise UndefinedImage(service_name) + service['name'] = service_name + service['need_gpu'] = service_name in self.with_gpus + service['ancestors'] = self.hierarchy[service_name] + return service + + def __getitem__(self, service_name): + return self.get(service_name) + + +class Docker(Command): + + def __init__(self, docker_bin=None): + self.bin = default_bin(docker_bin, "docker") + + +class DockerCompose(Command): + + def __init__(self, config_path, dotenv_path=None, compose_bin=None, + params=None): + compose_bin = default_bin(compose_bin, 'docker-compose') + self.config = ComposeConfig(config_path, dotenv_path, compose_bin, + params) + self.bin = compose_bin + self.pull_memory = set() + + def clear_pull_memory(self): + self.pull_memory = set() + + def _execute_compose(self, *args, **kwargs): + # execute as a docker compose command + try: + result = super().run('--file', str(self.config.path), *args, + env=self.config.env, **kwargs) + result.check_returncode() + except subprocess.CalledProcessError as e: + def formatdict(d, template): + return '\n'.join( + template.format(k, v) for k, v in sorted(d.items()) + ) + msg = ( + "`{cmd}` exited with a non-zero exit code {code}, see the " + "process log above.\n\nThe docker-compose command was " + "invoked with the following parameters:\n\nDefaults defined " + "in .env:\n{dotenv}\n\nArchery was called with:\n{params}" + ) + raise RuntimeError( + msg.format( + cmd=' '.join(e.cmd), + code=e.returncode, + dotenv=formatdict(self.config.dotenv, template=' {}: {}'), + params=formatdict( + self.config.params, template=' export {}={}' + ) + ) + ) + + def _execute_docker(self, *args, **kwargs): + # execute as a plain docker cli command + try: + result = Docker().run(*args, **kwargs) + result.check_returncode() + except subprocess.CalledProcessError as e: + raise RuntimeError( + "{} exited with non-zero exit code {}".format( + ' '.join(e.cmd), e.returncode + ) + ) + + def pull(self, service_name, pull_leaf=True, using_docker=False): + def _pull(service): + args = ['pull'] + if service['image'] in self.pull_memory: + return + + if using_docker: + try: + self._execute_docker(*args, service['image']) + except Exception as e: + # better --ignore-pull-failures handling + print(e) + else: + args.append('--ignore-pull-failures') + self._execute_compose(*args, service['name']) + + self.pull_memory.add(service['image']) + + service = self.config.get(service_name) + for ancestor in service['ancestors']: + _pull(self.config.get(ancestor)) + if pull_leaf: + _pull(service) + + def build(self, service_name, use_cache=True, use_leaf_cache=True, + using_docker=False, using_buildx=False, pull_parents=True): + def _build(service, use_cache): + if 'build' not in service: + # nothing to do + return + + args = [] + cache_from = list(service.get('build', {}).get('cache_from', [])) + if pull_parents: + for image in cache_from: + if image not in self.pull_memory: + try: + self._execute_docker('pull', image) + except Exception as e: + print(e) + finally: + self.pull_memory.add(image) + + if not use_cache: + args.append('--no-cache') + + # turn on inline build cache, this is a docker buildx feature + # used to bundle the image build cache to the pushed image manifest + # so the build cache can be reused across hosts, documented at + # https://github.com/docker/buildx#--cache-tonametypetypekeyvalue + if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1': + args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1']) + + if using_buildx: + for k, v in service['build'].get('args', {}).items(): + args.extend(['--build-arg', '{}={}'.format(k, v)]) + + if use_cache: + cache_ref = '{}-cache'.format(service['image']) + cache_from = 'type=registry,ref={}'.format(cache_ref) + cache_to = ( + 'type=registry,ref={},mode=max'.format(cache_ref) + ) + args.extend([ + '--cache-from', cache_from, + '--cache-to', cache_to, + ]) + + args.extend([ + '--output', 'type=docker', + '-f', service['build']['dockerfile'], + '-t', service['image'], + service['build'].get('context', '.') + ]) + self._execute_docker("buildx", "build", *args) + elif using_docker: + # better for caching + for k, v in service['build'].get('args', {}).items(): + args.extend(['--build-arg', '{}={}'.format(k, v)]) + for img in cache_from: + args.append('--cache-from="{}"'.format(img)) + args.extend([ + '-f', service['build']['dockerfile'], + '-t', service['image'], + service['build'].get('context', '.') + ]) + self._execute_docker("build", *args) + else: + self._execute_compose("build", *args, service['name']) + + service = self.config.get(service_name) + # build ancestor services + for ancestor in service['ancestors']: + _build(self.config.get(ancestor), use_cache=use_cache) + # build the leaf/target service + _build(service, use_cache=use_cache and use_leaf_cache) + + def run(self, service_name, command=None, *, env=None, volumes=None, + user=None, using_docker=False, resource_limit=None): + service = self.config.get(service_name) + + args = [] + if user is not None: + args.extend(['-u', user]) + + if env is not None: + for k, v in env.items(): + args.extend(['-e', '{}={}'.format(k, v)]) + + if volumes is not None: + for volume in volumes: + args.extend(['--volume', volume]) + + if using_docker or service['need_gpu'] or resource_limit: + # use gpus, requires docker>=19.03 + if service['need_gpu']: + args.extend(['--gpus', 'all']) + + if service.get('shm_size'): + args.extend(['--shm-size', service['shm_size']]) + + # append env variables from the compose conf + for k, v in service.get('environment', {}).items(): + args.extend(['-e', '{}={}'.format(k, v)]) + + # append volumes from the compose conf + for v in service.get('volumes', []): + if not isinstance(v, str): + # if not the compact string volume definition + v = "{}:{}".format(v['source'], v['target']) + args.extend(['-v', v]) + + # infer whether an interactive shell is desired or not + if command in ['cmd.exe', 'bash', 'sh', 'powershell']: + args.append('-it') + + if resource_limit: + limits = self.config.limit_presets.get(resource_limit) + if not limits: + raise ValueError( + f"Unknown resource limit preset '{resource_limit}'") + cpuset = limits.get('cpuset_cpus', []) + if cpuset: + args.append(f'--cpuset-cpus={",".join(map(str, cpuset))}') + memory = limits.get('memory') + if memory: + args.append(f'--memory={memory}') + args.append(f'--memory-swap={memory}') + + # get the actual docker image name instead of the compose service + # name which we refer as image in general + args.append(service['image']) + + # add command from compose if it wasn't overridden + if command is not None: + args.append(command) + else: + # replace whitespaces from the preformatted compose command + cmd = _sanitize_command(service.get('command', '')) + if cmd: + args.append(cmd) + + # execute as a plain docker cli command + self._execute_docker('run', '--rm', *args) + else: + # execute as a docker-compose command + args.append(service_name) + if command is not None: + args.append(command) + self._execute_compose('run', '--rm', *args) + + def push(self, service_name, user=None, password=None, using_docker=False): + def _push(service): + if using_docker: + return self._execute_docker('push', service['image']) + else: + return self._execute_compose('push', service['name']) + + if user is not None: + try: + # TODO(kszucs): have an option for a prompt + self._execute_docker('login', '-u', user, '-p', password) + except subprocess.CalledProcessError: + # hide credentials + msg = ('Failed to push `{}`, check the passed credentials' + .format(service_name)) + raise RuntimeError(msg) from None + + service = self.config.get(service_name) + for ancestor in service['ancestors']: + _push(self.config.get(ancestor)) + _push(service) + + def images(self): + return sorted(self.config.hierarchy.keys()) diff --git a/src/arrow/dev/archery/archery/docker/tests/test_docker.py b/src/arrow/dev/archery/archery/docker/tests/test_docker.py new file mode 100644 index 000000000..982f3bfc1 --- /dev/null +++ b/src/arrow/dev/archery/archery/docker/tests/test_docker.py @@ -0,0 +1,531 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import collections +import os +import re +import subprocess +from unittest import mock + +import pytest + +from archery.docker import DockerCompose +from archery.testing import assert_subprocess_calls, override_env, PartialEnv + + +missing_service_compose_yml = """ +version: '3.5' + +x-hierarchy: + - foo: + - sub-foo: + - sub-sub-foo + - another-sub-sub-foo + - bar: + - sub-bar + - baz + +services: + foo: + image: org/foo + sub-sub-foo: + image: org/sub-sub-foo + another-sub-sub-foo: + image: org/another-sub-sub-foo + bar: + image: org/bar + sub-bar: + image: org/sub-bar + baz: + image: org/baz +""" + +missing_node_compose_yml = """ +version: '3.5' + +x-hierarchy: + - foo: + - sub-foo: + - sub-sub-foo + - another-sub-sub-foo + - bar + - baz + +services: + foo: + image: org/foo + sub-foo: + image: org/sub-foo + sub-sub-foo: + image: org/sub-foo-foo + another-sub-sub-foo: + image: org/another-sub-sub-foo + bar: + image: org/bar + sub-bar: + image: org/sub-bar + baz: + image: org/baz +""" + +ok_compose_yml = """ +version: '3.5' + +x-hierarchy: + - foo: + - sub-foo: + - sub-sub-foo + - another-sub-sub-foo + - bar: + - sub-bar + - baz + +services: + foo: + image: org/foo + sub-foo: + image: org/sub-foo + sub-sub-foo: + image: org/sub-sub-foo + another-sub-sub-foo: + image: org/another-sub-sub-foo + bar: + image: org/bar + sub-bar: + image: org/sub-bar + baz: + image: org/baz +""" + +arrow_compose_yml = """ +version: '3.5' + +x-with-gpus: + - ubuntu-cuda + +x-hierarchy: + - conda-cpp: + - conda-python: + - conda-python-pandas + - conda-python-dask + - ubuntu-cpp: + - ubuntu-cpp-cmake32 + - ubuntu-c-glib: + - ubuntu-ruby + - ubuntu-cuda + +x-limit-presets: + github: + cpuset_cpus: [0, 1] + memory: 7g + +services: + conda-cpp: + image: org/conda-cpp + build: + context: . + dockerfile: ci/docker/conda-cpp.dockerfile + conda-python: + image: org/conda-python + build: + context: . + dockerfile: ci/docker/conda-cpp.dockerfile + args: + python: 3.6 + conda-python-pandas: + image: org/conda-python-pandas + build: + context: . + dockerfile: ci/docker/conda-python-pandas.dockerfile + conda-python-dask: + image: org/conda-python-dask + ubuntu-cpp: + image: org/ubuntu-cpp + build: + context: . + dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile + ubuntu-cpp-cmake32: + image: org/ubuntu-cpp-cmake32 + ubuntu-c-glib: + image: org/ubuntu-c-glib + ubuntu-ruby: + image: org/ubuntu-ruby + ubuntu-cuda: + image: org/ubuntu-cuda + environment: + CUDA_ENV: 1 + OTHER_ENV: 2 + volumes: + - /host:/container + command: /bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy" +""" + +arrow_compose_env = { + 'UBUNTU': '20.04', # overridden below + 'PYTHON': '3.6', + 'PANDAS': 'latest', + 'DASK': 'latest', # overridden below +} + + +def create_config(directory, yml_content, env_content=None): + env_path = directory / '.env' + config_path = directory / 'docker-compose.yml' + + with config_path.open('w') as fp: + fp.write(yml_content) + + if env_content is not None: + with env_path.open('w') as fp: + for k, v in env_content.items(): + fp.write("{}={}\n".format(k, v)) + + return config_path + + +def format_run(args): + cmd = ["run", "--rm"] + if isinstance(args, str): + return " ".join(cmd + [args]) + else: + return cmd + args + + +@pytest.fixture +def arrow_compose_path(tmpdir): + return create_config(tmpdir, arrow_compose_yml, arrow_compose_env) + + +def test_config_validation(tmpdir): + config_path = create_config(tmpdir, missing_service_compose_yml) + msg = "`sub-foo` is defined in `x-hierarchy` bot not in `services`" + with pytest.raises(ValueError, match=msg): + DockerCompose(config_path) + + config_path = create_config(tmpdir, missing_node_compose_yml) + msg = "`sub-bar` is defined in `services` but not in `x-hierarchy`" + with pytest.raises(ValueError, match=msg): + DockerCompose(config_path) + + config_path = create_config(tmpdir, ok_compose_yml) + DockerCompose(config_path) # no issue + + +def assert_docker_calls(compose, expected_args): + base_command = ['docker'] + expected_commands = [] + for args in expected_args: + if isinstance(args, str): + args = re.split(r"\s", args) + expected_commands.append(base_command + args) + return assert_subprocess_calls(expected_commands, check=True) + + +def assert_compose_calls(compose, expected_args, env=mock.ANY): + base_command = ['docker-compose', '--file', str(compose.config.path)] + expected_commands = [] + for args in expected_args: + if isinstance(args, str): + args = re.split(r"\s", args) + expected_commands.append(base_command + args) + return assert_subprocess_calls(expected_commands, check=True, env=env) + + +def test_arrow_example_validation_passes(arrow_compose_path): + DockerCompose(arrow_compose_path) + + +def test_compose_default_params_and_env(arrow_compose_path): + compose = DockerCompose(arrow_compose_path, params=dict( + UBUNTU='18.04', + DASK='master' + )) + assert compose.config.dotenv == arrow_compose_env + assert compose.config.params == { + 'UBUNTU': '18.04', + 'DASK': 'master', + } + + +def test_forwarding_env_variables(arrow_compose_path): + expected_calls = [ + "pull --ignore-pull-failures conda-cpp", + "build conda-cpp", + ] + expected_env = PartialEnv( + MY_CUSTOM_VAR_A='a', + MY_CUSTOM_VAR_B='b' + ) + with override_env({'MY_CUSTOM_VAR_A': 'a', 'MY_CUSTOM_VAR_B': 'b'}): + compose = DockerCompose(arrow_compose_path) + with assert_compose_calls(compose, expected_calls, env=expected_env): + assert os.environ['MY_CUSTOM_VAR_A'] == 'a' + assert os.environ['MY_CUSTOM_VAR_B'] == 'b' + compose.pull('conda-cpp') + compose.build('conda-cpp') + + +def test_compose_pull(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + + expected_calls = [ + "pull --ignore-pull-failures conda-cpp", + ] + with assert_compose_calls(compose, expected_calls): + compose.clear_pull_memory() + compose.pull('conda-cpp') + + expected_calls = [ + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", + "pull --ignore-pull-failures conda-python-pandas" + ] + with assert_compose_calls(compose, expected_calls): + compose.clear_pull_memory() + compose.pull('conda-python-pandas') + + expected_calls = [ + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", + ] + with assert_compose_calls(compose, expected_calls): + compose.clear_pull_memory() + compose.pull('conda-python-pandas', pull_leaf=False) + + +def test_compose_pull_params(arrow_compose_path): + expected_calls = [ + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", + ] + compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) + expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest') + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.clear_pull_memory() + compose.pull('conda-python-pandas', pull_leaf=False) + + +def test_compose_build(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + + expected_calls = [ + "build conda-cpp", + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-cpp') + + expected_calls = [ + "build --no-cache conda-cpp" + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-cpp', use_cache=False) + + expected_calls = [ + "build conda-cpp", + "build conda-python", + "build conda-python-pandas" + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-python-pandas') + + expected_calls = [ + "build --no-cache conda-cpp", + "build --no-cache conda-python", + "build --no-cache conda-python-pandas", + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-python-pandas', use_cache=False) + + expected_calls = [ + "build conda-cpp", + "build conda-python", + "build --no-cache conda-python-pandas", + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-python-pandas', use_cache=True, + use_leaf_cache=False) + + +@mock.patch.dict(os.environ, {"BUILDKIT_INLINE_CACHE": "1"}) +def test_compose_buildkit_inline_cache(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + + expected_calls = [ + "build --build-arg BUILDKIT_INLINE_CACHE=1 conda-cpp", + ] + with assert_compose_calls(compose, expected_calls): + compose.build('conda-cpp') + + +def test_compose_build_params(arrow_compose_path): + expected_calls = [ + "build ubuntu-cpp", + ] + + compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) + expected_env = PartialEnv(UBUNTU="18.04") + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.build('ubuntu-cpp') + + compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='16.04')) + expected_env = PartialEnv(UBUNTU="16.04") + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.build('ubuntu-cpp') + + expected_calls = [ + "build --no-cache conda-cpp", + "build --no-cache conda-python", + "build --no-cache conda-python-pandas", + ] + compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) + expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest') + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.build('conda-python-pandas', use_cache=False) + + +def test_compose_run(arrow_compose_path): + expected_calls = [ + format_run("conda-cpp"), + ] + compose = DockerCompose(arrow_compose_path) + with assert_compose_calls(compose, expected_calls): + compose.run('conda-cpp') + + expected_calls = [ + format_run("conda-python") + ] + expected_env = PartialEnv(PYTHON='3.6') + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.run('conda-python') + + compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) + expected_env = PartialEnv(PYTHON='3.8') + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.run('conda-python') + + compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) + for command in ["bash", "echo 1"]: + expected_calls = [ + format_run(["conda-python", command]), + ] + expected_env = PartialEnv(PYTHON='3.8') + with assert_compose_calls(compose, expected_calls, env=expected_env): + compose.run('conda-python', command) + + expected_calls = [ + ( + format_run("-e CONTAINER_ENV_VAR_A=a -e CONTAINER_ENV_VAR_B=b " + "conda-python") + ) + ] + compose = DockerCompose(arrow_compose_path) + expected_env = PartialEnv(PYTHON='3.6') + with assert_compose_calls(compose, expected_calls, env=expected_env): + env = collections.OrderedDict([ + ("CONTAINER_ENV_VAR_A", "a"), + ("CONTAINER_ENV_VAR_B", "b") + ]) + compose.run('conda-python', env=env) + + expected_calls = [ + ( + format_run("--volume /host/build:/build --volume " + "/host/ccache:/ccache:delegated conda-python") + ) + ] + compose = DockerCompose(arrow_compose_path) + with assert_compose_calls(compose, expected_calls): + volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated") + compose.run('conda-python', volumes=volumes) + + +def test_compose_run_with_resource_limits(arrow_compose_path): + expected_calls = [ + format_run([ + "--cpuset-cpus=0,1", + "--memory=7g", + "--memory-swap=7g", + "org/conda-cpp" + ]), + ] + compose = DockerCompose(arrow_compose_path) + with assert_docker_calls(compose, expected_calls): + compose.run('conda-cpp', resource_limit="github") + + +def test_compose_push(arrow_compose_path): + compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) + expected_env = PartialEnv(PYTHON="3.8") + expected_calls = [ + mock.call(["docker", "login", "-u", "user", "-p", "pass"], check=True), + ] + for image in ["conda-cpp", "conda-python", "conda-python-pandas"]: + expected_calls.append( + mock.call(["docker-compose", "--file", str(compose.config.path), + "push", image], check=True, env=expected_env) + ) + with assert_subprocess_calls(expected_calls): + compose.push('conda-python-pandas', user='user', password='pass') + + +def test_compose_error(arrow_compose_path): + compose = DockerCompose(arrow_compose_path, params=dict( + PYTHON='3.8', + PANDAS='master' + )) + + error = subprocess.CalledProcessError(99, []) + with mock.patch('subprocess.run', side_effect=error): + with pytest.raises(RuntimeError) as exc: + compose.run('conda-cpp') + + exception_message = str(exc.value) + assert "exited with a non-zero exit code 99" in exception_message + assert "PANDAS: latest" in exception_message + assert "export PANDAS=master" in exception_message + + +def test_image_with_gpu(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + + expected_calls = [ + [ + "run", "--rm", "--gpus", "all", + "-e", "CUDA_ENV=1", + "-e", "OTHER_ENV=2", + "-v", "/host:/container:rw", + "org/ubuntu-cuda", + '/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"' + ] + ] + with assert_docker_calls(compose, expected_calls): + compose.run('ubuntu-cuda') + + +def test_listing_images(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + assert sorted(compose.images()) == [ + 'conda-cpp', + 'conda-python', + 'conda-python-dask', + 'conda-python-pandas', + 'ubuntu-c-glib', + 'ubuntu-cpp', + 'ubuntu-cpp-cmake32', + 'ubuntu-cuda', + 'ubuntu-ruby', + ] diff --git a/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py b/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py new file mode 100644 index 000000000..ab39c7b9d --- /dev/null +++ b/src/arrow/dev/archery/archery/docker/tests/test_docker_cli.py @@ -0,0 +1,201 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from unittest.mock import patch + +from click.testing import CliRunner + +from archery.docker import DockerCompose +from archery.docker.cli import docker + + +@patch.object(DockerCompose, "pull") +@patch.object(DockerCompose, "build") +@patch.object(DockerCompose, "run") +def test_docker_run_with_custom_command(run, build, pull): + # with custom command + args = ["run", "ubuntu-cpp", "bash"] + result = CliRunner().invoke(docker, args) + + assert result.exit_code == 0 + pull.assert_called_once_with( + "ubuntu-cpp", pull_leaf=True, using_docker=False + ) + build.assert_called_once_with( + "ubuntu-cpp", + use_cache=True, + use_leaf_cache=True, + using_docker=False, + using_buildx=False + ) + run.assert_called_once_with( + "ubuntu-cpp", + command="bash", + env={}, + resource_limit=None, + user=None, + using_docker=False, + volumes=(), + ) + + +@patch.object(DockerCompose, "pull") +@patch.object(DockerCompose, "build") +@patch.object(DockerCompose, "run") +def test_docker_run_options(run, build, pull): + # environment variables and volumes + args = [ + "run", + "-e", + "ARROW_GANDIVA=OFF", + "-e", + "ARROW_FLIGHT=ON", + "--volume", + "./build:/build", + "-v", + "./ccache:/ccache:delegated", + "-u", + "root", + "ubuntu-cpp", + ] + result = CliRunner().invoke(docker, args) + assert result.exit_code == 0 + pull.assert_called_once_with( + "ubuntu-cpp", pull_leaf=True, using_docker=False + ) + build.assert_called_once_with( + "ubuntu-cpp", + use_cache=True, + use_leaf_cache=True, + using_docker=False, + using_buildx=False + ) + run.assert_called_once_with( + "ubuntu-cpp", + command=None, + env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"}, + resource_limit=None, + user="root", + using_docker=False, + volumes=( + "./build:/build", + "./ccache:/ccache:delegated", + ), + ) + + +@patch.object(DockerCompose, "run") +def test_docker_limit_options(run): + # environment variables and volumes + args = [ + "run", + "-e", + "ARROW_GANDIVA=OFF", + "-e", + "ARROW_FLIGHT=ON", + "--volume", + "./build:/build", + "-v", + "./ccache:/ccache:delegated", + "-u", + "root", + "--resource-limit=github", + "--no-build", + "--no-pull", + "ubuntu-cpp", + ] + result = CliRunner().invoke(docker, args) + assert result.exit_code == 0 + run.assert_called_once_with( + "ubuntu-cpp", + command=None, + env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"}, + resource_limit="github", + user="root", + using_docker=False, + volumes=( + "./build:/build", + "./ccache:/ccache:delegated", + ), + ) + + +@patch.object(DockerCompose, "run") +def test_docker_run_without_pulling_or_building(run): + args = ["run", "--no-pull", "--no-build", "ubuntu-cpp"] + result = CliRunner().invoke(docker, args) + assert result.exit_code == 0 + run.assert_called_once_with( + "ubuntu-cpp", + command=None, + env={}, + resource_limit=None, + user=None, + using_docker=False, + volumes=(), + ) + + +@patch.object(DockerCompose, "pull") +@patch.object(DockerCompose, "build") +def test_docker_run_only_pulling_and_building(build, pull): + args = ["run", "ubuntu-cpp", "--build-only"] + result = CliRunner().invoke(docker, args) + assert result.exit_code == 0 + pull.assert_called_once_with( + "ubuntu-cpp", pull_leaf=True, using_docker=False + ) + build.assert_called_once_with( + "ubuntu-cpp", + use_cache=True, + use_leaf_cache=True, + using_docker=False, + using_buildx=False + ) + + +@patch.object(DockerCompose, "build") +@patch.object(DockerCompose, "run") +def test_docker_run_without_build_cache(run, build): + args = [ + "run", + "--no-pull", + "--force-build", + "--user", + "me", + "--no-cache", + "--no-leaf-cache", + "ubuntu-cpp", + ] + result = CliRunner().invoke(docker, args) + assert result.exit_code == 0 + build.assert_called_once_with( + "ubuntu-cpp", + use_cache=False, + use_leaf_cache=False, + using_docker=False, + using_buildx=False + ) + run.assert_called_once_with( + "ubuntu-cpp", + command=None, + env={}, + resource_limit=None, + user="me", + using_docker=False, + volumes=(), + ) diff --git a/src/arrow/dev/archery/archery/integration/__init__.py b/src/arrow/dev/archery/archery/integration/__init__.py new file mode 100644 index 000000000..13a83393a --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/arrow/dev/archery/archery/integration/datagen.py b/src/arrow/dev/archery/archery/integration/datagen.py new file mode 100644 index 000000000..b764982bd --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/datagen.py @@ -0,0 +1,1662 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import namedtuple, OrderedDict +import binascii +import json +import os +import random +import tempfile + +import numpy as np + +from .util import frombytes, tobytes, random_bytes, random_utf8 + + +def metadata_key_values(pairs): + return [{'key': k, 'value': v} for k, v in pairs] + + +class Field(object): + + def __init__(self, name, *, nullable=True, metadata=None): + self.name = name + self.nullable = nullable + self.metadata = metadata or [] + + def get_json(self): + entries = [ + ('name', self.name), + ('type', self._get_type()), + ('nullable', self.nullable), + ('children', self._get_children()), + ] + + dct = self._get_dictionary() + if dct: + entries.append(('dictionary', dct)) + + if self.metadata is not None and len(self.metadata) > 0: + entries.append(('metadata', metadata_key_values(self.metadata))) + + return OrderedDict(entries) + + def _get_dictionary(self): + return None + + def _make_is_valid(self, size, null_probability=0.4): + if self.nullable: + return (np.random.random_sample(size) > null_probability + ).astype(np.int8) + else: + return np.ones(size, dtype=np.int8) + + +class Column(object): + + def __init__(self, name, count): + self.name = name + self.count = count + + def __len__(self): + return self.count + + def _get_children(self): + return [] + + def _get_buffers(self): + return [] + + def get_json(self): + entries = [ + ('name', self.name), + ('count', self.count) + ] + + buffers = self._get_buffers() + entries.extend(buffers) + + children = self._get_children() + if len(children) > 0: + entries.append(('children', children)) + + return OrderedDict(entries) + + +class PrimitiveField(Field): + + def _get_children(self): + return [] + + +class PrimitiveColumn(Column): + + def __init__(self, name, count, is_valid, values): + super().__init__(name, count) + self.is_valid = is_valid + self.values = values + + def _encode_value(self, x): + return x + + def _get_buffers(self): + return [ + ('VALIDITY', [int(v) for v in self.is_valid]), + ('DATA', list([self._encode_value(x) for x in self.values])) + ] + + +class NullColumn(Column): + # This subclass is for readability only + pass + + +class NullField(PrimitiveField): + + def __init__(self, name, metadata=None): + super().__init__(name, nullable=True, + metadata=metadata) + + def _get_type(self): + return OrderedDict([('name', 'null')]) + + def generate_column(self, size, name=None): + return NullColumn(name or self.name, size) + + +TEST_INT_MAX = 2 ** 31 - 1 +TEST_INT_MIN = ~TEST_INT_MAX + + +class IntegerField(PrimitiveField): + + def __init__(self, name, is_signed, bit_width, *, nullable=True, + metadata=None, + min_value=TEST_INT_MIN, + max_value=TEST_INT_MAX): + super().__init__(name, nullable=nullable, + metadata=metadata) + self.is_signed = is_signed + self.bit_width = bit_width + self.min_value = min_value + self.max_value = max_value + + def _get_generated_data_bounds(self): + if self.is_signed: + signed_iinfo = np.iinfo('int' + str(self.bit_width)) + min_value, max_value = signed_iinfo.min, signed_iinfo.max + else: + unsigned_iinfo = np.iinfo('uint' + str(self.bit_width)) + min_value, max_value = 0, unsigned_iinfo.max + + lower_bound = max(min_value, self.min_value) + upper_bound = min(max_value, self.max_value) + return lower_bound, upper_bound + + def _get_type(self): + return OrderedDict([ + ('name', 'int'), + ('isSigned', self.is_signed), + ('bitWidth', self.bit_width) + ]) + + def generate_column(self, size, name=None): + lower_bound, upper_bound = self._get_generated_data_bounds() + return self.generate_range(size, lower_bound, upper_bound, + name=name, include_extremes=True) + + def generate_range(self, size, lower, upper, name=None, + include_extremes=False): + values = np.random.randint(lower, upper, size=size, dtype=np.int64) + if include_extremes and size >= 2: + values[:2] = [lower, upper] + values = list(map(int if self.bit_width < 64 else str, values)) + + is_valid = self._make_is_valid(size) + + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + +class DateField(IntegerField): + + DAY = 0 + MILLISECOND = 1 + + # 1/1/1 to 12/31/9999 + _ranges = { + DAY: [-719162, 2932896], + MILLISECOND: [-62135596800000, 253402214400000] + } + + def __init__(self, name, unit, *, nullable=True, metadata=None): + bit_width = 32 if unit == self.DAY else 64 + + min_value, max_value = self._ranges[unit] + super().__init__( + name, True, bit_width, + nullable=nullable, metadata=metadata, + min_value=min_value, max_value=max_value + ) + self.unit = unit + + def _get_type(self): + return OrderedDict([ + ('name', 'date'), + ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND') + ]) + + +TIMEUNIT_NAMES = { + 's': 'SECOND', + 'ms': 'MILLISECOND', + 'us': 'MICROSECOND', + 'ns': 'NANOSECOND' +} + + +class TimeField(IntegerField): + + BIT_WIDTHS = { + 's': 32, + 'ms': 32, + 'us': 64, + 'ns': 64 + } + + _ranges = { + 's': [0, 86400], + 'ms': [0, 86400000], + 'us': [0, 86400000000], + 'ns': [0, 86400000000000] + } + + def __init__(self, name, unit='s', *, nullable=True, + metadata=None): + min_val, max_val = self._ranges[unit] + super().__init__(name, True, self.BIT_WIDTHS[unit], + nullable=nullable, metadata=metadata, + min_value=min_val, max_value=max_val) + self.unit = unit + + def _get_type(self): + return OrderedDict([ + ('name', 'time'), + ('unit', TIMEUNIT_NAMES[self.unit]), + ('bitWidth', self.bit_width) + ]) + + +class TimestampField(IntegerField): + + # 1/1/1 to 12/31/9999 + _ranges = { + 's': [-62135596800, 253402214400], + 'ms': [-62135596800000, 253402214400000], + 'us': [-62135596800000000, 253402214400000000], + + # Physical range for int64, ~584 years and change + 'ns': [np.iinfo('int64').min, np.iinfo('int64').max] + } + + def __init__(self, name, unit='s', tz=None, *, nullable=True, + metadata=None): + min_val, max_val = self._ranges[unit] + super().__init__(name, True, 64, + nullable=nullable, + metadata=metadata, + min_value=min_val, + max_value=max_val) + self.unit = unit + self.tz = tz + + def _get_type(self): + fields = [ + ('name', 'timestamp'), + ('unit', TIMEUNIT_NAMES[self.unit]) + ] + + if self.tz is not None: + fields.append(('timezone', self.tz)) + + return OrderedDict(fields) + + +class DurationIntervalField(IntegerField): + + def __init__(self, name, unit='s', *, nullable=True, + metadata=None): + min_val, max_val = np.iinfo('int64').min, np.iinfo('int64').max, + super().__init__( + name, True, 64, + nullable=nullable, metadata=metadata, + min_value=min_val, max_value=max_val) + self.unit = unit + + def _get_type(self): + fields = [ + ('name', 'duration'), + ('unit', TIMEUNIT_NAMES[self.unit]) + ] + + return OrderedDict(fields) + + +class YearMonthIntervalField(IntegerField): + def __init__(self, name, *, nullable=True, metadata=None): + min_val, max_val = [-10000*12, 10000*12] # +/- 10000 years. + super().__init__( + name, True, 32, + nullable=nullable, metadata=metadata, + min_value=min_val, max_value=max_val) + + def _get_type(self): + fields = [ + ('name', 'interval'), + ('unit', 'YEAR_MONTH'), + ] + + return OrderedDict(fields) + + +class DayTimeIntervalField(PrimitiveField): + def __init__(self, name, *, nullable=True, metadata=None): + super().__init__(name, + nullable=True, + metadata=metadata) + + @property + def numpy_type(self): + return object + + def _get_type(self): + + return OrderedDict([ + ('name', 'interval'), + ('unit', 'DAY_TIME'), + ]) + + def generate_column(self, size, name=None): + min_day_value, max_day_value = -10000*366, 10000*366 + values = [{'days': random.randint(min_day_value, max_day_value), + 'milliseconds': random.randint(-86400000, +86400000)} + for _ in range(size)] + + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + +class MonthDayNanoIntervalField(PrimitiveField): + def __init__(self, name, *, nullable=True, metadata=None): + super().__init__(name, + nullable=True, + metadata=metadata) + + @property + def numpy_type(self): + return object + + def _get_type(self): + + return OrderedDict([ + ('name', 'interval'), + ('unit', 'MONTH_DAY_NANO'), + ]) + + def generate_column(self, size, name=None): + I32 = 'int32' + min_int_value, max_int_value = np.iinfo(I32).min, np.iinfo(I32).max + I64 = 'int64' + min_nano_val, max_nano_val = np.iinfo(I64).min, np.iinfo(I64).max, + values = [{'months': random.randint(min_int_value, max_int_value), + 'days': random.randint(min_int_value, max_int_value), + 'nanoseconds': random.randint(min_nano_val, max_nano_val)} + for _ in range(size)] + + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + +class FloatingPointField(PrimitiveField): + + def __init__(self, name, bit_width, *, nullable=True, + metadata=None): + super().__init__(name, + nullable=nullable, + metadata=metadata) + + self.bit_width = bit_width + self.precision = { + 16: 'HALF', + 32: 'SINGLE', + 64: 'DOUBLE' + }[self.bit_width] + + @property + def numpy_type(self): + return 'float' + str(self.bit_width) + + def _get_type(self): + return OrderedDict([ + ('name', 'floatingpoint'), + ('precision', self.precision) + ]) + + def generate_column(self, size, name=None): + values = np.random.randn(size) * 1000 + values = np.round(values, 3) + + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + +DECIMAL_PRECISION_TO_VALUE = { + key: (1 << (8 * i - 1)) - 1 for i, key in enumerate( + [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36, + 40, 42, 44, 50, 60, 70], + start=1, + ) +} + + +def decimal_range_from_precision(precision): + assert 1 <= precision <= 76 + try: + max_value = DECIMAL_PRECISION_TO_VALUE[precision] + except KeyError: + return decimal_range_from_precision(precision - 1) + else: + return ~max_value, max_value + + +class DecimalField(PrimitiveField): + def __init__(self, name, precision, scale, bit_width, *, + nullable=True, metadata=None): + super().__init__(name, nullable=True, + metadata=metadata) + self.precision = precision + self.scale = scale + self.bit_width = bit_width + + @property + def numpy_type(self): + return object + + def _get_type(self): + return OrderedDict([ + ('name', 'decimal'), + ('precision', self.precision), + ('scale', self.scale), + ('bitWidth', self.bit_width), + ]) + + def generate_column(self, size, name=None): + min_value, max_value = decimal_range_from_precision(self.precision) + values = [random.randint(min_value, max_value) for _ in range(size)] + + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return DecimalColumn(name, size, is_valid, values, self.bit_width) + + +class DecimalColumn(PrimitiveColumn): + + def __init__(self, name, count, is_valid, values, bit_width): + super().__init__(name, count, is_valid, values) + self.bit_width = bit_width + + def _encode_value(self, x): + return str(x) + + +class BooleanField(PrimitiveField): + bit_width = 1 + + def _get_type(self): + return OrderedDict([('name', 'bool')]) + + @property + def numpy_type(self): + return 'bool' + + def generate_column(self, size, name=None): + values = list(map(bool, np.random.randint(0, 2, size=size))) + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + +class FixedSizeBinaryField(PrimitiveField): + + def __init__(self, name, byte_width, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, + metadata=metadata) + self.byte_width = byte_width + + @property + def numpy_type(self): + return object + + @property + def column_class(self): + return FixedSizeBinaryColumn + + def _get_type(self): + return OrderedDict([('name', 'fixedsizebinary'), + ('byteWidth', self.byte_width)]) + + def generate_column(self, size, name=None): + is_valid = self._make_is_valid(size) + values = [] + + for i in range(size): + values.append(random_bytes(self.byte_width)) + + if name is None: + name = self.name + return self.column_class(name, size, is_valid, values) + + +class BinaryField(PrimitiveField): + + @property + def numpy_type(self): + return object + + @property + def column_class(self): + return BinaryColumn + + def _get_type(self): + return OrderedDict([('name', 'binary')]) + + def _random_sizes(self, size): + return np.random.exponential(scale=4, size=size).astype(np.int32) + + def generate_column(self, size, name=None): + is_valid = self._make_is_valid(size) + values = [] + + sizes = self._random_sizes(size) + + for i, nbytes in enumerate(sizes): + if is_valid[i]: + values.append(random_bytes(nbytes)) + else: + values.append(b"") + + if name is None: + name = self.name + return self.column_class(name, size, is_valid, values) + + +class StringField(BinaryField): + + @property + def column_class(self): + return StringColumn + + def _get_type(self): + return OrderedDict([('name', 'utf8')]) + + def generate_column(self, size, name=None): + K = 7 + is_valid = self._make_is_valid(size) + values = [] + + for i in range(size): + if is_valid[i]: + values.append(tobytes(random_utf8(K))) + else: + values.append(b"") + + if name is None: + name = self.name + return self.column_class(name, size, is_valid, values) + + +class LargeBinaryField(BinaryField): + + @property + def column_class(self): + return LargeBinaryColumn + + def _get_type(self): + return OrderedDict([('name', 'largebinary')]) + + +class LargeStringField(StringField): + + @property + def column_class(self): + return LargeStringColumn + + def _get_type(self): + return OrderedDict([('name', 'largeutf8')]) + + +class Schema(object): + + def __init__(self, fields, metadata=None): + self.fields = fields + self.metadata = metadata + + def get_json(self): + entries = [ + ('fields', [field.get_json() for field in self.fields]) + ] + + if self.metadata is not None and len(self.metadata) > 0: + entries.append(('metadata', metadata_key_values(self.metadata))) + + return OrderedDict(entries) + + +class _NarrowOffsetsMixin: + + def _encode_offsets(self, offsets): + return list(map(int, offsets)) + + +class _LargeOffsetsMixin: + + def _encode_offsets(self, offsets): + # 64-bit offsets have to be represented as strings to roundtrip + # through JSON. + return list(map(str, offsets)) + + +class _BaseBinaryColumn(PrimitiveColumn): + + def _encode_value(self, x): + return frombytes(binascii.hexlify(x).upper()) + + def _get_buffers(self): + offset = 0 + offsets = [0] + + data = [] + for i, v in enumerate(self.values): + if self.is_valid[i]: + offset += len(v) + else: + v = b"" + + offsets.append(offset) + data.append(self._encode_value(v)) + + return [ + ('VALIDITY', [int(x) for x in self.is_valid]), + ('OFFSET', self._encode_offsets(offsets)), + ('DATA', data) + ] + + +class _BaseStringColumn(_BaseBinaryColumn): + + def _encode_value(self, x): + return frombytes(x) + + +class BinaryColumn(_BaseBinaryColumn, _NarrowOffsetsMixin): + pass + + +class StringColumn(_BaseStringColumn, _NarrowOffsetsMixin): + pass + + +class LargeBinaryColumn(_BaseBinaryColumn, _LargeOffsetsMixin): + pass + + +class LargeStringColumn(_BaseStringColumn, _LargeOffsetsMixin): + pass + + +class FixedSizeBinaryColumn(PrimitiveColumn): + + def _encode_value(self, x): + return frombytes(binascii.hexlify(x).upper()) + + def _get_buffers(self): + data = [] + for i, v in enumerate(self.values): + data.append(self._encode_value(v)) + + return [ + ('VALIDITY', [int(x) for x in self.is_valid]), + ('DATA', data) + ] + + +class ListField(Field): + + def __init__(self, name, value_field, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, + metadata=metadata) + self.value_field = value_field + + @property + def column_class(self): + return ListColumn + + def _get_type(self): + return OrderedDict([ + ('name', 'list') + ]) + + def _get_children(self): + return [self.value_field.get_json()] + + def generate_column(self, size, name=None): + MAX_LIST_SIZE = 4 + + is_valid = self._make_is_valid(size) + list_sizes = np.random.randint(0, MAX_LIST_SIZE + 1, size=size) + offsets = [0] + + offset = 0 + for i in range(size): + if is_valid[i]: + offset += int(list_sizes[i]) + offsets.append(offset) + + # The offset now is the total number of elements in the child array + values = self.value_field.generate_column(offset) + + if name is None: + name = self.name + return self.column_class(name, size, is_valid, offsets, values) + + +class LargeListField(ListField): + + @property + def column_class(self): + return LargeListColumn + + def _get_type(self): + return OrderedDict([ + ('name', 'largelist') + ]) + + +class _BaseListColumn(Column): + + def __init__(self, name, count, is_valid, offsets, values): + super().__init__(name, count) + self.is_valid = is_valid + self.offsets = offsets + self.values = values + + def _get_buffers(self): + return [ + ('VALIDITY', [int(v) for v in self.is_valid]), + ('OFFSET', self._encode_offsets(self.offsets)) + ] + + def _get_children(self): + return [self.values.get_json()] + + +class ListColumn(_BaseListColumn, _NarrowOffsetsMixin): + pass + + +class LargeListColumn(_BaseListColumn, _LargeOffsetsMixin): + pass + + +class MapField(Field): + + def __init__(self, name, key_field, item_field, *, nullable=True, + metadata=None, keys_sorted=False, entries_name='entries'): + super().__init__(name, nullable=nullable, + metadata=metadata) + + assert not key_field.nullable + self.key_field = key_field + self.item_field = item_field + self.pair_field = StructField(entries_name, [key_field, item_field], + nullable=False) + self.keys_sorted = keys_sorted + + def _get_type(self): + return OrderedDict([ + ('name', 'map'), + ('keysSorted', self.keys_sorted) + ]) + + def _get_children(self): + return [self.pair_field.get_json()] + + def generate_column(self, size, name=None): + MAX_MAP_SIZE = 4 + + is_valid = self._make_is_valid(size) + map_sizes = np.random.randint(0, MAX_MAP_SIZE + 1, size=size) + offsets = [0] + + offset = 0 + for i in range(size): + if is_valid[i]: + offset += int(map_sizes[i]) + offsets.append(offset) + + # The offset now is the total number of elements in the child array + pairs = self.pair_field.generate_column(offset) + if name is None: + name = self.name + + return MapColumn(name, size, is_valid, offsets, pairs) + + +class MapColumn(Column): + + def __init__(self, name, count, is_valid, offsets, pairs): + super().__init__(name, count) + self.is_valid = is_valid + self.offsets = offsets + self.pairs = pairs + + def _get_buffers(self): + return [ + ('VALIDITY', [int(v) for v in self.is_valid]), + ('OFFSET', list(self.offsets)) + ] + + def _get_children(self): + return [self.pairs.get_json()] + + +class FixedSizeListField(Field): + + def __init__(self, name, value_field, list_size, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, + metadata=metadata) + self.value_field = value_field + self.list_size = list_size + + def _get_type(self): + return OrderedDict([ + ('name', 'fixedsizelist'), + ('listSize', self.list_size) + ]) + + def _get_children(self): + return [self.value_field.get_json()] + + def generate_column(self, size, name=None): + is_valid = self._make_is_valid(size) + values = self.value_field.generate_column(size * self.list_size) + + if name is None: + name = self.name + return FixedSizeListColumn(name, size, is_valid, values) + + +class FixedSizeListColumn(Column): + + def __init__(self, name, count, is_valid, values): + super().__init__(name, count) + self.is_valid = is_valid + self.values = values + + def _get_buffers(self): + return [ + ('VALIDITY', [int(v) for v in self.is_valid]) + ] + + def _get_children(self): + return [self.values.get_json()] + + +class StructField(Field): + + def __init__(self, name, fields, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, + metadata=metadata) + self.fields = fields + + def _get_type(self): + return OrderedDict([ + ('name', 'struct') + ]) + + def _get_children(self): + return [field.get_json() for field in self.fields] + + def generate_column(self, size, name=None): + is_valid = self._make_is_valid(size) + + field_values = [field.generate_column(size) for field in self.fields] + if name is None: + name = self.name + return StructColumn(name, size, is_valid, field_values) + + +class _BaseUnionField(Field): + + def __init__(self, name, fields, type_ids=None, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, metadata=metadata) + if type_ids is None: + type_ids = list(range(fields)) + else: + assert len(fields) == len(type_ids) + self.fields = fields + self.type_ids = type_ids + assert all(x >= 0 for x in self.type_ids) + + def _get_type(self): + return OrderedDict([ + ('name', 'union'), + ('mode', self.mode), + ('typeIds', self.type_ids), + ]) + + def _get_children(self): + return [field.get_json() for field in self.fields] + + def _make_type_ids(self, size): + return np.random.choice(self.type_ids, size) + + +class SparseUnionField(_BaseUnionField): + mode = 'SPARSE' + + def generate_column(self, size, name=None): + array_type_ids = self._make_type_ids(size) + field_values = [field.generate_column(size) for field in self.fields] + + if name is None: + name = self.name + return SparseUnionColumn(name, size, array_type_ids, field_values) + + +class DenseUnionField(_BaseUnionField): + mode = 'DENSE' + + def generate_column(self, size, name=None): + # Reverse mapping {logical type id => physical child id} + child_ids = [None] * (max(self.type_ids) + 1) + for i, type_id in enumerate(self.type_ids): + child_ids[type_id] = i + + array_type_ids = self._make_type_ids(size) + offsets = [] + child_sizes = [0] * len(self.fields) + + for i in range(size): + child_id = child_ids[array_type_ids[i]] + offset = child_sizes[child_id] + offsets.append(offset) + child_sizes[child_id] = offset + 1 + + field_values = [ + field.generate_column(child_size) + for field, child_size in zip(self.fields, child_sizes)] + + if name is None: + name = self.name + return DenseUnionColumn(name, size, array_type_ids, offsets, + field_values) + + +class Dictionary(object): + + def __init__(self, id_, field, size, name=None, ordered=False): + self.id_ = id_ + self.field = field + self.values = field.generate_column(size=size, name=name) + self.ordered = ordered + + def __len__(self): + return len(self.values) + + def get_json(self): + dummy_batch = RecordBatch(len(self.values), [self.values]) + return OrderedDict([ + ('id', self.id_), + ('data', dummy_batch.get_json()) + ]) + + +class DictionaryField(Field): + + def __init__(self, name, index_field, dictionary, *, nullable=True, + metadata=None): + super().__init__(name, nullable=nullable, + metadata=metadata) + assert index_field.name == '' + assert isinstance(index_field, IntegerField) + assert isinstance(dictionary, Dictionary) + + self.index_field = index_field + self.dictionary = dictionary + + def _get_type(self): + return self.dictionary.field._get_type() + + def _get_children(self): + return self.dictionary.field._get_children() + + def _get_dictionary(self): + return OrderedDict([ + ('id', self.dictionary.id_), + ('indexType', self.index_field._get_type()), + ('isOrdered', self.dictionary.ordered) + ]) + + def generate_column(self, size, name=None): + if name is None: + name = self.name + return self.index_field.generate_range(size, 0, len(self.dictionary), + name=name) + + +ExtensionType = namedtuple( + 'ExtensionType', ['extension_name', 'serialized', 'storage_field']) + + +class ExtensionField(Field): + + def __init__(self, name, extension_type, *, nullable=True, metadata=None): + metadata = (metadata or []) + [ + ('ARROW:extension:name', extension_type.extension_name), + ('ARROW:extension:metadata', extension_type.serialized), + ] + super().__init__(name, nullable=nullable, metadata=metadata) + self.extension_type = extension_type + + def _get_type(self): + return self.extension_type.storage_field._get_type() + + def _get_children(self): + return self.extension_type.storage_field._get_children() + + def _get_dictionary(self): + return self.extension_type.storage_field._get_dictionary() + + def generate_column(self, size, name=None): + if name is None: + name = self.name + return self.extension_type.storage_field.generate_column(size, name) + + +class StructColumn(Column): + + def __init__(self, name, count, is_valid, field_values): + super().__init__(name, count) + self.is_valid = is_valid + self.field_values = field_values + + def _get_buffers(self): + return [ + ('VALIDITY', [int(v) for v in self.is_valid]) + ] + + def _get_children(self): + return [field.get_json() for field in self.field_values] + + +class SparseUnionColumn(Column): + + def __init__(self, name, count, type_ids, field_values): + super().__init__(name, count) + self.type_ids = type_ids + self.field_values = field_values + + def _get_buffers(self): + return [ + ('TYPE_ID', [int(v) for v in self.type_ids]) + ] + + def _get_children(self): + return [field.get_json() for field in self.field_values] + + +class DenseUnionColumn(Column): + + def __init__(self, name, count, type_ids, offsets, field_values): + super().__init__(name, count) + self.type_ids = type_ids + self.offsets = offsets + self.field_values = field_values + + def _get_buffers(self): + return [ + ('TYPE_ID', [int(v) for v in self.type_ids]), + ('OFFSET', [int(v) for v in self.offsets]), + ] + + def _get_children(self): + return [field.get_json() for field in self.field_values] + + +class RecordBatch(object): + + def __init__(self, count, columns): + self.count = count + self.columns = columns + + def get_json(self): + return OrderedDict([ + ('count', self.count), + ('columns', [col.get_json() for col in self.columns]) + ]) + + +class File(object): + + def __init__(self, name, schema, batches, dictionaries=None, + skip=None, path=None): + self.name = name + self.schema = schema + self.dictionaries = dictionaries or [] + self.batches = batches + self.skip = set() + self.path = path + if skip: + self.skip.update(skip) + + def get_json(self): + entries = [ + ('schema', self.schema.get_json()) + ] + + if len(self.dictionaries) > 0: + entries.append(('dictionaries', + [dictionary.get_json() + for dictionary in self.dictionaries])) + + entries.append(('batches', [batch.get_json() + for batch in self.batches])) + return OrderedDict(entries) + + def write(self, path): + with open(path, 'wb') as f: + f.write(json.dumps(self.get_json(), indent=2).encode('utf-8')) + self.path = path + + def skip_category(self, category): + """Skip this test for the given category. + + Category should be SKIP_ARROW or SKIP_FLIGHT. + """ + self.skip.add(category) + return self + + +def get_field(name, type_, **kwargs): + if type_ == 'binary': + return BinaryField(name, **kwargs) + elif type_ == 'utf8': + return StringField(name, **kwargs) + elif type_ == 'largebinary': + return LargeBinaryField(name, **kwargs) + elif type_ == 'largeutf8': + return LargeStringField(name, **kwargs) + elif type_.startswith('fixedsizebinary_'): + byte_width = int(type_.split('_')[1]) + return FixedSizeBinaryField(name, byte_width=byte_width, **kwargs) + + dtype = np.dtype(type_) + + if dtype.kind in ('i', 'u'): + signed = dtype.kind == 'i' + bit_width = dtype.itemsize * 8 + return IntegerField(name, signed, bit_width, **kwargs) + elif dtype.kind == 'f': + bit_width = dtype.itemsize * 8 + return FloatingPointField(name, bit_width, **kwargs) + elif dtype.kind == 'b': + return BooleanField(name, **kwargs) + else: + raise TypeError(dtype) + + +def _generate_file(name, fields, batch_sizes, dictionaries=None, skip=None, + metadata=None): + schema = Schema(fields, metadata=metadata) + batches = [] + for size in batch_sizes: + columns = [] + for field in fields: + col = field.generate_column(size) + columns.append(col) + + batches.append(RecordBatch(size, columns)) + + return File(name, schema, batches, dictionaries, skip=skip) + + +def generate_custom_metadata_case(): + def meta(items): + # Generate a simple block of metadata where each value is '{}'. + # Keys are delimited by whitespace in `items`. + return [(k, '{}') for k in items.split()] + + fields = [ + get_field('sort_of_pandas', 'int8', metadata=meta('pandas')), + + get_field('lots_of_meta', 'int8', metadata=meta('a b c d .. w x y z')), + + get_field( + 'unregistered_extension', 'int8', + metadata=[ + ('ARROW:extension:name', '!nonexistent'), + ('ARROW:extension:metadata', ''), + ('ARROW:integration:allow_unregistered_extension', 'true'), + ]), + + ListField('list_with_odd_values', + get_field('item', 'int32', metadata=meta('odd_values'))), + ] + + batch_sizes = [1] + return _generate_file('custom_metadata', fields, batch_sizes, + metadata=meta('schema_custom_0 schema_custom_1')) + + +def generate_duplicate_fieldnames_case(): + fields = [ + get_field('ints', 'int8'), + get_field('ints', 'int32'), + + StructField('struct', [get_field('', 'int32'), get_field('', 'utf8')]), + ] + + batch_sizes = [1] + return _generate_file('duplicate_fieldnames', fields, batch_sizes) + + +def generate_primitive_case(batch_sizes, name='primitive'): + types = ['bool', 'int8', 'int16', 'int32', 'int64', + 'uint8', 'uint16', 'uint32', 'uint64', + 'float32', 'float64', 'binary', 'utf8', + 'fixedsizebinary_19', 'fixedsizebinary_120'] + + fields = [] + + for type_ in types: + fields.append(get_field(type_ + "_nullable", type_, nullable=True)) + fields.append(get_field(type_ + "_nonnullable", type_, nullable=False)) + + return _generate_file(name, fields, batch_sizes) + + +def generate_primitive_large_offsets_case(batch_sizes): + types = ['largebinary', 'largeutf8'] + + fields = [] + + for type_ in types: + fields.append(get_field(type_ + "_nullable", type_, nullable=True)) + fields.append(get_field(type_ + "_nonnullable", type_, nullable=False)) + + return _generate_file('primitive_large_offsets', fields, batch_sizes) + + +def generate_null_case(batch_sizes): + # Interleave null with non-null types to ensure the appropriate number of + # buffers (0) is read and written + fields = [ + NullField(name='f0'), + get_field('f1', 'int32'), + NullField(name='f2'), + get_field('f3', 'float64'), + NullField(name='f4') + ] + return _generate_file('null', fields, batch_sizes) + + +def generate_null_trivial_case(batch_sizes): + # Generate a case with no buffers + fields = [ + NullField(name='f0'), + ] + return _generate_file('null_trivial', fields, batch_sizes) + + +def generate_decimal128_case(): + fields = [ + DecimalField(name='f{}'.format(i), precision=precision, scale=2, + bit_width=128) + for i, precision in enumerate(range(3, 39)) + ] + + possible_batch_sizes = 7, 10 + batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))] + # 'decimal' is the original name for the test, and it must match + # provide "gold" files that test backwards compatibility, so they + # can be appropriately skipped. + return _generate_file('decimal', fields, batch_sizes) + + +def generate_decimal256_case(): + fields = [ + DecimalField(name='f{}'.format(i), precision=precision, scale=5, + bit_width=256) + for i, precision in enumerate(range(37, 70)) + ] + + possible_batch_sizes = 7, 10 + batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))] + return _generate_file('decimal256', fields, batch_sizes) + + +def generate_datetime_case(): + fields = [ + DateField('f0', DateField.DAY), + DateField('f1', DateField.MILLISECOND), + TimeField('f2', 's'), + TimeField('f3', 'ms'), + TimeField('f4', 'us'), + TimeField('f5', 'ns'), + TimestampField('f6', 's'), + TimestampField('f7', 'ms'), + TimestampField('f8', 'us'), + TimestampField('f9', 'ns'), + TimestampField('f10', 'ms', tz=None), + TimestampField('f11', 's', tz='UTC'), + TimestampField('f12', 'ms', tz='US/Eastern'), + TimestampField('f13', 'us', tz='Europe/Paris'), + TimestampField('f14', 'ns', tz='US/Pacific'), + ] + + batch_sizes = [7, 10] + return _generate_file("datetime", fields, batch_sizes) + + +def generate_interval_case(): + fields = [ + DurationIntervalField('f1', 's'), + DurationIntervalField('f2', 'ms'), + DurationIntervalField('f3', 'us'), + DurationIntervalField('f4', 'ns'), + YearMonthIntervalField('f5'), + DayTimeIntervalField('f6'), + ] + + batch_sizes = [7, 10] + return _generate_file("interval", fields, batch_sizes) + + +def generate_month_day_nano_interval_case(): + fields = [ + MonthDayNanoIntervalField('f1'), + ] + + batch_sizes = [7, 10] + return _generate_file("interval_mdn", fields, batch_sizes) + + +def generate_map_case(): + fields = [ + MapField('map_nullable', get_field('key', 'utf8', nullable=False), + get_field('value', 'int32')), + ] + + batch_sizes = [7, 10] + return _generate_file("map", fields, batch_sizes) + + +def generate_non_canonical_map_case(): + fields = [ + MapField('map_other_names', + get_field('some_key', 'utf8', nullable=False), + get_field('some_value', 'int32'), + entries_name='some_entries'), + ] + + batch_sizes = [7] + return _generate_file("map_non_canonical", fields, batch_sizes) + + +def generate_nested_case(): + fields = [ + ListField('list_nullable', get_field('item', 'int32')), + FixedSizeListField('fixedsizelist_nullable', + get_field('item', 'int32'), 4), + StructField('struct_nullable', [get_field('f1', 'int32'), + get_field('f2', 'utf8')]), + # Fails on Go (ARROW-8452) + # ListField('list_nonnullable', get_field('item', 'int32'), + # nullable=False), + ] + + batch_sizes = [7, 10] + return _generate_file("nested", fields, batch_sizes) + + +def generate_recursive_nested_case(): + fields = [ + ListField('lists_list', + ListField('inner_list', get_field('item', 'int16'))), + ListField('structs_list', + StructField('inner_struct', + [get_field('f1', 'int32'), + get_field('f2', 'utf8')])), + ] + + batch_sizes = [7, 10] + return _generate_file("recursive_nested", fields, batch_sizes) + + +def generate_nested_large_offsets_case(): + fields = [ + LargeListField('large_list_nullable', get_field('item', 'int32')), + LargeListField('large_list_nonnullable', + get_field('item', 'int32'), nullable=False), + LargeListField('large_list_nested', + ListField('inner_list', get_field('item', 'int16'))), + ] + + batch_sizes = [0, 13] + return _generate_file("nested_large_offsets", fields, batch_sizes) + + +def generate_unions_case(): + fields = [ + SparseUnionField('sparse', [get_field('f1', 'int32'), + get_field('f2', 'utf8')], + type_ids=[5, 7]), + DenseUnionField('dense', [get_field('f1', 'int16'), + get_field('f2', 'binary')], + type_ids=[10, 20]), + SparseUnionField('sparse', [get_field('f1', 'float32', nullable=False), + get_field('f2', 'bool')], + type_ids=[5, 7], nullable=False), + DenseUnionField('dense', [get_field('f1', 'uint8', nullable=False), + get_field('f2', 'uint16'), + NullField('f3')], + type_ids=[42, 43, 44], nullable=False), + ] + + batch_sizes = [0, 11] + return _generate_file("union", fields, batch_sizes) + + +def generate_dictionary_case(): + dict0 = Dictionary(0, StringField('dictionary1'), size=10, name='DICT0') + dict1 = Dictionary(1, StringField('dictionary1'), size=5, name='DICT1') + dict2 = Dictionary(2, get_field('dictionary2', 'int64'), + size=50, name='DICT2') + + fields = [ + DictionaryField('dict0', get_field('', 'int8'), dict0), + DictionaryField('dict1', get_field('', 'int32'), dict1), + DictionaryField('dict2', get_field('', 'int16'), dict2) + ] + batch_sizes = [7, 10] + return _generate_file("dictionary", fields, batch_sizes, + dictionaries=[dict0, dict1, dict2]) + + +def generate_dictionary_unsigned_case(): + dict0 = Dictionary(0, StringField('dictionary0'), size=5, name='DICT0') + dict1 = Dictionary(1, StringField('dictionary1'), size=5, name='DICT1') + dict2 = Dictionary(2, StringField('dictionary2'), size=5, name='DICT2') + + # TODO: JavaScript does not support uint64 dictionary indices, so disabled + # for now + + # dict3 = Dictionary(3, StringField('dictionary3'), size=5, name='DICT3') + fields = [ + DictionaryField('f0', get_field('', 'uint8'), dict0), + DictionaryField('f1', get_field('', 'uint16'), dict1), + DictionaryField('f2', get_field('', 'uint32'), dict2), + # DictionaryField('f3', get_field('', 'uint64'), dict3) + ] + batch_sizes = [7, 10] + return _generate_file("dictionary_unsigned", fields, batch_sizes, + dictionaries=[dict0, dict1, dict2]) + + +def generate_nested_dictionary_case(): + dict0 = Dictionary(0, StringField('str'), size=10, name='DICT0') + + list_of_dict = ListField( + 'list', + DictionaryField('str_dict', get_field('', 'int8'), dict0)) + dict1 = Dictionary(1, list_of_dict, size=30, name='DICT1') + + struct_of_dict = StructField('struct', [ + DictionaryField('str_dict_a', get_field('', 'int8'), dict0), + DictionaryField('str_dict_b', get_field('', 'int8'), dict0) + ]) + dict2 = Dictionary(2, struct_of_dict, size=30, name='DICT2') + + fields = [ + DictionaryField('list_dict', get_field('', 'int8'), dict1), + DictionaryField('struct_dict', get_field('', 'int8'), dict2) + ] + + batch_sizes = [10, 13] + return _generate_file("nested_dictionary", fields, batch_sizes, + dictionaries=[dict0, dict1, dict2]) + + +def generate_extension_case(): + dict0 = Dictionary(0, StringField('dictionary0'), size=5, name='DICT0') + + uuid_type = ExtensionType('uuid', 'uuid-serialized', + FixedSizeBinaryField('', 16)) + dict_ext_type = ExtensionType( + 'dict-extension', 'dict-extension-serialized', + DictionaryField('str_dict', get_field('', 'int8'), dict0)) + + fields = [ + ExtensionField('uuids', uuid_type), + ExtensionField('dict_exts', dict_ext_type), + ] + + batch_sizes = [0, 13] + return _generate_file("extension", fields, batch_sizes, + dictionaries=[dict0]) + + +def get_generated_json_files(tempdir=None): + tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-') + + def _temp_path(): + return + + file_objs = [ + generate_primitive_case([], name='primitive_no_batches'), + generate_primitive_case([17, 20], name='primitive'), + generate_primitive_case([0, 0, 0], name='primitive_zerolength'), + + generate_primitive_large_offsets_case([17, 20]) + .skip_category('C#') + .skip_category('Go') + .skip_category('JS'), + + generate_null_case([10, 0]) + .skip_category('C#') + .skip_category('JS'), # TODO(ARROW-7900) + + generate_null_trivial_case([0, 0]) + .skip_category('C#') + .skip_category('JS'), # TODO(ARROW-7900) + + generate_decimal128_case() + .skip_category('Rust'), + + generate_decimal256_case() + .skip_category('Go') # TODO(ARROW-7948): Decimal + Go + .skip_category('JS') + .skip_category('Rust'), + + generate_datetime_case() + .skip_category('C#'), + + generate_interval_case() + .skip_category('C#') + .skip_category('JS') # TODO(ARROW-5239): Intervals + JS + .skip_category('Rust'), + + generate_month_day_nano_interval_case() + .skip_category('C#') + .skip_category('JS') + .skip_category('Rust'), + + + generate_map_case() + .skip_category('C#') + .skip_category('Rust'), + + generate_non_canonical_map_case() + .skip_category('C#') + .skip_category('Java') # TODO(ARROW-8715) + .skip_category('JS') # TODO(ARROW-8716) + .skip_category('Rust'), + + generate_nested_case() + .skip_category('C#'), + + generate_recursive_nested_case() + .skip_category('C#'), + + generate_nested_large_offsets_case() + .skip_category('C#') + .skip_category('Go') + .skip_category('JS') + .skip_category('Rust'), + + generate_unions_case() + .skip_category('C#') + .skip_category('Go') + .skip_category('JS') + .skip_category('Rust'), + + generate_custom_metadata_case() + .skip_category('C#') + .skip_category('JS'), + + generate_duplicate_fieldnames_case() + .skip_category('C#') + .skip_category('Go') + .skip_category('JS'), + + # TODO(ARROW-3039, ARROW-5267): Dictionaries in GO + generate_dictionary_case() + .skip_category('C#') + .skip_category('Go'), + + generate_dictionary_unsigned_case() + .skip_category('C#') + .skip_category('Go') # TODO(ARROW-9378) + .skip_category('Java'), # TODO(ARROW-9377) + + generate_nested_dictionary_case() + .skip_category('C#') + .skip_category('Go') + .skip_category('Java') # TODO(ARROW-7779) + .skip_category('JS') + .skip_category('Rust'), + + generate_extension_case() + .skip_category('C#') + .skip_category('Go') # TODO(ARROW-3039): requires dictionaries + .skip_category('JS') + .skip_category('Rust'), + ] + + generated_paths = [] + for file_obj in file_objs: + out_path = os.path.join(tempdir, 'generated_' + + file_obj.name + '.json') + file_obj.write(out_path) + generated_paths.append(file_obj) + + return generated_paths diff --git a/src/arrow/dev/archery/archery/integration/runner.py b/src/arrow/dev/archery/archery/integration/runner.py new file mode 100644 index 000000000..463917b81 --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/runner.py @@ -0,0 +1,429 @@ +# licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import namedtuple +from concurrent.futures import ThreadPoolExecutor +from functools import partial +import glob +import gzip +import itertools +import os +import sys +import tempfile +import traceback + +from .scenario import Scenario +from .tester_cpp import CPPTester +from .tester_go import GoTester +from .tester_rust import RustTester +from .tester_java import JavaTester +from .tester_js import JSTester +from .tester_csharp import CSharpTester +from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT, + printer) +from . import datagen + + +Failure = namedtuple('Failure', + ('test_case', 'producer', 'consumer', 'exc_info')) + +log = printer.print + + +class Outcome: + def __init__(self): + self.failure = None + self.skipped = False + + +class IntegrationRunner(object): + + def __init__(self, json_files, flight_scenarios, testers, tempdir=None, + debug=False, stop_on_error=True, gold_dirs=None, + serial=False, match=None, **unused_kwargs): + self.json_files = json_files + self.flight_scenarios = flight_scenarios + self.testers = testers + self.temp_dir = tempdir or tempfile.mkdtemp() + self.debug = debug + self.stop_on_error = stop_on_error + self.serial = serial + self.gold_dirs = gold_dirs + self.failures = [] + self.match = match + + if self.match is not None: + print("-- Only running tests with {} in their name" + .format(self.match)) + self.json_files = [json_file for json_file in self.json_files + if self.match in json_file.name] + + def run(self): + """ + Run Arrow IPC integration tests for the matrix of enabled + implementations. + """ + for producer, consumer in itertools.product( + filter(lambda t: t.PRODUCER, self.testers), + filter(lambda t: t.CONSUMER, self.testers)): + self._compare_implementations( + producer, consumer, self._produce_consume, + self.json_files) + if self.gold_dirs: + for gold_dir, consumer in itertools.product( + self.gold_dirs, + filter(lambda t: t.CONSUMER, self.testers)): + log('\n\n\n\n') + log('******************************************************') + log('Tests against golden files in {}'.format(gold_dir)) + log('******************************************************') + + def run_gold(producer, consumer, outcome, test_case): + self._run_gold(gold_dir, producer, consumer, outcome, + test_case) + self._compare_implementations( + consumer, consumer, run_gold, + self._gold_tests(gold_dir)) + + def run_flight(self): + """ + Run Arrow Flight integration tests for the matrix of enabled + implementations. + """ + servers = filter(lambda t: t.FLIGHT_SERVER, self.testers) + clients = filter(lambda t: (t.FLIGHT_CLIENT and t.CONSUMER), + self.testers) + for server, client in itertools.product(servers, clients): + self._compare_flight_implementations(server, client) + + def _gold_tests(self, gold_dir): + prefix = os.path.basename(os.path.normpath(gold_dir)) + SUFFIX = ".json.gz" + golds = [jf for jf in os.listdir(gold_dir) if jf.endswith(SUFFIX)] + for json_path in golds: + name = json_path[json_path.index('_')+1: -len(SUFFIX)] + base_name = prefix + "_" + name + ".gold.json" + out_path = os.path.join(self.temp_dir, base_name) + with gzip.open(os.path.join(gold_dir, json_path)) as i: + with open(out_path, "wb") as out: + out.write(i.read()) + + try: + skip = next(f for f in self.json_files + if f.name == name).skip + except StopIteration: + skip = set() + if name == 'union' and prefix == '0.17.1': + skip.add("Java") + if prefix == '1.0.0-bigendian' or prefix == '1.0.0-littleendian': + skip.add("C#") + skip.add("Go") + skip.add("Java") + skip.add("JS") + skip.add("Rust") + if prefix == '2.0.0-compression': + skip.add("C#") + skip.add("JS") + skip.add("Rust") + + # See https://github.com/apache/arrow/pull/9822 for how to + # disable specific compression type tests. + + if prefix == '4.0.0-shareddict': + skip.add("C#") + skip.add("Go") + + yield datagen.File(name, None, None, skip=skip, path=out_path) + + def _run_test_cases(self, producer, consumer, case_runner, + test_cases): + def case_wrapper(test_case): + with printer.cork(): + return case_runner(test_case) + + if self.failures and self.stop_on_error: + return + + if self.serial: + for outcome in map(case_wrapper, test_cases): + if outcome.failure is not None: + self.failures.append(outcome.failure) + if self.stop_on_error: + break + + else: + with ThreadPoolExecutor() as executor: + for outcome in executor.map(case_wrapper, test_cases): + if outcome.failure is not None: + self.failures.append(outcome.failure) + if self.stop_on_error: + break + + def _compare_implementations( + self, producer, consumer, run_binaries, test_cases): + """ + Compare Arrow IPC for two implementations (one producer, one consumer). + """ + log('##########################################################') + log('IPC: {0} producing, {1} consuming' + .format(producer.name, consumer.name)) + log('##########################################################') + + case_runner = partial(self._run_ipc_test_case, + producer, consumer, run_binaries) + self._run_test_cases(producer, consumer, case_runner, test_cases) + + def _run_ipc_test_case(self, producer, consumer, run_binaries, test_case): + """ + Run one IPC test case. + """ + outcome = Outcome() + + json_path = test_case.path + log('==========================================================') + log('Testing file {0}'.format(json_path)) + log('==========================================================') + + if producer.name in test_case.skip: + log('-- Skipping test because producer {0} does ' + 'not support'.format(producer.name)) + outcome.skipped = True + + elif consumer.name in test_case.skip: + log('-- Skipping test because consumer {0} does ' + 'not support'.format(consumer.name)) + outcome.skipped = True + + elif SKIP_ARROW in test_case.skip: + log('-- Skipping test') + outcome.skipped = True + + else: + try: + run_binaries(producer, consumer, outcome, test_case) + except Exception: + traceback.print_exc(file=printer.stdout) + outcome.failure = Failure(test_case, producer, consumer, + sys.exc_info()) + + return outcome + + def _produce_consume(self, producer, consumer, outcome, test_case): + # Make the random access file + json_path = test_case.path + file_id = guid()[:8] + name = os.path.splitext(os.path.basename(json_path))[0] + + producer_file_path = os.path.join(self.temp_dir, file_id + '_' + + name + '.json_as_file') + producer_stream_path = os.path.join(self.temp_dir, file_id + '_' + + name + '.producer_file_as_stream') + consumer_file_path = os.path.join(self.temp_dir, file_id + '_' + + name + '.consumer_stream_as_file') + + log('-- Creating binary inputs') + producer.json_to_file(json_path, producer_file_path) + + # Validate the file + log('-- Validating file') + consumer.validate(json_path, producer_file_path) + + log('-- Validating stream') + producer.file_to_stream(producer_file_path, producer_stream_path) + consumer.stream_to_file(producer_stream_path, consumer_file_path) + consumer.validate(json_path, consumer_file_path) + + def _run_gold(self, gold_dir, producer, consumer, outcome, test_case): + json_path = test_case.path + + # Validate the file + log('-- Validating file') + producer_file_path = os.path.join( + gold_dir, "generated_" + test_case.name + ".arrow_file") + consumer.validate(json_path, producer_file_path) + + log('-- Validating stream') + consumer_stream_path = os.path.join( + gold_dir, "generated_" + test_case.name + ".stream") + file_id = guid()[:8] + name = os.path.splitext(os.path.basename(json_path))[0] + + consumer_file_path = os.path.join(self.temp_dir, file_id + '_' + + name + '.consumer_stream_as_file') + + consumer.stream_to_file(consumer_stream_path, consumer_file_path) + consumer.validate(json_path, consumer_file_path) + + def _compare_flight_implementations(self, producer, consumer): + log('##########################################################') + log('Flight: {0} serving, {1} requesting' + .format(producer.name, consumer.name)) + log('##########################################################') + + case_runner = partial(self._run_flight_test_case, producer, consumer) + self._run_test_cases(producer, consumer, case_runner, + self.json_files + self.flight_scenarios) + + def _run_flight_test_case(self, producer, consumer, test_case): + """ + Run one Flight test case. + """ + outcome = Outcome() + + log('=' * 58) + log('Testing file {0}'.format(test_case.name)) + log('=' * 58) + + if producer.name in test_case.skip: + log('-- Skipping test because producer {0} does ' + 'not support'.format(producer.name)) + outcome.skipped = True + + elif consumer.name in test_case.skip: + log('-- Skipping test because consumer {0} does ' + 'not support'.format(consumer.name)) + outcome.skipped = True + + elif SKIP_FLIGHT in test_case.skip: + log('-- Skipping test') + outcome.skipped = True + + else: + try: + if isinstance(test_case, Scenario): + server = producer.flight_server(test_case.name) + client_args = {'scenario_name': test_case.name} + else: + server = producer.flight_server() + client_args = {'json_path': test_case.path} + + with server as port: + # Have the client upload the file, then download and + # compare + consumer.flight_request(port, **client_args) + except Exception: + traceback.print_exc(file=printer.stdout) + outcome.failure = Failure(test_case, producer, consumer, + sys.exc_info()) + + return outcome + + +def get_static_json_files(): + glob_pattern = os.path.join(ARROW_ROOT_DEFAULT, + 'integration', 'data', '*.json') + return [ + datagen.File(name=os.path.basename(p), path=p, skip=set(), + schema=None, batches=None) + for p in glob.glob(glob_pattern) + ] + + +def run_all_tests(with_cpp=True, with_java=True, with_js=True, + with_csharp=True, with_go=True, with_rust=False, + run_flight=False, tempdir=None, **kwargs): + tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-') + + testers = [] + + if with_cpp: + testers.append(CPPTester(**kwargs)) + + if with_java: + testers.append(JavaTester(**kwargs)) + + if with_js: + testers.append(JSTester(**kwargs)) + + if with_csharp: + testers.append(CSharpTester(**kwargs)) + + if with_go: + testers.append(GoTester(**kwargs)) + + if with_rust: + testers.append(RustTester(**kwargs)) + + static_json_files = get_static_json_files() + generated_json_files = datagen.get_generated_json_files(tempdir=tempdir) + json_files = static_json_files + generated_json_files + + # Additional integration test cases for Arrow Flight. + flight_scenarios = [ + Scenario( + "auth:basic_proto", + description="Authenticate using the BasicAuth protobuf."), + Scenario( + "middleware", + description="Ensure headers are propagated via middleware.", + skip={"Rust"} # TODO(ARROW-10961): tonic upgrade needed + ), + ] + + runner = IntegrationRunner(json_files, flight_scenarios, testers, **kwargs) + runner.run() + if run_flight: + runner.run_flight() + + fail_count = 0 + if runner.failures: + log("################# FAILURES #################") + for test_case, producer, consumer, exc_info in runner.failures: + fail_count += 1 + log("FAILED TEST:", end=" ") + log(test_case.name, producer.name, "producing, ", + consumer.name, "consuming") + if exc_info: + traceback.print_exception(*exc_info) + log() + + log(fail_count, "failures") + if fail_count > 0: + sys.exit(1) + + +def write_js_test_json(directory): + datagen.generate_map_case().write( + os.path.join(directory, 'map.json') + ) + datagen.generate_nested_case().write( + os.path.join(directory, 'nested.json') + ) + datagen.generate_decimal128_case().write( + os.path.join(directory, 'decimal.json') + ) + datagen.generate_decimal256_case().write( + os.path.join(directory, 'decimal256.json') + ) + datagen.generate_datetime_case().write( + os.path.join(directory, 'datetime.json') + ) + datagen.generate_dictionary_case().write( + os.path.join(directory, 'dictionary.json') + ) + datagen.generate_dictionary_unsigned_case().write( + os.path.join(directory, 'dictionary_unsigned.json') + ) + datagen.generate_primitive_case([]).write( + os.path.join(directory, 'primitive_no_batches.json') + ) + datagen.generate_primitive_case([7, 10]).write( + os.path.join(directory, 'primitive.json') + ) + datagen.generate_primitive_case([0, 0, 0]).write( + os.path.join(directory, 'primitive-empty.json') + ) diff --git a/src/arrow/dev/archery/archery/integration/scenario.py b/src/arrow/dev/archery/archery/integration/scenario.py new file mode 100644 index 000000000..1fcbca64e --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/scenario.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Scenario: + """ + An integration test scenario for Arrow Flight. + + Does not correspond to a particular IPC JSON file. + """ + + def __init__(self, name, description, skip=None): + self.name = name + self.description = description + self.skip = skip or set() diff --git a/src/arrow/dev/archery/archery/integration/tester.py b/src/arrow/dev/archery/archery/integration/tester.py new file mode 100644 index 000000000..122e4f2e4 --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Base class for language-specific integration test harnesses + +import subprocess + +from .util import log + + +class Tester(object): + PRODUCER = False + CONSUMER = False + FLIGHT_SERVER = False + FLIGHT_CLIENT = False + + def __init__(self, debug=False, **args): + self.args = args + self.debug = debug + + def run_shell_command(self, cmd): + cmd = ' '.join(cmd) + if self.debug: + log(cmd) + subprocess.check_call(cmd, shell=True) + + def json_to_file(self, json_path, arrow_path): + raise NotImplementedError + + def stream_to_file(self, stream_path, file_path): + raise NotImplementedError + + def file_to_stream(self, file_path, stream_path): + raise NotImplementedError + + def validate(self, json_path, arrow_path): + raise NotImplementedError + + def flight_server(self, scenario_name=None): + """Start the Flight server on a free port. + + This should be a context manager that returns the port as the + managed object, and cleans up the server on exit. + """ + raise NotImplementedError + + def flight_request(self, port, json_path=None, scenario_name=None): + raise NotImplementedError diff --git a/src/arrow/dev/archery/archery/integration/tester_cpp.py b/src/arrow/dev/archery/archery/integration/tester_cpp.py new file mode 100644 index 000000000..d35c9550e --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_cpp.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import contextlib +import os +import subprocess + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +class CPPTester(Tester): + PRODUCER = True + CONSUMER = True + FLIGHT_SERVER = True + FLIGHT_CLIENT = True + + EXE_PATH = os.environ.get( + 'ARROW_CPP_EXE_PATH', + os.path.join(ARROW_ROOT_DEFAULT, 'cpp/build/debug')) + + CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'arrow-json-integration-test') + STREAM_TO_FILE = os.path.join(EXE_PATH, 'arrow-stream-to-file') + FILE_TO_STREAM = os.path.join(EXE_PATH, 'arrow-file-to-stream') + + FLIGHT_SERVER_CMD = [ + os.path.join(EXE_PATH, 'flight-test-integration-server')] + FLIGHT_CLIENT_CMD = [ + os.path.join(EXE_PATH, 'flight-test-integration-client'), + "-host", "localhost"] + + name = 'C++' + + def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): + cmd = [self.CPP_INTEGRATION_EXE, '--integration'] + + if arrow_path is not None: + cmd.append('--arrow=' + arrow_path) + + if json_path is not None: + cmd.append('--json=' + json_path) + + cmd.append('--mode=' + command) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'VALIDATE') + + def json_to_file(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'JSON_TO_ARROW') + + def stream_to_file(self, stream_path, file_path): + cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path] + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path] + self.run_shell_command(cmd) + + @contextlib.contextmanager + def flight_server(self, scenario_name=None): + cmd = self.FLIGHT_SERVER_CMD + ['-port=0'] + if scenario_name: + cmd = cmd + ["-scenario", scenario_name] + if self.debug: + log(' '.join(cmd)) + server = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + output = server.stdout.readline().decode() + if not output.startswith("Server listening on localhost:"): + server.kill() + out, err = server.communicate() + raise RuntimeError( + "Flight-C++ server did not start properly, " + "stdout:\n{}\n\nstderr:\n{}\n" + .format(output + out.decode(), err.decode())) + port = int(output.split(":")[1]) + yield port + finally: + server.kill() + server.wait(5) + + def flight_request(self, port, json_path=None, scenario_name=None): + cmd = self.FLIGHT_CLIENT_CMD + [ + '-port=' + str(port), + ] + if json_path: + cmd.extend(('-path', json_path)) + elif scenario_name: + cmd.extend(('-scenario', scenario_name)) + else: + raise TypeError("Must provide one of json_path or scenario_name") + + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) diff --git a/src/arrow/dev/archery/archery/integration/tester_csharp.py b/src/arrow/dev/archery/archery/integration/tester_csharp.py new file mode 100644 index 000000000..130c49cfe --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_csharp.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +class CSharpTester(Tester): + PRODUCER = True + CONSUMER = True + + EXE_PATH = os.path.join( + ARROW_ROOT_DEFAULT, + 'csharp/artifacts/Apache.Arrow.IntegrationTest', + 'Debug/netcoreapp3.1/Apache.Arrow.IntegrationTest') + + name = 'C#' + + def _run(self, json_path=None, arrow_path=None, command='validate'): + cmd = [self.EXE_PATH] + + cmd.extend(['--mode', command]) + + if json_path is not None: + cmd.extend(['-j', json_path]) + + if arrow_path is not None: + cmd.extend(['-a', arrow_path]) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(json_path, arrow_path, 'validate') + + def json_to_file(self, json_path, arrow_path): + return self._run(json_path, arrow_path, 'json-to-arrow') + + def stream_to_file(self, stream_path, file_path): + cmd = [self.EXE_PATH] + cmd.extend(['--mode', 'stream-to-file', '-a', file_path]) + cmd.extend(['<', stream_path]) + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = [self.EXE_PATH] + cmd.extend(['--mode', 'file-to-stream']) + cmd.extend(['-a', file_path, '>', stream_path]) + self.run_shell_command(cmd) diff --git a/src/arrow/dev/archery/archery/integration/tester_go.py b/src/arrow/dev/archery/archery/integration/tester_go.py new file mode 100644 index 000000000..eeba38fe5 --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_go.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import contextlib +import os +import subprocess + +from .tester import Tester +from .util import run_cmd, log + + +class GoTester(Tester): + PRODUCER = True + CONSUMER = True + FLIGHT_SERVER = True + FLIGHT_CLIENT = True + + # FIXME(sbinet): revisit for Go modules + HOME = os.getenv('HOME', '~') + GOPATH = os.getenv('GOPATH', os.path.join(HOME, 'go')) + GOBIN = os.environ.get('GOBIN', os.path.join(GOPATH, 'bin')) + + GO_INTEGRATION_EXE = os.path.join(GOBIN, 'arrow-json-integration-test') + STREAM_TO_FILE = os.path.join(GOBIN, 'arrow-stream-to-file') + FILE_TO_STREAM = os.path.join(GOBIN, 'arrow-file-to-stream') + + FLIGHT_SERVER_CMD = [ + os.path.join(GOBIN, 'arrow-flight-integration-server')] + FLIGHT_CLIENT_CMD = [ + os.path.join(GOBIN, 'arrow-flight-integration-client'), + '-host', 'localhost'] + + name = 'Go' + + def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): + cmd = [self.GO_INTEGRATION_EXE] + + if arrow_path is not None: + cmd.extend(['-arrow', arrow_path]) + + if json_path is not None: + cmd.extend(['-json', json_path]) + + cmd.extend(['-mode', command]) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'VALIDATE') + + def json_to_file(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'JSON_TO_ARROW') + + def stream_to_file(self, stream_path, file_path): + cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path] + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path] + self.run_shell_command(cmd) + + @contextlib.contextmanager + def flight_server(self, scenario_name=None): + cmd = self.FLIGHT_SERVER_CMD + ['-port=0'] + if scenario_name: + cmd = cmd + ['-scenario', scenario_name] + if self.debug: + log(' '.join(cmd)) + server = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + try: + output = server.stdout.readline().decode() + if not output.startswith("Server listening on localhost:"): + server.kill() + out, err = server.communicate() + raise RuntimeError( + "Flight-Go server did not start properly, " + "stdout: \n{}\n\nstderr:\n{}\n" + .format(output + out.decode(), err.decode()) + ) + port = int(output.split(":")[1]) + yield port + finally: + server.kill() + server.wait(5) + + def flight_request(self, port, json_path=None, scenario_name=None): + cmd = self.FLIGHT_CLIENT_CMD + [ + '-port=' + str(port), + ] + if json_path: + cmd.extend(('-path', json_path)) + elif scenario_name: + cmd.extend(('-scenario', scenario_name)) + else: + raise TypeError("Must provide one of json_path or scenario_name") + + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) diff --git a/src/arrow/dev/archery/archery/integration/tester_java.py b/src/arrow/dev/archery/archery/integration/tester_java.py new file mode 100644 index 000000000..f283f6cd2 --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_java.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import contextlib +import os +import subprocess + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +def load_version_from_pom(): + import xml.etree.ElementTree as ET + tree = ET.parse(os.path.join(ARROW_ROOT_DEFAULT, 'java', 'pom.xml')) + tag_pattern = '{http://maven.apache.org/POM/4.0.0}version' + version_tag = list(tree.getroot().findall(tag_pattern))[0] + return version_tag.text + + +class JavaTester(Tester): + PRODUCER = True + CONSUMER = True + FLIGHT_SERVER = True + FLIGHT_CLIENT = True + + JAVA_OPTS = ['-Dio.netty.tryReflectionSetAccessible=true', + '-Darrow.struct.conflict.policy=CONFLICT_APPEND'] + + _arrow_version = load_version_from_pom() + ARROW_TOOLS_JAR = os.environ.get( + 'ARROW_JAVA_INTEGRATION_JAR', + os.path.join(ARROW_ROOT_DEFAULT, + 'java/tools/target/arrow-tools-{}-' + 'jar-with-dependencies.jar'.format(_arrow_version))) + ARROW_FLIGHT_JAR = os.environ.get( + 'ARROW_FLIGHT_JAVA_INTEGRATION_JAR', + os.path.join(ARROW_ROOT_DEFAULT, + 'java/flight/flight-core/target/flight-core-{}-' + 'jar-with-dependencies.jar'.format(_arrow_version))) + ARROW_FLIGHT_SERVER = ('org.apache.arrow.flight.example.integration.' + 'IntegrationTestServer') + ARROW_FLIGHT_CLIENT = ('org.apache.arrow.flight.example.integration.' + 'IntegrationTestClient') + + name = 'Java' + + def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): + cmd = ['java'] + self.JAVA_OPTS + \ + ['-cp', self.ARROW_TOOLS_JAR, 'org.apache.arrow.tools.Integration'] + + if arrow_path is not None: + cmd.extend(['-a', arrow_path]) + + if json_path is not None: + cmd.extend(['-j', json_path]) + + cmd.extend(['-c', command]) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'VALIDATE') + + def json_to_file(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'JSON_TO_ARROW') + + def stream_to_file(self, stream_path, file_path): + cmd = ['java'] + self.JAVA_OPTS + \ + ['-cp', self.ARROW_TOOLS_JAR, + 'org.apache.arrow.tools.StreamToFile', stream_path, file_path] + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = ['java'] + self.JAVA_OPTS + \ + ['-cp', self.ARROW_TOOLS_JAR, + 'org.apache.arrow.tools.FileToStream', file_path, stream_path] + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) + + def flight_request(self, port, json_path=None, scenario_name=None): + cmd = ['java'] + self.JAVA_OPTS + \ + ['-cp', self.ARROW_FLIGHT_JAR, self.ARROW_FLIGHT_CLIENT, + '-port', str(port)] + + if json_path: + cmd.extend(('-j', json_path)) + elif scenario_name: + cmd.extend(('-scenario', scenario_name)) + else: + raise TypeError("Must provide one of json_path or scenario_name") + + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) + + @contextlib.contextmanager + def flight_server(self, scenario_name=None): + cmd = ['java'] + self.JAVA_OPTS + \ + ['-cp', self.ARROW_FLIGHT_JAR, self.ARROW_FLIGHT_SERVER, + '-port', '0'] + if scenario_name: + cmd.extend(('-scenario', scenario_name)) + if self.debug: + log(' '.join(cmd)) + server = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + output = server.stdout.readline().decode() + if not output.startswith("Server listening on localhost:"): + server.kill() + out, err = server.communicate() + raise RuntimeError( + "Flight-Java server did not start properly, " + "stdout:\n{}\n\nstderr:\n{}\n" + .format(output + out.decode(), err.decode())) + port = int(output.split(":")[1]) + yield port + finally: + server.kill() + server.wait(5) diff --git a/src/arrow/dev/archery/archery/integration/tester_js.py b/src/arrow/dev/archery/archery/integration/tester_js.py new file mode 100644 index 000000000..e24eec0ca --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_js.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +class JSTester(Tester): + PRODUCER = True + CONSUMER = True + + EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin') + VALIDATE = os.path.join(EXE_PATH, 'integration.js') + JSON_TO_ARROW = os.path.join(EXE_PATH, 'json-to-arrow.js') + STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file.js') + FILE_TO_STREAM = os.path.join(EXE_PATH, 'file-to-stream.js') + + name = 'JS' + + def _run(self, exe_cmd, arrow_path=None, json_path=None, + command='VALIDATE'): + cmd = [exe_cmd] + + if arrow_path is not None: + cmd.extend(['-a', arrow_path]) + + if json_path is not None: + cmd.extend(['-j', json_path]) + + cmd.extend(['--mode', command]) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(self.VALIDATE, arrow_path, json_path, 'VALIDATE') + + def json_to_file(self, json_path, arrow_path): + cmd = ['node', + '--no-warnings', self.JSON_TO_ARROW, + '-a', arrow_path, + '-j', json_path] + self.run_shell_command(cmd) + + def stream_to_file(self, stream_path, file_path): + cmd = ['node', '--no-warnings', self.STREAM_TO_FILE, + '<', stream_path, + '>', file_path] + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = ['node', '--no-warnings', self.FILE_TO_STREAM, + '<', file_path, + '>', stream_path] + self.run_shell_command(cmd) diff --git a/src/arrow/dev/archery/archery/integration/tester_rust.py b/src/arrow/dev/archery/archery/integration/tester_rust.py new file mode 100644 index 000000000..bca80ebae --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/tester_rust.py @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import contextlib +import os +import subprocess + +from .tester import Tester +from .util import run_cmd, ARROW_ROOT_DEFAULT, log + + +class RustTester(Tester): + PRODUCER = True + CONSUMER = True + FLIGHT_SERVER = True + FLIGHT_CLIENT = True + + EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'rust/target/debug') + + RUST_INTEGRATION_EXE = os.path.join(EXE_PATH, + 'arrow-json-integration-test') + STREAM_TO_FILE = os.path.join(EXE_PATH, 'arrow-stream-to-file') + FILE_TO_STREAM = os.path.join(EXE_PATH, 'arrow-file-to-stream') + + FLIGHT_SERVER_CMD = [ + os.path.join(EXE_PATH, 'flight-test-integration-server')] + FLIGHT_CLIENT_CMD = [ + os.path.join(EXE_PATH, 'flight-test-integration-client'), + "--host", "localhost"] + + name = 'Rust' + + def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): + cmd = [self.RUST_INTEGRATION_EXE, '--integration'] + + if arrow_path is not None: + cmd.append('--arrow=' + arrow_path) + + if json_path is not None: + cmd.append('--json=' + json_path) + + cmd.append('--mode=' + command) + + if self.debug: + log(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'VALIDATE') + + def json_to_file(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'JSON_TO_ARROW') + + def stream_to_file(self, stream_path, file_path): + cmd = [self.STREAM_TO_FILE, '<', stream_path, '>', file_path] + self.run_shell_command(cmd) + + def file_to_stream(self, file_path, stream_path): + cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path] + self.run_shell_command(cmd) + + @contextlib.contextmanager + def flight_server(self, scenario_name=None): + cmd = self.FLIGHT_SERVER_CMD + ['--port=0'] + if scenario_name: + cmd = cmd + ["--scenario", scenario_name] + if self.debug: + log(' '.join(cmd)) + server = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + output = server.stdout.readline().decode() + if not output.startswith("Server listening on localhost:"): + server.kill() + out, err = server.communicate() + raise RuntimeError( + "Flight-Rust server did not start properly, " + "stdout:\n{}\n\nstderr:\n{}\n" + .format(output + out.decode(), err.decode())) + port = int(output.split(":")[1]) + yield port + finally: + server.kill() + server.wait(5) + + def flight_request(self, port, json_path=None, scenario_name=None): + cmd = self.FLIGHT_CLIENT_CMD + [ + '--port=' + str(port), + ] + if json_path: + cmd.extend(('--path', json_path)) + elif scenario_name: + cmd.extend(('--scenario', scenario_name)) + else: + raise TypeError("Must provide one of json_path or scenario_name") + + if self.debug: + log(' '.join(cmd)) + run_cmd(cmd) diff --git a/src/arrow/dev/archery/archery/integration/util.py b/src/arrow/dev/archery/archery/integration/util.py new file mode 100644 index 000000000..a4c4982ec --- /dev/null +++ b/src/arrow/dev/archery/archery/integration/util.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import contextlib +import io +import os +import random +import socket +import subprocess +import sys +import threading +import uuid + +import numpy as np + + +def guid(): + return uuid.uuid4().hex + + +# SKIP categories +SKIP_ARROW = 'arrow' +SKIP_FLIGHT = 'flight' + +ARROW_ROOT_DEFAULT = os.environ.get( + 'ARROW_ROOT', + os.path.abspath(__file__).rsplit("/", 5)[0] +) + + +class _Printer: + """ + A print()-providing object that can override the stream output on + a per-thread basis. + """ + + def __init__(self): + self._tls = threading.local() + + def _get_stdout(self): + try: + return self._tls.stdout + except AttributeError: + self._tls.stdout = sys.stdout + self._tls.corked = False + return self._tls.stdout + + def print(self, *args, **kwargs): + """ + A variant of print() that writes to a thread-local stream. + """ + print(*args, file=self._get_stdout(), **kwargs) + + @property + def stdout(self): + """ + A thread-local stdout wrapper that may be temporarily buffered + using `cork()`. + """ + return self._get_stdout() + + @contextlib.contextmanager + def cork(self): + """ + Temporarily buffer this thread's stream and write out its contents + at the end of the context manager. Useful to avoid interleaved + output when multiple threads output progress information. + """ + outer_stdout = self._get_stdout() + assert not self._tls.corked, "reentrant call" + inner_stdout = self._tls.stdout = io.StringIO() + self._tls.corked = True + try: + yield + finally: + self._tls.stdout = outer_stdout + self._tls.corked = False + outer_stdout.write(inner_stdout.getvalue()) + outer_stdout.flush() + + +printer = _Printer() +log = printer.print + + +_RAND_CHARS = np.array(list("abcdefghijklmnop123456Ârrôwµ£°€矢"), dtype="U") + + +def random_utf8(nchars): + """ + Generate one random UTF8 string. + """ + return ''.join(np.random.choice(_RAND_CHARS, nchars)) + + +def random_bytes(nbytes): + """ + Generate one random binary string. + """ + # NOTE getrandbits(0) fails + if nbytes > 0: + return random.getrandbits(nbytes * 8).to_bytes(nbytes, + byteorder='little') + else: + return b"" + + +def tobytes(o): + if isinstance(o, str): + return o.encode('utf8') + return o + + +def frombytes(o): + if isinstance(o, bytes): + return o.decode('utf8') + return o + + +def run_cmd(cmd): + if isinstance(cmd, str): + cmd = cmd.split(' ') + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + # this avoids hiding the stdout / stderr of failed processes + sio = io.StringIO() + print('Command failed:', " ".join(cmd), file=sio) + print('With output:', file=sio) + print('--------------', file=sio) + print(frombytes(e.output), file=sio) + print('--------------', file=sio) + raise RuntimeError(sio.getvalue()) + + return frombytes(output) + + +# Adapted from CPython +def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): + """Returns an unused port that should be suitable for binding. This is + achieved by creating a temporary socket with the same family and type as + the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to + the specified host address (defaults to 0.0.0.0) with the port set to 0, + eliciting an unused ephemeral port from the OS. The temporary socket is + then closed and deleted, and the ephemeral port is returned. + """ + with socket.socket(family, socktype) as tempsock: + tempsock.bind(('', 0)) + port = tempsock.getsockname()[1] + del tempsock + return port diff --git a/src/arrow/dev/archery/archery/lang/__init__.py b/src/arrow/dev/archery/archery/lang/__init__.py new file mode 100644 index 000000000..13a83393a --- /dev/null +++ b/src/arrow/dev/archery/archery/lang/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/arrow/dev/archery/archery/lang/cpp.py b/src/arrow/dev/archery/archery/lang/cpp.py new file mode 100644 index 000000000..c2b1ca680 --- /dev/null +++ b/src/arrow/dev/archery/archery/lang/cpp.py @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from ..utils.cmake import CMakeDefinition + + +def truthifier(value): + return "ON" if value else "OFF" + + +def or_else(value, default): + return value if value else default + + +def coalesce(value, fallback): + return fallback if value is None else value + + +LLVM_VERSION = 7 + + +class CppConfiguration: + def __init__(self, + + # toolchain + cc=None, cxx=None, cxx_flags=None, + build_type=None, warn_level=None, + cpp_package_prefix=None, install_prefix=None, use_conda=None, + build_static=False, build_shared=True, build_unity=True, + # tests & examples + with_tests=None, with_benchmarks=None, with_examples=None, + with_integration=None, + # static checks + use_asan=None, use_tsan=None, use_ubsan=None, + with_fuzzing=None, + # Components + with_compute=None, with_csv=None, with_cuda=None, + with_dataset=None, with_filesystem=None, with_flight=None, + with_gandiva=None, with_hdfs=None, with_hiveserver2=None, + with_ipc=True, with_json=None, with_jni=None, + with_mimalloc=None, + with_parquet=None, with_plasma=None, with_python=True, + with_r=None, with_s3=None, + # Compressions + with_brotli=None, with_bz2=None, with_lz4=None, + with_snappy=None, with_zlib=None, with_zstd=None, + # extras + with_lint_only=False, + use_gold_linker=True, + simd_level="SSE4_2", + cmake_extras=None): + self._cc = cc + self._cxx = cxx + self.cxx_flags = cxx_flags + + self._build_type = build_type + self.warn_level = warn_level + self._install_prefix = install_prefix + self._package_prefix = cpp_package_prefix + self._use_conda = use_conda + self.build_static = build_static + self.build_shared = build_shared + self.build_unity = build_unity + + self.with_tests = with_tests + self.with_benchmarks = with_benchmarks + self.with_examples = with_examples + self.with_integration = with_integration + + self.use_asan = use_asan + self.use_tsan = use_tsan + self.use_ubsan = use_ubsan + self.with_fuzzing = with_fuzzing + + self.with_compute = with_compute + self.with_csv = with_csv + self.with_cuda = with_cuda + self.with_dataset = with_dataset + self.with_filesystem = with_filesystem + self.with_flight = with_flight + self.with_gandiva = with_gandiva + self.with_hdfs = with_hdfs + self.with_hiveserver2 = with_hiveserver2 + self.with_ipc = with_ipc + self.with_json = with_json + self.with_jni = with_jni + self.with_mimalloc = with_mimalloc + self.with_parquet = with_parquet + self.with_plasma = with_plasma + self.with_python = with_python + self.with_r = with_r + self.with_s3 = with_s3 + + self.with_brotli = with_brotli + self.with_bz2 = with_bz2 + self.with_lz4 = with_lz4 + self.with_snappy = with_snappy + self.with_zlib = with_zlib + self.with_zstd = with_zstd + + self.with_lint_only = with_lint_only + self.use_gold_linker = use_gold_linker + self.simd_level = simd_level + + self.cmake_extras = cmake_extras + + # Fixup required dependencies by providing sane defaults if the caller + # didn't specify the option. + if self.with_r: + self.with_csv = coalesce(with_csv, True) + self.with_dataset = coalesce(with_dataset, True) + self.with_filesystem = coalesce(with_filesystem, True) + self.with_ipc = coalesce(with_ipc, True) + self.with_json = coalesce(with_json, True) + self.with_parquet = coalesce(with_parquet, True) + + if self.with_python: + self.with_zlib = coalesce(with_zlib, True) + self.with_lz4 = coalesce(with_lz4, True) + + if self.with_dataset: + self.with_filesystem = coalesce(with_filesystem, True) + self.with_parquet = coalesce(with_parquet, True) + + if self.with_parquet: + self.with_snappy = coalesce(with_snappy, True) + + @property + def build_type(self): + if self._build_type: + return self._build_type + + if self.with_fuzzing: + return "relwithdebinfo" + + return "release" + + @property + def cc(self): + if self._cc: + return self._cc + + if self.with_fuzzing: + return "clang-{}".format(LLVM_VERSION) + + return None + + @property + def cxx(self): + if self._cxx: + return self._cxx + + if self.with_fuzzing: + return "clang++-{}".format(LLVM_VERSION) + + return None + + def _gen_defs(self): + if self.cxx_flags: + yield ("ARROW_CXXFLAGS", self.cxx_flags) + + yield ("CMAKE_EXPORT_COMPILE_COMMANDS", truthifier(True)) + yield ("CMAKE_BUILD_TYPE", self.build_type) + + if not self.with_lint_only: + yield ("BUILD_WARNING_LEVEL", + or_else(self.warn_level, "production")) + + # if not ctx.quiet: + # yield ("ARROW_VERBOSE_THIRDPARTY_BUILD", "ON") + + maybe_prefix = self.install_prefix + if maybe_prefix: + yield ("CMAKE_INSTALL_PREFIX", maybe_prefix) + + if self._package_prefix is not None: + yield ("ARROW_DEPENDENCY_SOURCE", "SYSTEM") + yield ("ARROW_PACKAGE_PREFIX", self._package_prefix) + + yield ("ARROW_BUILD_STATIC", truthifier(self.build_static)) + yield ("ARROW_BUILD_SHARED", truthifier(self.build_shared)) + yield ("CMAKE_UNITY_BUILD", truthifier(self.build_unity)) + + # Tests and benchmarks + yield ("ARROW_BUILD_TESTS", truthifier(self.with_tests)) + yield ("ARROW_BUILD_BENCHMARKS", truthifier(self.with_benchmarks)) + yield ("ARROW_BUILD_EXAMPLES", truthifier(self.with_examples)) + yield ("ARROW_BUILD_INTEGRATION", truthifier(self.with_integration)) + + # Static checks + yield ("ARROW_USE_ASAN", truthifier(self.use_asan)) + yield ("ARROW_USE_TSAN", truthifier(self.use_tsan)) + yield ("ARROW_USE_UBSAN", truthifier(self.use_ubsan)) + yield ("ARROW_FUZZING", truthifier(self.with_fuzzing)) + + # Components + yield ("ARROW_COMPUTE", truthifier(self.with_compute)) + yield ("ARROW_CSV", truthifier(self.with_csv)) + yield ("ARROW_CUDA", truthifier(self.with_cuda)) + yield ("ARROW_DATASET", truthifier(self.with_dataset)) + yield ("ARROW_FILESYSTEM", truthifier(self.with_filesystem)) + yield ("ARROW_FLIGHT", truthifier(self.with_flight)) + yield ("ARROW_GANDIVA", truthifier(self.with_gandiva)) + yield ("ARROW_PARQUET", truthifier(self.with_parquet)) + yield ("ARROW_HDFS", truthifier(self.with_hdfs)) + yield ("ARROW_HIVESERVER2", truthifier(self.with_hiveserver2)) + yield ("ARROW_IPC", truthifier(self.with_ipc)) + yield ("ARROW_JSON", truthifier(self.with_json)) + yield ("ARROW_JNI", truthifier(self.with_jni)) + yield ("ARROW_MIMALLOC", truthifier(self.with_mimalloc)) + yield ("ARROW_PLASMA", truthifier(self.with_plasma)) + yield ("ARROW_PYTHON", truthifier(self.with_python)) + yield ("ARROW_S3", truthifier(self.with_s3)) + + # Compressions + yield ("ARROW_WITH_BROTLI", truthifier(self.with_brotli)) + yield ("ARROW_WITH_BZ2", truthifier(self.with_bz2)) + yield ("ARROW_WITH_LZ4", truthifier(self.with_lz4)) + yield ("ARROW_WITH_SNAPPY", truthifier(self.with_snappy)) + yield ("ARROW_WITH_ZLIB", truthifier(self.with_zlib)) + yield ("ARROW_WITH_ZSTD", truthifier(self.with_zstd)) + + yield ("ARROW_LINT_ONLY", truthifier(self.with_lint_only)) + + # Some configurations don't like gnu gold linker. + broken_with_gold_ld = [self.with_fuzzing, self.with_gandiva] + if self.use_gold_linker and not any(broken_with_gold_ld): + yield ("ARROW_USE_LD_GOLD", truthifier(self.use_gold_linker)) + yield ("ARROW_SIMD_LEVEL", or_else(self.simd_level, "SSE4_2")) + + # Detect custom conda toolchain + if self.use_conda: + for d, v in [('CMAKE_AR', 'AR'), ('CMAKE_RANLIB', 'RANLIB')]: + v = os.environ.get(v) + if v: + yield (d, v) + + @property + def install_prefix(self): + if self._install_prefix: + return self._install_prefix + + if self.use_conda: + return os.environ.get("CONDA_PREFIX") + + return None + + @property + def use_conda(self): + # If the user didn't specify a preference, guess via environment + if self._use_conda is None: + return os.environ.get("CONDA_PREFIX") is not None + + return self._use_conda + + @property + def definitions(self): + extras = list(self.cmake_extras) if self.cmake_extras else [] + definitions = ["-D{}={}".format(d[0], d[1]) for d in self._gen_defs()] + return definitions + extras + + @property + def environment(self): + env = os.environ.copy() + + if self.cc: + env["CC"] = self.cc + + if self.cxx: + env["CXX"] = self.cxx + + return env + + +class CppCMakeDefinition(CMakeDefinition): + def __init__(self, source, conf, **kwargs): + self.configuration = conf + super().__init__(source, **kwargs, + definitions=conf.definitions, env=conf.environment, + build_type=conf.build_type) diff --git a/src/arrow/dev/archery/archery/lang/java.py b/src/arrow/dev/archery/archery/lang/java.py new file mode 100644 index 000000000..bc169adf6 --- /dev/null +++ b/src/arrow/dev/archery/archery/lang/java.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from ..utils.command import Command, CommandStackMixin, default_bin +from ..utils.maven import MavenDefinition + + +class Java(Command): + def __init__(self, java_bin=None): + self.bin = default_bin(java_bin, "java") + + +class Jar(CommandStackMixin, Java): + def __init__(self, jar, *args, **kwargs): + self.jar = jar + self.argv = ("-jar", jar) + Java.__init__(self, *args, **kwargs) + + +class JavaConfiguration: + def __init__(self, + + # toolchain + java_home=None, java_options=None, + # build & benchmark + build_extras=None, benchmark_extras=None): + self.java_home = java_home + self.java_options = java_options + + self.build_extras = list(build_extras) if build_extras else [] + self.benchmark_extras = list( + benchmark_extras) if benchmark_extras else [] + + @property + def build_definitions(self): + return self.build_extras + + @property + def benchmark_definitions(self): + return self.benchmark_extras + + @property + def environment(self): + env = os.environ.copy() + + if self.java_home: + env["JAVA_HOME"] = self.java_home + + if self.java_options: + env["JAVA_OPTIONS"] = self.java_options + + return env + + +class JavaMavenDefinition(MavenDefinition): + def __init__(self, source, conf, **kwargs): + self.configuration = conf + super().__init__(source, **kwargs, + build_definitions=conf.build_definitions, + benchmark_definitions=conf.benchmark_definitions, + env=conf.environment) diff --git a/src/arrow/dev/archery/archery/lang/python.py b/src/arrow/dev/archery/archery/lang/python.py new file mode 100644 index 000000000..c6ebbe650 --- /dev/null +++ b/src/arrow/dev/archery/archery/lang/python.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import inspect +import tokenize +from contextlib import contextmanager + +try: + from numpydoc.validate import Docstring, validate +except ImportError: + have_numpydoc = False +else: + have_numpydoc = True + +from ..utils.logger import logger +from ..utils.command import Command, capture_stdout, default_bin + + +class Flake8(Command): + def __init__(self, flake8_bin=None): + self.bin = default_bin(flake8_bin, "flake8") + + +class Autopep8(Command): + def __init__(self, autopep8_bin=None): + self.bin = default_bin(autopep8_bin, "autopep8") + + @capture_stdout() + def run_captured(self, *args, **kwargs): + return self.run(*args, **kwargs) + + +def _tokenize_signature(s): + lines = s.encode('ascii').splitlines() + generator = iter(lines).__next__ + return tokenize.tokenize(generator) + + +def _convert_typehint(tokens): + names = [] + opening_bracket_reached = False + for token in tokens: + # omit the tokens before the opening bracket + if not opening_bracket_reached: + if token.string == '(': + opening_bracket_reached = True + else: + continue + + if token.type == 1: # type 1 means NAME token + names.append(token) + else: + if len(names) == 1: + yield (names[0].type, names[0].string) + elif len(names) == 2: + # two "NAME" tokens follow each other which means a cython + # typehint like `bool argument`, so remove the typehint + # note that we could convert it to python typehints, but hints + # are not supported by _signature_fromstr + yield (names[1].type, names[1].string) + elif len(names) > 2: + raise ValueError('More than two NAME tokens follow each other') + names = [] + yield (token.type, token.string) + + +def inspect_signature(obj): + """ + Custom signature inspection primarily for cython generated callables. + + Cython puts the signatures to the first line of the docstrings, which we + can reuse to parse the python signature from, but some gymnastics are + required, like removing the cython typehints. + + It converts the cython signature: + array(obj, type=None, mask=None, size=None, from_pandas=None, + bool safe=True, MemoryPool memory_pool=None) + To: + <Signature (obj, type=None, mask=None, size=None, from_pandas=None, + safe=True, memory_pool=None)> + """ + cython_signature = obj.__doc__.splitlines()[0] + cython_tokens = _tokenize_signature(cython_signature) + python_tokens = _convert_typehint(cython_tokens) + python_signature = tokenize.untokenize(python_tokens) + return inspect._signature_fromstr(inspect.Signature, obj, python_signature) + + +class NumpyDoc: + + def __init__(self, symbols=None): + if not have_numpydoc: + raise RuntimeError( + 'Numpydoc is not available, install the development version ' + 'with command: pip install numpydoc==1.1.0' + ) + self.symbols = set(symbols or {'pyarrow'}) + + def traverse(self, fn, obj, from_package): + """Apply a function on publicly exposed API components. + + Recursively iterates over the members of the passed object. It omits + any '_' prefixed and thirdparty (non pyarrow) symbols. + + Parameters + ---------- + obj : Any + from_package : string, default 'pyarrow' + Predicate to only consider objects from this package. + """ + todo = [obj] + seen = set() + + while todo: + obj = todo.pop() + if obj in seen: + continue + else: + seen.add(obj) + + fn(obj) + + for name in dir(obj): + if name.startswith('_'): + continue + + member = getattr(obj, name) + module = getattr(member, '__module__', None) + if not (module and module.startswith(from_package)): + continue + + todo.append(member) + + @contextmanager + def _apply_patches(self): + """ + Patch Docstring class to bypass loading already loaded python objects. + """ + orig_load_obj = Docstring._load_obj + orig_signature = inspect.signature + + @staticmethod + def _load_obj(obj): + # By default it expects a qualname and import the object, but we + # have already loaded object after the API traversal. + if isinstance(obj, str): + return orig_load_obj(obj) + else: + return obj + + def signature(obj): + # inspect.signature tries to parse __text_signature__ if other + # properties like __signature__ doesn't exists, but cython + # doesn't set that property despite that embedsignature cython + # directive is set. The only way to inspect a cython compiled + # callable's signature to parse it from __doc__ while + # embedsignature directive is set during the build phase. + # So path inspect.signature function to attempt to parse the first + # line of callable.__doc__ as a signature. + try: + return orig_signature(obj) + except Exception as orig_error: + try: + return inspect_signature(obj) + except Exception: + raise orig_error + + try: + Docstring._load_obj = _load_obj + inspect.signature = signature + yield + finally: + Docstring._load_obj = orig_load_obj + inspect.signature = orig_signature + + def validate(self, from_package='', allow_rules=None, + disallow_rules=None): + results = [] + + def callback(obj): + try: + result = validate(obj) + except OSError as e: + symbol = f"{obj.__module__}.{obj.__name__}" + logger.warning(f"Unable to validate `{symbol}` due to `{e}`") + return + + errors = [] + for errcode, errmsg in result.get('errors', []): + if allow_rules and errcode not in allow_rules: + continue + if disallow_rules and errcode in disallow_rules: + continue + errors.append((errcode, errmsg)) + + if len(errors): + result['errors'] = errors + results.append((obj, result)) + + with self._apply_patches(): + for symbol in self.symbols: + try: + obj = Docstring._load_obj(symbol) + except (ImportError, AttributeError): + print('{} is not available for import'.format(symbol)) + else: + self.traverse(callback, obj, from_package=from_package) + + return results diff --git a/src/arrow/dev/archery/archery/linking.py b/src/arrow/dev/archery/archery/linking.py new file mode 100644 index 000000000..c2e6f1772 --- /dev/null +++ b/src/arrow/dev/archery/archery/linking.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import platform +import subprocess + +from .utils.command import Command + + +_ldd = Command("ldd") +_otool = Command("otool") + + +class DependencyError(Exception): + pass + + +class DynamicLibrary: + + def __init__(self, path): + self.path = path + + def list_dependencies(self): + """ + List the full name of the library dependencies. + """ + system = platform.system() + if system == "Linux": + result = _ldd.run(self.path, stdout=subprocess.PIPE) + lines = result.stdout.splitlines() + return [ll.split(None, 1)[0].decode() for ll in lines] + elif system == "Darwin": + result = _otool.run("-L", self.path, stdout=subprocess.PIPE) + lines = result.stdout.splitlines() + return [dl.split(None, 1)[0].decode() for dl in lines] + else: + raise ValueError(f"{platform} is not supported") + + def list_dependency_names(self): + """ + List the truncated names of the dynamic library dependencies. + """ + names = [] + for dependency in self.list_dependencies(): + *_, library = dependency.rsplit("/", 1) + name, *_ = library.split(".", 1) + names.append(name) + return names + + +def check_dynamic_library_dependencies(path, allowed, disallowed): + dylib = DynamicLibrary(path) + for dep in dylib.list_dependency_names(): + if allowed and dep not in allowed: + raise DependencyError( + f"Unexpected shared dependency found in {dylib.path}: `{dep}`" + ) + if disallowed and dep in disallowed: + raise DependencyError( + f"Disallowed shared dependency found in {dylib.path}: `{dep}`" + ) diff --git a/src/arrow/dev/archery/archery/release.py b/src/arrow/dev/archery/archery/release.py new file mode 100644 index 000000000..6baeabc9d --- /dev/null +++ b/src/arrow/dev/archery/archery/release.py @@ -0,0 +1,535 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import defaultdict +import functools +import os +import re +import pathlib +import shelve +import warnings + +from git import Repo +from jira import JIRA +from semver import VersionInfo as SemVer + +from .utils.source import ArrowSources +from .utils.report import JinjaReport + + +def cached_property(fn): + return property(functools.lru_cache(maxsize=1)(fn)) + + +class Version(SemVer): + + __slots__ = ('released', 'release_date') + + def __init__(self, released=False, release_date=None, **kwargs): + super().__init__(**kwargs) + self.released = released + self.release_date = release_date + + @classmethod + def parse(cls, version, **kwargs): + return cls(**SemVer.parse(version).to_dict(), **kwargs) + + @classmethod + def from_jira(cls, jira_version): + return cls.parse( + jira_version.name, + released=jira_version.released, + release_date=getattr(jira_version, 'releaseDate', None) + ) + + +class Issue: + + def __init__(self, key, type, summary): + self.key = key + self.type = type + self.summary = summary + + @classmethod + def from_jira(cls, jira_issue): + return cls( + key=jira_issue.key, + type=jira_issue.fields.issuetype.name, + summary=jira_issue.fields.summary + ) + + @property + def project(self): + return self.key.split('-')[0] + + @property + def number(self): + return int(self.key.split('-')[1]) + + +class Jira(JIRA): + + def __init__(self, user=None, password=None, + url='https://issues.apache.org/jira'): + user = user or os.environ.get('APACHE_JIRA_USER') + password = password or os.environ.get('APACHE_JIRA_PASSWORD') + super().__init__(url, basic_auth=(user, password)) + + def project_version(self, version_string, project='ARROW'): + # query version from jira to populated with additional metadata + versions = {str(v): v for v in self.project_versions(project)} + return versions[version_string] + + def project_versions(self, project): + versions = [] + for v in super().project_versions(project): + try: + versions.append(Version.from_jira(v)) + except ValueError: + # ignore invalid semantic versions like JS-0.4.0 + continue + return sorted(versions, reverse=True) + + def issue(self, key): + return Issue.from_jira(super().issue(key)) + + def project_issues(self, version, project='ARROW'): + query = "project={} AND fixVersion={}".format(project, version) + issues = super().search_issues(query, maxResults=False) + return list(map(Issue.from_jira, issues)) + + +class CachedJira: + + def __init__(self, cache_path, jira=None): + self.jira = jira or Jira() + self.cache_path = cache_path + + def __getattr__(self, name): + attr = getattr(self.jira, name) + return self._cached(name, attr) if callable(attr) else attr + + def _cached(self, name, method): + def wrapper(*args, **kwargs): + key = str((name, args, kwargs)) + with shelve.open(self.cache_path) as cache: + try: + result = cache[key] + except KeyError: + cache[key] = result = method(*args, **kwargs) + return result + return wrapper + + +_TITLE_REGEX = re.compile( + r"(?P<issue>(?P<project>(ARROW|PARQUET))\-\d+)?\s*:?\s*" + r"(?P<components>\[.*\])?\s*(?P<summary>.*)" +) +_COMPONENT_REGEX = re.compile(r"\[([^\[\]]+)\]") + + +class CommitTitle: + + def __init__(self, summary, project=None, issue=None, components=None): + self.project = project + self.issue = issue + self.components = components or [] + self.summary = summary + + def __str__(self): + out = "" + if self.issue: + out += "{}: ".format(self.issue) + if self.components: + for component in self.components: + out += "[{}]".format(component) + out += " " + out += self.summary + return out + + def __eq__(self, other): + return ( + self.summary == other.summary and + self.project == other.project and + self.issue == other.issue and + self.components == other.components + ) + + def __hash__(self): + return hash( + (self.summary, self.project, self.issue, tuple(self.components)) + ) + + @classmethod + def parse(cls, headline): + matches = _TITLE_REGEX.match(headline) + if matches is None: + warnings.warn( + "Unable to parse commit message `{}`".format(headline) + ) + return CommitTitle(headline) + + values = matches.groupdict() + components = values.get('components') or '' + components = _COMPONENT_REGEX.findall(components) + + return CommitTitle( + values['summary'], + project=values.get('project'), + issue=values.get('issue'), + components=components + ) + + +class Commit: + + def __init__(self, wrapped): + self._title = CommitTitle.parse(wrapped.summary) + self._wrapped = wrapped + + def __getattr__(self, attr): + if hasattr(self._title, attr): + return getattr(self._title, attr) + else: + return getattr(self._wrapped, attr) + + def __repr__(self): + template = '<Commit sha={!r} issue={!r} components={!r} summary={!r}>' + return template.format(self.hexsha, self.issue, self.components, + self.summary) + + @property + def url(self): + return 'https://github.com/apache/arrow/commit/{}'.format(self.hexsha) + + @property + def title(self): + return self._title + + +class ReleaseCuration(JinjaReport): + templates = { + 'console': 'release_curation.txt.j2' + } + fields = [ + 'release', + 'within', + 'outside', + 'nojira', + 'parquet', + 'nopatch' + ] + + +class JiraChangelog(JinjaReport): + templates = { + 'markdown': 'release_changelog.md.j2', + 'html': 'release_changelog.html.j2' + } + fields = [ + 'release', + 'categories' + ] + + +class Release: + + def __init__(self): + raise TypeError("Do not initialize Release class directly, use " + "Release.from_jira(version) instead.") + + def __repr__(self): + if self.version.released: + status = "released_at={!r}".format(self.version.release_date) + else: + status = "pending" + return "<{} {!r} {}>".format(self.__class__.__name__, + str(self.version), status) + + @staticmethod + def from_jira(version, jira=None, repo=None): + if jira is None: + jira = Jira() + elif isinstance(jira, str): + jira = Jira(jira) + elif not isinstance(jira, (Jira, CachedJira)): + raise TypeError("`jira` argument must be a server url or a valid " + "Jira instance") + + if repo is None: + arrow = ArrowSources.find() + repo = Repo(arrow.path) + elif isinstance(repo, (str, pathlib.Path)): + repo = Repo(repo) + elif not isinstance(repo, Repo): + raise TypeError("`repo` argument must be a path or a valid Repo " + "instance") + + if isinstance(version, str): + version = jira.project_version(version, project='ARROW') + elif not isinstance(version, Version): + raise TypeError(version) + + # decide the type of the release based on the version number + if version.patch == 0: + if version.minor == 0: + klass = MajorRelease + elif version.major == 0: + # handle minor releases before 1.0 as major releases + klass = MajorRelease + else: + klass = MinorRelease + else: + klass = PatchRelease + + # prevent instantiating release object directly + obj = klass.__new__(klass) + obj.version = version + obj.jira = jira + obj.repo = repo + + return obj + + @property + def is_released(self): + return self.version.released + + @property + def tag(self): + return "apache-arrow-{}".format(str(self.version)) + + @property + def branch(self): + raise NotImplementedError() + + @property + def siblings(self): + """ + Releases to consider when calculating previous and next releases. + """ + raise NotImplementedError() + + @cached_property + def previous(self): + # select all non-patch releases + position = self.siblings.index(self.version) + try: + previous = self.siblings[position + 1] + except IndexError: + # first release doesn't have a previous one + return None + else: + return Release.from_jira(previous, jira=self.jira, repo=self.repo) + + @cached_property + def next(self): + # select all non-patch releases + position = self.siblings.index(self.version) + if position <= 0: + raise ValueError("There is no upcoming release set in JIRA after " + "version {}".format(self.version)) + upcoming = self.siblings[position - 1] + return Release.from_jira(upcoming, jira=self.jira, repo=self.repo) + + @cached_property + def issues(self): + issues = self.jira.project_issues(self.version, project='ARROW') + return {i.key: i for i in issues} + + @cached_property + def commits(self): + """ + All commits applied between two versions. + """ + if self.previous is None: + # first release + lower = '' + else: + lower = self.repo.tags[self.previous.tag] + + if self.version.released: + upper = self.repo.tags[self.tag] + else: + try: + upper = self.repo.branches[self.branch] + except IndexError: + warnings.warn("Release branch `{}` doesn't exist." + .format(self.branch)) + return [] + + commit_range = "{}..{}".format(lower, upper) + return list(map(Commit, self.repo.iter_commits(commit_range))) + + def curate(self): + # handle commits with parquet issue key specially and query them from + # jira and add it to the issues + release_issues = self.issues + + within, outside, nojira, parquet = [], [], [], [] + for c in self.commits: + if c.issue is None: + nojira.append(c) + elif c.issue in release_issues: + within.append((release_issues[c.issue], c)) + elif c.project == 'PARQUET': + parquet.append((self.jira.issue(c.issue), c)) + else: + outside.append((self.jira.issue(c.issue), c)) + + # remaining jira tickets + within_keys = {i.key for i, c in within} + nopatch = [issue for key, issue in release_issues.items() + if key not in within_keys] + + return ReleaseCuration(release=self, within=within, outside=outside, + nojira=nojira, parquet=parquet, nopatch=nopatch) + + def changelog(self): + release_issues = [] + + # get organized report for the release + curation = self.curate() + + # jira tickets having patches in the release + for issue, _ in curation.within: + release_issues.append(issue) + + # jira tickets without patches + for issue in curation.nopatch: + release_issues.append(issue) + + # parquet patches in the release + for issue, _ in curation.parquet: + release_issues.append(issue) + + # organize issues into categories + issue_types = { + 'Bug': 'Bug Fixes', + 'Improvement': 'New Features and Improvements', + 'New Feature': 'New Features and Improvements', + 'Sub-task': 'New Features and Improvements', + 'Task': 'New Features and Improvements', + 'Test': 'Bug Fixes', + 'Wish': 'New Features and Improvements', + } + categories = defaultdict(list) + for issue in release_issues: + categories[issue_types[issue.type]].append(issue) + + # sort issues by the issue key in ascending order + for name, issues in categories.items(): + issues.sort(key=lambda issue: (issue.project, issue.number)) + + return JiraChangelog(release=self, categories=categories) + + +class MaintenanceMixin: + """ + Utility methods for cherry-picking commits from the main branch. + """ + + def commits_to_pick(self, exclude_already_applied=True): + # collect commits applied on the main branch since the root of the + # maintenance branch (the previous major release) + if self.version.major == 0: + # treat minor releases as major releases preceeding 1.0.0 release + commit_range = "apache-arrow-0.{}.0..master".format( + self.version.minor + ) + else: + commit_range = "apache-arrow-{}.0.0..master".format( + self.version.major + ) + + # keeping the original order of the commits helps to minimize the merge + # conflicts during cherry-picks + commits = map(Commit, self.repo.iter_commits(commit_range)) + + # exclude patches that have been already applied to the maintenance + # branch, we cannot identify patches based on sha because it changes + # after the cherry pick so use commit title instead + if exclude_already_applied: + already_applied = {c.title for c in self.commits} + else: + already_applied = set() + + # iterate over the commits applied on the main branch and filter out + # the ones that are included in the jira release + patches_to_pick = [c for c in commits if + c.issue in self.issues and + c.title not in already_applied] + + return reversed(patches_to_pick) + + def cherry_pick_commits(self, recreate_branch=True): + if recreate_branch: + # delete, create and checkout the maintenance branch based off of + # the previous tag + if self.branch in self.repo.branches: + self.repo.git.branch('-D', self.branch) + self.repo.git.checkout(self.previous.tag, b=self.branch) + else: + # just checkout the already existing maintenance branch + self.repo.git.checkout(self.branch) + + # cherry pick the commits based on the jira tickets + for commit in self.commits_to_pick(): + self.repo.git.cherry_pick(commit.hexsha) + + +class MajorRelease(Release): + + @property + def branch(self): + return "master" + + @cached_property + def siblings(self): + """ + Filter only the major releases. + """ + # handle minor releases before 1.0 as major releases + return [v for v in self.jira.project_versions('ARROW') + if v.patch == 0 and (v.major == 0 or v.minor == 0)] + + +class MinorRelease(Release, MaintenanceMixin): + + @property + def branch(self): + return "maint-{}.x.x".format(self.version.major) + + @cached_property + def siblings(self): + """ + Filter the major and minor releases. + """ + return [v for v in self.jira.project_versions('ARROW') if v.patch == 0] + + +class PatchRelease(Release, MaintenanceMixin): + + @property + def branch(self): + return "maint-{}.{}.x".format(self.version.major, self.version.minor) + + @cached_property + def siblings(self): + """ + No filtering, consider all releases. + """ + return self.jira.project_versions('ARROW') diff --git a/src/arrow/dev/archery/archery/templates/release_changelog.md.j2 b/src/arrow/dev/archery/archery/templates/release_changelog.md.j2 new file mode 100644 index 000000000..c0406ddf4 --- /dev/null +++ b/src/arrow/dev/archery/archery/templates/release_changelog.md.j2 @@ -0,0 +1,29 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#} +# Apache Arrow {{ release.version }} ({{ release.version.release_date or today() }}) + +{% for category, issues in categories.items() -%} + +## {{ category }} + +{% for issue in issues -%} +* [{{ issue.key }}](https://issues.apache.org/jira/browse/{{ issue.key }}) - {{ issue.summary | md }} +{% endfor %} + +{% endfor %} diff --git a/src/arrow/dev/archery/archery/templates/release_curation.txt.j2 b/src/arrow/dev/archery/archery/templates/release_curation.txt.j2 new file mode 100644 index 000000000..a5d11e9d4 --- /dev/null +++ b/src/arrow/dev/archery/archery/templates/release_curation.txt.j2 @@ -0,0 +1,41 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#} +Total number of JIRA tickets assigned to version {{ release.version }}: {{ release.issues|length }} + +Total number of applied patches since version {{ release.previous.version }}: {{ release.commits|length }} + +Patches with assigned issue in version {{ release.version }}: +{% for issue, commit in within -%} + - {{ commit.url }} {{ commit.title }} +{% endfor %} + +Patches with assigned issue outside of version {{ release.version }}: +{% for issue, commit in outside -%} + - {{ commit.url }} {{ commit.title }} +{% endfor %} + +Patches in version {{ release.version }} without a linked issue: +{% for commit in nojira -%} + - {{ commit.url }} {{ commit.title }} +{% endfor %} + +JIRA issues in version {{ release.version }} without a linked patch: +{% for issue in nopatch -%} + - https://issues.apache.org/jira/browse/{{ issue.key }} +{% endfor %} diff --git a/src/arrow/dev/archery/archery/testing.py b/src/arrow/dev/archery/archery/testing.py new file mode 100644 index 000000000..471a54d4c --- /dev/null +++ b/src/arrow/dev/archery/archery/testing.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from contextlib import contextmanager +import os +from unittest import mock +import re + + +class DotDict(dict): + + def __getattr__(self, key): + try: + item = self[key] + except KeyError: + raise AttributeError(key) + if isinstance(item, dict): + return DotDict(item) + else: + return item + + +class PartialEnv(dict): + + def __eq__(self, other): + return self.items() <= other.items() + + +_mock_call_type = type(mock.call()) + + +def _ensure_mock_call_object(obj, **kwargs): + if isinstance(obj, _mock_call_type): + return obj + elif isinstance(obj, str): + cmd = re.split(r"\s+", obj) + return mock.call(cmd, **kwargs) + elif isinstance(obj, list): + return mock.call(obj, **kwargs) + else: + raise TypeError(obj) + + +class SuccessfulSubprocessResult: + + def check_returncode(self): + return + + +@contextmanager +def assert_subprocess_calls(expected_commands_or_calls, **kwargs): + calls = [ + _ensure_mock_call_object(obj, **kwargs) + for obj in expected_commands_or_calls + ] + with mock.patch('subprocess.run', autospec=True) as run: + run.return_value = SuccessfulSubprocessResult() + yield run + run.assert_has_calls(calls) + + +@contextmanager +def override_env(mapping): + original = os.environ + try: + os.environ = dict(os.environ, **mapping) + yield os.environ + finally: + os.environ = original diff --git a/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl new file mode 100644 index 000000000..5854eb75c --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff-empty-lines.jsonl @@ -0,0 +1,6 @@ +{"benchmark": "RegressionSumKernel/32768/10", "change": 0.0046756468886368545, "regression": false, "baseline": 13265442258.099466, "contender": 13327466781.91994, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"} +{"benchmark": "RegressionSumKernel/32768/1", "change": 0.0025108399115900733, "regression": false, "baseline": 15181891659.539782, "contender": 15220010959.05199, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"} + +{"benchmark": "RegressionSumKernel/32768/50", "change": 0.00346735806287155, "regression": false, "baseline": 11471825667.817123, "contender": 11511602595.042286, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"} + +{"benchmark": "RegressionSumKernel/32768/0", "change": 0.010140954727954987, "regression": false, "baseline": 18316987019.994465, "contender": 18502738756.116768, "unit": "bytes_per_second", "less_is_better": false, "suite": "arrow-compute-aggregate-benchmark"} diff --git a/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl new file mode 100644 index 000000000..1e25810d7 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/archery-benchmark-diff.jsonl @@ -0,0 +1,4 @@ +{"benchmark":"RegressionSumKernel/32768/50","change":-0.001550846227215492,"regression":false,"baseline":19241207435.428757,"contender":19211367281.47045,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"} +{"benchmark":"RegressionSumKernel/32768/1","change":0.0020681767923465765,"regression":true,"baseline":24823170673.777943,"contender":24771831968.277977,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"} +{"benchmark":"RegressionSumKernel/32768/10","change":0.0033323376378746905,"regression":false,"baseline":21902707565.968014,"contender":21975694782.76145,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"} +{"benchmark":"RegressionSumKernel/32768/0","change":-0.004918126090954414,"regression":true,"baseline":27685006611.446762,"contender":27821164964.790764,"unit":"bytes_per_second","less_is_better":false,"suite":"arrow-compute-aggregate-benchmark"} diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json new file mode 100644 index 000000000..d591105f0 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-build-command.json @@ -0,0 +1,212 @@ +{ + "action": "created", + "comment": { + "author_association": "MEMBER", + "body": "@ursabot build", + "created_at": "2019-04-05T11:55:43Z", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726", + "id": 480248726, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==", + "updated_at": "2019-04-05T11:55:43Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "MEMBER", + "body": "", + "closed_at": null, + "comments": 3, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "created_at": "2019-04-05T11:22:15Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "pull_request": { + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Unittests for GithubHook", + "updated_at": "2019-04-05T11:55:43Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json new file mode 100644 index 000000000..5a8f3461c --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-non-authorized-user.json @@ -0,0 +1,212 @@ +{ + "action": "created", + "comment": { + "author_association": "NONE", + "body": "Unknown command \"\"", + "created_at": "2019-04-05T11:35:47Z", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815", + "id": 480243815, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==", + "updated_at": "2019-04-05T11:35:47Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815", + "user": { + "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4", + "events_url": "https://api.github.com/users/ursabot/events{/privacy}", + "followers_url": "https://api.github.com/users/ursabot/followers", + "following_url": "https://api.github.com/users/ursabot/following{/other_user}", + "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursabot", + "id": 49275095, + "login": "someone", + "node_id": "MDQ6VXNlcjQ5Mjc1MDk1", + "organizations_url": "https://api.github.com/users/ursabot/orgs", + "received_events_url": "https://api.github.com/users/ursabot/received_events", + "repos_url": "https://api.github.com/users/ursabot/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions", + "type": "User", + "url": "https://api.github.com/users/ursabot" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "NONE", + "body": "", + "closed_at": null, + "comments": 2, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "created_at": "2019-04-05T11:22:15Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "pull_request": { + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Unittests for GithubHook", + "updated_at": "2019-04-05T11:35:47Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "someone", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4", + "events_url": "https://api.github.com/users/ursabot/events{/privacy}", + "followers_url": "https://api.github.com/users/ursabot/followers", + "following_url": "https://api.github.com/users/ursabot/following{/other_user}", + "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursabot", + "id": 49275095, + "login": "someone", + "node_id": "MDQ6VXNlcjQ5Mjc1MDk1", + "organizations_url": "https://api.github.com/users/ursabot/orgs", + "received_events_url": "https://api.github.com/users/ursabot/received_events", + "repos_url": "https://api.github.com/users/ursabot/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions", + "type": "User", + "url": "https://api.github.com/users/ursabot" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json new file mode 100644 index 000000000..bfb7210df --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-by-ursabot.json @@ -0,0 +1,212 @@ +{ + "action": "created", + "comment": { + "author_association": "NONE", + "body": "Unknown command \"\"", + "created_at": "2019-04-05T11:35:47Z", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815", + "id": 480243815, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==", + "updated_at": "2019-04-05T11:35:47Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815", + "user": { + "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4", + "events_url": "https://api.github.com/users/ursabot/events{/privacy}", + "followers_url": "https://api.github.com/users/ursabot/followers", + "following_url": "https://api.github.com/users/ursabot/following{/other_user}", + "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursabot", + "id": 49275095, + "login": "ursabot", + "node_id": "MDQ6VXNlcjQ5Mjc1MDk1", + "organizations_url": "https://api.github.com/users/ursabot/orgs", + "received_events_url": "https://api.github.com/users/ursabot/received_events", + "repos_url": "https://api.github.com/users/ursabot/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions", + "type": "User", + "url": "https://api.github.com/users/ursabot" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "MEMBER", + "body": "", + "closed_at": null, + "comments": 2, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "created_at": "2019-04-05T11:22:15Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "pull_request": { + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Unittests for GithubHook", + "updated_at": "2019-04-05T11:35:47Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars2.githubusercontent.com/u/49275095?v=4", + "events_url": "https://api.github.com/users/ursabot/events{/privacy}", + "followers_url": "https://api.github.com/users/ursabot/followers", + "following_url": "https://api.github.com/users/ursabot/following{/other_user}", + "gists_url": "https://api.github.com/users/ursabot/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursabot", + "id": 49275095, + "login": "ursabot", + "node_id": "MDQ6VXNlcjQ5Mjc1MDk1", + "organizations_url": "https://api.github.com/users/ursabot/orgs", + "received_events_url": "https://api.github.com/users/ursabot/received_events", + "repos_url": "https://api.github.com/users/ursabot/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursabot/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursabot/subscriptions", + "type": "User", + "url": "https://api.github.com/users/ursabot" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json new file mode 100644 index 000000000..a3d450078 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-not-mentioning-ursabot.json @@ -0,0 +1,212 @@ +{ + "action": "created", + "comment": { + "author_association": "MEMBER", + "body": "bear is no game", + "created_at": "2019-04-05T11:26:56Z", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727", + "id": 480241727, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==", + "updated_at": "2019-04-05T11:26:56Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "MEMBER", + "body": "", + "closed_at": null, + "comments": 0, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "created_at": "2019-04-05T11:22:15Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "pull_request": { + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Unittests for GithubHook", + "updated_at": "2019-04-05T11:26:56Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json new file mode 100644 index 000000000..c88197c8e --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-with-empty-command.json @@ -0,0 +1,217 @@ +{ + "action": "created", + "comment": { + "author_association": "MEMBER", + "body": "@ursabot ", + "body_html": "", + "body_text": "", + "created_at": "2019-04-05T11:35:46Z", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811", + "id": 480243811, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==", + "updated_at": "2019-04-05T11:35:46Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "MEMBER", + "body": "", + "body_html": "", + "body_text": "", + "closed_at": null, + "closed_by": null, + "comments": 1, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "created_at": "2019-04-05T11:22:15Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "pull_request": { + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Unittests for GithubHook", + "updated_at": "2019-04-05T11:35:46Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json new file mode 100644 index 000000000..9e362fc0e --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-issue-comment-without-pull-request.json @@ -0,0 +1,206 @@ +{ + "action": "created", + "comment": { + "author_association": "MEMBER", + "body": "@ursabot build", + "created_at": "2019-04-05T13:07:57Z", + "html_url": "https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480268708", + "id": 480268708, + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19", + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI2ODcwOA==", + "updated_at": "2019-04-05T13:07:57Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480268708", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "issue": { + "assignee": null, + "assignees": [], + "author_association": "MEMBER", + "body": "", + "closed_at": null, + "comments": 5, + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments", + "created_at": "2019-04-02T09:56:41Z", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/events", + "html_url": "https://github.com/ursa-labs/ursabot/issues/19", + "id": 428131685, + "labels": [], + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}", + "locked": false, + "milestone": null, + "node_id": "MDU6SXNzdWU0MjgxMzE2ODU=", + "number": 19, + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "state": "open", + "title": "Build ursabot itself via ursabot", + "updated_at": "2019-04-05T13:07:57Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19", + "user": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } + }, + "organization": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "description": "Innovation lab for open source data science tools, powered by Apache Arrow", + "events_url": "https://api.github.com/orgs/ursa-labs/events", + "hooks_url": "https://api.github.com/orgs/ursa-labs/hooks", + "id": 46514972, + "issues_url": "https://api.github.com/orgs/ursa-labs/issues", + "login": "ursa-labs", + "members_url": "https://api.github.com/orgs/ursa-labs/members{/member}", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "public_members_url": "https://api.github.com/orgs/ursa-labs/public_members{/member}", + "repos_url": "https://api.github.com/orgs/ursa-labs/repos", + "url": "https://api.github.com/orgs/ursa-labs" + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T12:01:40Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 898, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json b/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json new file mode 100644 index 000000000..9cf5c0dda --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/event-pull-request-opened.json @@ -0,0 +1,445 @@ +{ + "action": "opened", + "number": 26, + "pull_request": { + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26", + "id": 267785552, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "number": 26, + "state": "open", + "locked": false, + "title": "Unittests for GithubHook", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "body": "", + "created_at": "2019-04-05T11:22:15Z", + "updated_at": "2019-04-05T12:01:40Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "cc5dc3606988b3824be54df779ed2028776113cb", + "assignee": null, + "assignees": [], + "requested_reviewers": [], + "requested_teams": [], + "labels": [], + "milestone": null, + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits", + "review_comments_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments", + "review_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d", + "head": { + "label": "ursa-labs:test-hook", + "ref": "test-hook", + "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d", + "user": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 169101701, + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "name": "ursabot", + "full_name": "ursa-labs/ursabot", + "private": false, + "owner": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/ursa-labs/ursabot", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "created_at": "2019-02-04T15:40:31Z", + "updated_at": "2019-04-04T17:49:10Z", + "pushed_at": "2019-04-05T12:01:40Z", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "svn_url": "https://github.com/ursa-labs/ursabot", + "homepage": null, + "size": 898, + "stargazers_count": 1, + "watchers_count": 1, + "language": "Jupyter Notebook", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 19, + "license": null, + "forks": 0, + "open_issues": 19, + "watchers": 1, + "default_branch": "master" + } + }, + "base": { + "label": "ursa-labs:master", + "ref": "master", + "sha": "a162ad254b589b924db47e057791191b39613fd5", + "user": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 169101701, + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "name": "ursabot", + "full_name": "ursa-labs/ursabot", + "private": false, + "owner": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/ursa-labs/ursabot", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "created_at": "2019-02-04T15:40:31Z", + "updated_at": "2019-04-04T17:49:10Z", + "pushed_at": "2019-04-05T12:01:40Z", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "svn_url": "https://github.com/ursa-labs/ursabot", + "homepage": null, + "size": 898, + "stargazers_count": 1, + "watchers_count": 1, + "language": "Jupyter Notebook", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 19, + "license": null, + "forks": 0, + "open_issues": 19, + "watchers": 1, + "default_branch": "master" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "html": { + "href": "https://github.com/ursa-labs/ursabot/pull/26" + }, + "issue": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26" + }, + "comments": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d" + } + }, + "author_association": "MEMBER", + "merged": false, + "mergeable": true, + "rebaseable": true, + "mergeable_state": "unstable", + "merged_by": null, + "comments": 5, + "review_comments": 0, + "maintainer_can_modify": false, + "commits": 2, + "additions": 1124, + "deletions": 0, + "changed_files": 7 + }, + "repository": { + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "archived": false, + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "created_at": "2019-02-04T15:40:31Z", + "default_branch": "master", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "description": null, + "disabled": false, + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "full_name": "ursa-labs/ursabot", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_projects": true, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "html_url": "https://github.com/ursa-labs/ursabot", + "id": 169101701, + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "language": "Jupyter Notebook", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "license": null, + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "mirror_url": null, + "name": "ursabot", + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "open_issues": 19, + "open_issues_count": 19, + "owner": { + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/ursa-labs", + "id": 46514972, + "login": "ursa-labs", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "type": "Organization", + "url": "https://api.github.com/users/ursa-labs" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "pushed_at": "2019-04-05T11:22:16Z", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "size": 892, + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "stargazers_count": 1, + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "svn_url": "https://github.com/ursa-labs/ursabot", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "updated_at": "2019-04-04T17:49:10Z", + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "watchers": 1, + "watchers_count": 1 + }, + "sender": { + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/kszucs", + "id": 961747, + "login": "kszucs", + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "repos_url": "https://api.github.com/users/kszucs/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "type": "User", + "url": "https://api.github.com/users/kszucs" + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json new file mode 100644 index 000000000..1e4939776 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-19.json @@ -0,0 +1,64 @@ +{ + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19", + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/19/events", + "html_url": "https://github.com/ursa-labs/ursabot/issues/19", + "id": 428131685, + "node_id": "MDU6SXNzdWU0MjgxMzE2ODU=", + "number": 19, + "title": "Build ursabot itself via ursabot", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "labels": [], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [], + "milestone": null, + "comments": 8, + "created_at": "2019-04-02T09:56:41Z", + "updated_at": "2019-04-05T13:30:49Z", + "closed_at": "2019-04-05T13:30:49Z", + "author_association": "MEMBER", + "body": "", + "closed_by": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json new file mode 100644 index 000000000..44c4d3bed --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-26.json @@ -0,0 +1,70 @@ +{ + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "repository_url": "https://api.github.com/repos/ursa-labs/ursabot", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/events", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "id": 429706959, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "number": 26, + "title": "Unittests for GithubHook + native asyncio syntax", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "labels": [], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [], + "milestone": null, + "comments": 9, + "created_at": "2019-04-05T11:22:15Z", + "updated_at": "2019-08-28T00:34:19Z", + "closed_at": "2019-04-05T13:54:34Z", + "author_association": "MEMBER", + "pull_request": { + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch" + }, + "body": "Resolves:\r\n- #26 Unittests for GithubHook + native asyncio syntax\r\n- #27 Use native async/await keywords instead of @inlineCallbacks and yield\r\n", + "closed_by": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + } +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json new file mode 100644 index 000000000..93ee4b13c --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480243811.json @@ -0,0 +1,31 @@ +{ + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/479081273", + "html_url": "https://github.com/ursa-labs/ursabot/pull/21#issuecomment-479081273", + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/21", + "id": 480243811, + "node_id": "MDEyOklzc3VlQ29tbWVudDQ3OTA4MTI3Mw==", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2019-04-02T16:29:46Z", + "updated_at": "2019-04-02T16:29:46Z", + "author_association": "MEMBER", + "body": "@ursabot" +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json new file mode 100644 index 000000000..f3cd34083 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/issue-comment-480248726.json @@ -0,0 +1,31 @@ +{ + "url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726", + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "id": 480248726, + "node_id": "MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2019-04-05T11:55:43Z", + "updated_at": "2019-04-05T11:55:43Z", + "author_association": "MEMBER", + "body": "@ursabot build" +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json new file mode 100644 index 000000000..ffc48943a --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-commit.json @@ -0,0 +1,158 @@ +{ + "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d", + "node_id": "MDY6Q29tbWl0MTY5MTAxNzAxOjI3MDVkYTJiNjE2Yjk4ZmE2MDEwYTI1ODEzYzVhN2EyNzQ1NmY3MWQ=", + "commit": { + "author": { + "name": "Krisztián Szűcs", + "email": "szucs.krisztian@gmail.com", + "date": "2019-04-05T12:01:31Z" + }, + "committer": { + "name": "Krisztián Szűcs", + "email": "szucs.krisztian@gmail.com", + "date": "2019-04-05T12:01:31Z" + }, + "message": "add recorded event requests", + "tree": { + "sha": "16a7bb186833a67e9c2d84a58393503b85500ceb", + "url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees/16a7bb186833a67e9c2d84a58393503b85500ceb" + }, + "url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits/2705da2b616b98fa6010a25813c5a7a27456f71d", + "comment_count": 0, + "verification": { + "verified": true, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\niQFOBAABCAA4FiEEOOW2r8dr6sA77zHlgjqBKYe1QKUFAlynQ58aHHN6dWNzLmty\naXN6dGlhbkBnbWFpbC5jb20ACgkQgjqBKYe1QKUYKwf6AiXDMaLqNLNSjRY7lIXX\nudioewz0hSb4bgIXBv30nswu9CoOA0+mHCokEVtZhYbXzXDsZ1KJrilSC4j+Ws4q\nkRGA6iEmrne2HcSKNZXzcVnwV9zpwKxlVh2QCTNb1PuOYFBLH0kwE704uWIWMGDN\nbo8cjQPwegePCRguCvPh/5wa5J3uiq5gmJLG6bC/d1XYE+FJVtlnyzqzLMIryGKe\ntIciw+wwkF413Q/YVbZ49vLUeCX9H8PHC4mZYGDWuvjFW1WTfkjK5bAH+oaTVM6h\n350I5ZFloHmMA/QeRge5qFxXoEBMDGiXHHktzYZDXnliFOQNxzqwirA5lQQ6LRSS\naQ==\n=7rqi\n-----END PGP SIGNATURE-----", + "payload": "tree 16a7bb186833a67e9c2d84a58393503b85500ceb\nparent 446ae69b9385e8d0f40aa9595f723d34383af2f7\nauthor Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\ncommitter Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\n\nadd recorded event requests\n" + } + }, + "url": "https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d", + "html_url": "https://github.com/ursa-labs/ursabot/commit/2705da2b616b98fa6010a25813c5a7a27456f71d", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d/comments", + "author": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "446ae69b9385e8d0f40aa9595f723d34383af2f7", + "url": "https://api.github.com/repos/ursa-labs/ursabot/commits/446ae69b9385e8d0f40aa9595f723d34383af2f7", + "html_url": "https://github.com/ursa-labs/ursabot/commit/446ae69b9385e8d0f40aa9595f723d34383af2f7" + } + ], + "stats": { + "total": 1062, + "additions": 1058, + "deletions": 4 + }, + "files": [ + { + "sha": "dfae6eeaef384ae6180c6302a58b49e39982dc33", + "filename": "ursabot/tests/fixtures/issue-comment-build-command.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"I've successfully started builds for this PR\",\n+ \"created_at\": \"2019-04-05T11:55:44Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248730\",\n+ \"id\": 480248730,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODczMA==\",\n+ \"updated_at\": \"2019-04-05T11:55:44Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248730\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 4,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:55:44Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}" + }, + { + "sha": "7ef554e333327f0e62aa1fd76b4b17844a39adeb", + "filename": "ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Unknown command \\\"\\\"\",\n+ \"created_at\": \"2019-04-05T11:35:47Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\",\n+ \"id\": 480243815,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 2,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}" + }, + { + "sha": "a8082dbc91fdfe815b795e49ec10e49000771ef5", + "filename": "ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"bear is no game\",\n+ \"created_at\": \"2019-04-05T11:26:56Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\",\n+ \"id\": 480241727,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 0,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "2770e29ba9086394455315e590c0b433d08e437e", + "filename": "ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot \",\n+ \"created_at\": \"2019-04-05T11:35:46Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\",\n+ \"id\": 480243811,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 1,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "80ff46510a2f39ae60f7c3a98e5fdaef8e688784", + "filename": "ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "status": "added", + "additions": 206, + "deletions": 0, + "changes": 206, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -0,0 +1,206 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Ursabot only listens to pull request comments!\",\n+ \"created_at\": \"2019-04-05T11:53:43Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480248217\",\n+ \"id\": 480248217,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODIxNw==\",\n+ \"updated_at\": \"2019-04-05T11:53:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248217\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 4,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\",\n+ \"created_at\": \"2019-04-02T09:56:41Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19\",\n+ \"id\": 428131685,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDU6SXNzdWU0MjgxMzE2ODU=\",\n+ \"number\": 19,\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Build ursabot itself via ursabot\",\n+ \"updated_at\": \"2019-04-05T11:53:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}" + }, + { + "sha": "c738bb0eb54c87ba0f23e97e827d77c2be74d0b6", + "filename": "ursabot/tests/test_hooks.py", + "status": "modified", + "additions": 4, + "deletions": 4, + "changes": 8, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=2705da2b616b98fa6010a25813c5a7a27456f71d", + "patch": "@@ -54,7 +54,7 @@ class TestGithubHook(ChangeHookTestCase):\n await self.request('ping', {})\n assert len(self.hook.master.data.updates.changesAdded) == 0\n \n- @ensure_deferred\n- async def test_issue_comment(self):\n- payload = {}\n- await self.request('issue_comment', payload)\n+ # @ensure_deferred\n+ # async def test_issue_comment(self):\n+ # payload = {}\n+ # await self.request('issue_comment', payload)" + } + ] +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json new file mode 100644 index 000000000..b039b3d10 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26-files.json @@ -0,0 +1,170 @@ +[ + { + "sha": "ebfe3f6c5e98723f9751c99ce8ce798f1ba529c5", + "filename": ".travis.yml", + "status": "modified", + "additions": 4, + "deletions": 1, + "changes": 5, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/.travis.yml", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/.travis.yml", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/.travis.yml?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -4,7 +4,10 @@ services:\n python:\n - 3.6\n script:\n- - pip install \"pytest>=3.9\" flake8 -e .\n+ # --no-binary buildbot is required because buildbot doesn't bundle its tests\n+ # to binary wheels, but ursabot's test suite depends on buildbot's so install\n+ # it from source\n+ - pip install --no-binary buildbot \"pytest>=3.9\" mock flake8 -e .\n \n # run linter\n - flake8 ursabot" + }, + { + "sha": "86ad809d3f74c175b92ac58c6c645b0fbf5fa2c5", + "filename": "setup.py", + "status": "modified", + "additions": 6, + "deletions": 1, + "changes": 7, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/setup.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/setup.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/setup.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -1,8 +1,13 @@\n #!/usr/bin/env python\n \n+import sys\n from setuptools import setup\n \n \n+if sys.version_info < (3, 6):\n+ sys.exit('Python < 3.6 is not supported due to missing asyncio support')\n+\n+\n # TODO(kszucs): add package data, change maintainer\n setup(\n name='ursabot',\n@@ -15,7 +20,7 @@\n setup_requires=['setuptools_scm'],\n install_requires=['click', 'dask', 'docker', 'docker-map', 'toolz',\n 'buildbot', 'treq'],\n- tests_require=['pytest>=3.9'],\n+ tests_require=['pytest>=3.9', 'mock'],\n entry_points='''\n [console_scripts]\n ursabot=ursabot.cli:ursabot" + }, + { + "sha": "c884f3f85bba499d77d9ad28bcd0ff5edf80f957", + "filename": "ursabot/factories.py", + "status": "modified", + "additions": 6, + "deletions": 2, + "changes": 8, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/factories.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/factories.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/factories.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -79,8 +79,12 @@ def prepend_step(self, step):\n repourl='https://github.com/ursa-labs/ursabot',\n mode='full'),\n ShellCommand(command=['ls', '-lah']),\n- ShellCommand(command=['pip', 'install', 'pytest', 'flake8']),\n- ShellCommand(command=['pip', 'install', '-e', '.']),\n+ ShellCommand(command=['pip', 'install', 'pytest', 'flake8', 'mock']),\n+ # --no-binary buildbot is required because buildbot doesn't bundle its\n+ # tests to binary wheels, but ursabot's test suite depends on buildbot's\n+ # so install it from source\n+ ShellCommand(command=['pip', 'install', '--no-binary', 'buildbot',\n+ '-e', '.']),\n ShellCommand(command=['flake8']),\n ShellCommand(command=['pytest', '-v', '-m', 'not docker', 'ursabot']),\n ShellCommand(command=['buildbot', 'checkconfig', '.'])" + }, + { + "sha": "0265cfbd9c2882f492469882a7bf513a1c1b5af4", + "filename": "ursabot/hooks.py", + "status": "modified", + "additions": 17, + "deletions": 19, + "changes": 36, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/hooks.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/hooks.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/hooks.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -1,11 +1,11 @@\n from urllib.parse import urlparse\n \n from twisted.python import log\n-from twisted.internet import defer\n \n from buildbot.www.hooks.github import GitHubEventHandler\n from buildbot.util.httpclientservice import HTTPClientService\n \n+from .utils import ensure_deferred\n \n BOTNAME = 'ursabot'\n \n@@ -22,20 +22,18 @@ def _client(self):\n self.master, self.github_api_endpoint, headers=headers,\n debug=self.debug, verify=self.verify)\n \n- @defer.inlineCallbacks\n- def _get(self, url):\n+ async def _get(self, url):\n url = urlparse(url)\n- client = yield self._client()\n- response = yield client.get(url.path)\n- result = yield response.json()\n+ client = await self._client()\n+ response = await client.get(url.path)\n+ result = await response.json()\n return result\n \n- @defer.inlineCallbacks\n- def _post(self, url, data):\n+ async def _post(self, url, data):\n url = urlparse(url)\n- client = yield self._client()\n- response = yield client.post(url.path, json=data)\n- result = yield response.json()\n+ client = await self._client()\n+ response = await client.post(url.path, json=data)\n+ result = await response.json()\n log.msg(f'POST to {url} with the following result: {result}')\n return result\n \n@@ -46,8 +44,8 @@ def _parse_command(self, message):\n return message.split(mention)[-1].lower().strip()\n return None\n \n- @defer.inlineCallbacks\n- def handle_issue_comment(self, payload, event):\n+ @ensure_deferred\n+ async def handle_issue_comment(self, payload, event):\n issue = payload['issue']\n comments_url = issue['comments_url']\n command = self._parse_command(payload['comment']['body'])\n@@ -64,16 +62,16 @@ def handle_issue_comment(self, payload, event):\n elif command == 'build':\n if 'pull_request' not in issue:\n message = 'Ursabot only listens to pull request comments!'\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return [], 'git'\n else:\n message = f'Unknown command \"{command}\"'\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return [], 'git'\n \n try:\n- pull_request = yield self._get(issue['pull_request']['url'])\n- changes, _ = yield self.handle_pull_request({\n+ pull_request = await self._get(issue['pull_request']['url'])\n+ changes, _ = await self.handle_pull_request({\n 'action': 'synchronize',\n 'sender': payload['sender'],\n 'repository': payload['repository'],\n@@ -82,11 +80,11 @@ def handle_issue_comment(self, payload, event):\n }, event)\n except Exception as e:\n message = \"I've failed to start builds for this PR\"\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n raise e\n else:\n message = \"I've successfully started builds for this PR\"\n- yield self._post(comments_url, {'body': message})\n+ await self._post(comments_url, {'body': message})\n return changes, 'git'\n \n # TODO(kszucs):" + }, + { + "sha": "1e1ecf2ce47da929dbf1b93632640e7e6ae1cfe0", + "filename": "ursabot/steps.py", + "status": "modified", + "additions": 13, + "deletions": 13, + "changes": 26, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/steps.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/steps.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/steps.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -1,9 +1,9 @@\n-from twisted.internet import defer\n-\n from buildbot.plugins import steps, util\n from buildbot.process import buildstep\n from buildbot.process.results import SUCCESS\n \n+from .utils import ensure_deferred\n+\n \n class ShellMixin(buildstep.ShellMixin):\n \"\"\"Run command in a login bash shell\n@@ -49,10 +49,10 @@ def __init__(self, **kwargs):\n kwargs = self.setupShellMixin(kwargs)\n super().__init__(**kwargs)\n \n- @defer.inlineCallbacks\n- def run(self):\n- cmd = yield self.makeRemoteShellCommand(command=self.command)\n- yield self.runCommand(cmd)\n+ @ensure_deferred\n+ async def run(self):\n+ cmd = await self.makeRemoteShellCommand(command=self.command)\n+ await self.runCommand(cmd)\n return cmd.results()\n \n \n@@ -71,8 +71,8 @@ class CMake(ShellMixin, steps.CMake):\n \n name = 'CMake'\n \n- @defer.inlineCallbacks\n- def run(self):\n+ @ensure_deferred\n+ async def run(self):\n \"\"\"Create and run CMake command\n \n Copied from the original CMake implementation to handle None values as\n@@ -94,8 +94,8 @@ def run(self):\n if self.options is not None:\n command.extend(self.options)\n \n- cmd = yield self.makeRemoteShellCommand(command=command)\n- yield self.runCommand(cmd)\n+ cmd = await self.makeRemoteShellCommand(command=command)\n+ await self.runCommand(cmd)\n \n return cmd.results()\n \n@@ -117,8 +117,8 @@ def __init__(self, variables, source='WorkerEnvironment', **kwargs):\n self.source = source\n super().__init__(**kwargs)\n \n- @defer.inlineCallbacks\n- def run(self):\n+ @ensure_deferred\n+ async def run(self):\n # on Windows, environment variables are case-insensitive, but we have\n # a case-sensitive dictionary in worker_environ. Fortunately, that\n # dictionary is also folded to uppercase, so we can simply fold the\n@@ -139,7 +139,7 @@ def run(self):\n # TODO(kszucs) try with self.setProperty similarly like in\n # SetProperties\n properties.setProperty(prop, value, self.source, runtime=True)\n- yield self.addCompleteLog('set-prop', f'{prop}: {value}')\n+ await self.addCompleteLog('set-prop', f'{prop}: {value}')\n \n return SUCCESS\n " + }, + { + "sha": "6a7d5308be6608f542a810d410f9240157a1340f", + "filename": "ursabot/tests/fixtures/issue-comment-build-command.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-build-command.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-build-command.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot build\",\n+ \"created_at\": \"2019-04-05T11:55:43Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248726\",\n+ \"id\": 480248726,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODcyNg==\",\n+ \"updated_at\": \"2019-04-05T11:55:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248726\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 3,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:55:43Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "7ef554e333327f0e62aa1fd76b4b17844a39adeb", + "filename": "ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-by-ursabot.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"NONE\",\n+ \"body\": \"Unknown command \\\"\\\"\",\n+ \"created_at\": \"2019-04-05T11:35:47Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\",\n+ \"id\": 480243815,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 2,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:47Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/49275095?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursabot/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursabot/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursabot/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursabot/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursabot\",\n+ \"id\": 49275095,\n+ \"login\": \"ursabot\",\n+ \"node_id\": \"MDQ6VXNlcjQ5Mjc1MDk1\",\n+ \"organizations_url\": \"https://api.github.com/users/ursabot/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursabot/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursabot/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursabot/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursabot/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/ursabot\"\n+ }\n+}" + }, + { + "sha": "a8082dbc91fdfe815b795e49ec10e49000771ef5", + "filename": "ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"bear is no game\",\n+ \"created_at\": \"2019-04-05T11:26:56Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\",\n+ \"id\": 480241727,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 0,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:26:56Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "2770e29ba9086394455315e590c0b433d08e437e", + "filename": "ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "status": "added", + "additions": 212, + "deletions": 0, + "changes": 212, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-with-empty-command.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,212 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot \",\n+ \"created_at\": \"2019-04-05T11:35:46Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\",\n+ \"id\": 480243811,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 1,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"id\": 429706959,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"number\": 26,\n+ \"pull_request\": {\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Unittests for GithubHook\",\n+ \"updated_at\": \"2019-04-05T11:35:46Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T11:22:16Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 892,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "b7de8d838332944101812ee2a46c08dd0144efe3", + "filename": "ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "status": "added", + "additions": 206, + "deletions": 0, + "changes": 206, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/issue-comment-without-pull-request.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,206 @@\n+{\n+ \"action\": \"created\",\n+ \"comment\": {\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"@ursabot build\",\n+ \"created_at\": \"2019-04-05T13:07:57Z\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480268708\",\n+ \"id\": 480268708,\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"node_id\": \"MDEyOklzc3VlQ29tbWVudDQ4MDI2ODcwOA==\",\n+ \"updated_at\": \"2019-04-05T13:07:57Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480268708\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"issue\": {\n+ \"assignee\": null,\n+ \"assignees\": [],\n+ \"author_association\": \"MEMBER\",\n+ \"body\": \"\",\n+ \"closed_at\": null,\n+ \"comments\": 5,\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\",\n+ \"created_at\": \"2019-04-02T09:56:41Z\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/issues/19\",\n+ \"id\": 428131685,\n+ \"labels\": [],\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\",\n+ \"locked\": false,\n+ \"milestone\": null,\n+ \"node_id\": \"MDU6SXNzdWU0MjgxMzE2ODU=\",\n+ \"number\": 19,\n+ \"repository_url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"state\": \"open\",\n+ \"title\": \"Build ursabot itself via ursabot\",\n+ \"updated_at\": \"2019-04-05T13:07:57Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/19\",\n+ \"user\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+ },\n+ \"organization\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"description\": \"Innovation lab for open source data science tools, powered by Apache Arrow\",\n+ \"events_url\": \"https://api.github.com/orgs/ursa-labs/events\",\n+ \"hooks_url\": \"https://api.github.com/orgs/ursa-labs/hooks\",\n+ \"id\": 46514972,\n+ \"issues_url\": \"https://api.github.com/orgs/ursa-labs/issues\",\n+ \"login\": \"ursa-labs\",\n+ \"members_url\": \"https://api.github.com/orgs/ursa-labs/members{/member}\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"public_members_url\": \"https://api.github.com/orgs/ursa-labs/public_members{/member}\",\n+ \"repos_url\": \"https://api.github.com/orgs/ursa-labs/repos\",\n+ \"url\": \"https://api.github.com/orgs/ursa-labs\"\n+ },\n+ \"repository\": {\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"archived\": false,\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"default_branch\": \"master\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"description\": null,\n+ \"disabled\": false,\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"fork\": false,\n+ \"forks\": 0,\n+ \"forks_count\": 0,\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"has_downloads\": true,\n+ \"has_issues\": true,\n+ \"has_pages\": false,\n+ \"has_projects\": true,\n+ \"has_wiki\": true,\n+ \"homepage\": null,\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"id\": 169101701,\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"language\": \"Jupyter Notebook\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"license\": null,\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"mirror_url\": null,\n+ \"name\": \"ursabot\",\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"open_issues\": 19,\n+ \"open_issues_count\": 19,\n+ \"owner\": {\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"id\": 46514972,\n+ \"login\": \"ursa-labs\",\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"type\": \"Organization\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\"\n+ },\n+ \"private\": false,\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"size\": 898,\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"stargazers_count\": 1,\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"watchers\": 1,\n+ \"watchers_count\": 1\n+ },\n+ \"sender\": {\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"gravatar_id\": \"\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"id\": 961747,\n+ \"login\": \"kszucs\",\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"site_admin\": false,\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"type\": \"User\",\n+ \"url\": \"https://api.github.com/users/kszucs\"\n+ }\n+}" + }, + { + "sha": "33e051455e866fb4774a16ae02ad40dcf9e6a7fd", + "filename": "ursabot/tests/fixtures/pull-request-26-commit.json", + "status": "added", + "additions": 158, + "deletions": 0, + "changes": 158, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26-commit.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26-commit.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/pull-request-26-commit.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,158 @@\n+{\n+ \"sha\": \"2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"node_id\": \"MDY6Q29tbWl0MTY5MTAxNzAxOjI3MDVkYTJiNjE2Yjk4ZmE2MDEwYTI1ODEzYzVhN2EyNzQ1NmY3MWQ=\",\n+ \"commit\": {\n+ \"author\": {\n+ \"name\": \"Krisztián Szűcs\",\n+ \"email\": \"szucs.krisztian@gmail.com\",\n+ \"date\": \"2019-04-05T12:01:31Z\"\n+ },\n+ \"committer\": {\n+ \"name\": \"Krisztián Szűcs\",\n+ \"email\": \"szucs.krisztian@gmail.com\",\n+ \"date\": \"2019-04-05T12:01:31Z\"\n+ },\n+ \"message\": \"add recorded event requests\",\n+ \"tree\": {\n+ \"sha\": \"16a7bb186833a67e9c2d84a58393503b85500ceb\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees/16a7bb186833a67e9c2d84a58393503b85500ceb\"\n+ },\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"comment_count\": 0,\n+ \"verification\": {\n+ \"verified\": true,\n+ \"reason\": \"valid\",\n+ \"signature\": \"-----BEGIN PGP SIGNATURE-----\\n\\niQFOBAABCAA4FiEEOOW2r8dr6sA77zHlgjqBKYe1QKUFAlynQ58aHHN6dWNzLmty\\naXN6dGlhbkBnbWFpbC5jb20ACgkQgjqBKYe1QKUYKwf6AiXDMaLqNLNSjRY7lIXX\\nudioewz0hSb4bgIXBv30nswu9CoOA0+mHCokEVtZhYbXzXDsZ1KJrilSC4j+Ws4q\\nkRGA6iEmrne2HcSKNZXzcVnwV9zpwKxlVh2QCTNb1PuOYFBLH0kwE704uWIWMGDN\\nbo8cjQPwegePCRguCvPh/5wa5J3uiq5gmJLG6bC/d1XYE+FJVtlnyzqzLMIryGKe\\ntIciw+wwkF413Q/YVbZ49vLUeCX9H8PHC4mZYGDWuvjFW1WTfkjK5bAH+oaTVM6h\\n350I5ZFloHmMA/QeRge5qFxXoEBMDGiXHHktzYZDXnliFOQNxzqwirA5lQQ6LRSS\\naQ==\\n=7rqi\\n-----END PGP SIGNATURE-----\",\n+ \"payload\": \"tree 16a7bb186833a67e9c2d84a58393503b85500ceb\\nparent 446ae69b9385e8d0f40aa9595f723d34383af2f7\\nauthor Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\\ncommitter Krisztián Szűcs <szucs.krisztian@gmail.com> 1554465691 +0200\\n\\nadd recorded event requests\\n\"\n+ }\n+ },\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/commit/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/2705da2b616b98fa6010a25813c5a7a27456f71d/comments\",\n+ \"author\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"committer\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"parents\": [\n+ {\n+ \"sha\": \"446ae69b9385e8d0f40aa9595f723d34383af2f7\",\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits/446ae69b9385e8d0f40aa9595f723d34383af2f7\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/commit/446ae69b9385e8d0f40aa9595f723d34383af2f7\"\n+ }\n+ ],\n+ \"stats\": {\n+ \"total\": 1062,\n+ \"additions\": 1058,\n+ \"deletions\": 4\n+ },\n+ \"files\": [\n+ {\n+ \"sha\": \"dfae6eeaef384ae6180c6302a58b49e39982dc33\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-build-command.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-build-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"I've successfully started builds for this PR\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480248730\\\",\\n+ \\\"id\\\": 480248730,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODczMA==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248730\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 4,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:55:44Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"7ef554e333327f0e62aa1fd76b4b17844a39adeb\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-by-ursabot.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-by-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"Unknown command \\\\\\\"\\\\\\\"\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243815\\\",\\n+ \\\"id\\\": 480243815,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxNQ==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243815\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 2,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:47Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"a8082dbc91fdfe815b795e49ec10e49000771ef5\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-not-mentioning-ursabot.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"bear is no game\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480241727\\\",\\n+ \\\"id\\\": 480241727,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MTcyNw==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480241727\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 0,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:26:56Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"2770e29ba9086394455315e590c0b433d08e437e\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"status\": \"added\",\n+ \"additions\": 212,\n+ \"deletions\": 0,\n+ \"changes\": 212,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-with-empty-command.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-with-empty-command.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,212 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"@ursabot \\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26#issuecomment-480243811\\\",\\n+ \\\"id\\\": 480243811,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0MzgxMQ==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480243811\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 1,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:22:15Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"id\\\": 429706959,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\\\",\\n+ \\\"number\\\": 26,\\n+ \\\"pull_request\\\": {\\n+ \\\"diff_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.diff\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26\\\",\\n+ \\\"patch_url\\\": \\\"https://github.com/ursa-labs/ursabot/pull/26.patch\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\\\"\\n+ },\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Unittests for GithubHook\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:35:46Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/26\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"80ff46510a2f39ae60f7c3a98e5fdaef8e688784\",\n+ \"filename\": \"ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"status\": \"added\",\n+ \"additions\": 206,\n+ \"deletions\": 0,\n+ \"changes\": 206,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/fixtures/issue-comment-without-pull-request.json\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/issue-comment-without-pull-request.json?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -0,0 +1,206 @@\\n+{\\n+ \\\"action\\\": \\\"created\\\",\\n+ \\\"comment\\\": {\\n+ \\\"author_association\\\": \\\"NONE\\\",\\n+ \\\"body\\\": \\\"Ursabot only listens to pull request comments!\\\",\\n+ \\\"created_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/issues/19#issuecomment-480248217\\\",\\n+ \\\"id\\\": 480248217,\\n+ \\\"issue_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"node_id\\\": \\\"MDEyOklzc3VlQ29tbWVudDQ4MDI0ODIxNw==\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments/480248217\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+ },\\n+ \\\"issue\\\": {\\n+ \\\"assignee\\\": null,\\n+ \\\"assignees\\\": [],\\n+ \\\"author_association\\\": \\\"MEMBER\\\",\\n+ \\\"body\\\": \\\"\\\",\\n+ \\\"closed_at\\\": null,\\n+ \\\"comments\\\": 4,\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/comments\\\",\\n+ \\\"created_at\\\": \\\"2019-04-02T09:56:41Z\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/events\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"id\\\": 428131685,\\n+ \\\"labels\\\": [],\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19/labels{/name}\\\",\\n+ \\\"locked\\\": false,\\n+ \\\"milestone\\\": null,\\n+ \\\"node_id\\\": \\\"MDU6SXNzdWU0MjgxMzE2ODU=\\\",\\n+ \\\"number\\\": 19,\\n+ \\\"repository_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"state\\\": \\\"open\\\",\\n+ \\\"title\\\": \\\"Build ursabot itself via ursabot\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-05T11:53:43Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/19\\\",\\n+ \\\"user\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars1.githubusercontent.com/u/961747?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/kszucs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/kszucs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/kszucs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/kszucs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/kszucs\\\",\\n+ \\\"id\\\": 961747,\\n+ \\\"login\\\": \\\"kszucs\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjk2MTc0Nw==\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/kszucs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/kszucs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/kszucs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/kszucs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/kszucs/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/kszucs\\\"\\n+ }\\n+ },\\n+ \\\"organization\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"description\\\": \\\"Innovation lab for open source data science tools, powered by Apache Arrow\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/orgs/ursa-labs/events\\\",\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/orgs/ursa-labs/hooks\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"issues_url\\\": \\\"https://api.github.com/orgs/ursa-labs/issues\\\",\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/members{/member}\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"public_members_url\\\": \\\"https://api.github.com/orgs/ursa-labs/public_members{/member}\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/orgs/ursa-labs/repos\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/orgs/ursa-labs\\\"\\n+ },\\n+ \\\"repository\\\": {\\n+ \\\"archive_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\\\",\\n+ \\\"archived\\\": false,\\n+ \\\"assignees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\\\",\\n+ \\\"blobs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\\\",\\n+ \\\"branches_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\\\",\\n+ \\\"clone_url\\\": \\\"https://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"collaborators_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\\\",\\n+ \\\"comments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\\\",\\n+ \\\"commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\\\",\\n+ \\\"compare_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\\\",\\n+ \\\"contents_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\\\",\\n+ \\\"contributors_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/contributors\\\",\\n+ \\\"created_at\\\": \\\"2019-02-04T15:40:31Z\\\",\\n+ \\\"default_branch\\\": \\\"master\\\",\\n+ \\\"deployments_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/deployments\\\",\\n+ \\\"description\\\": null,\\n+ \\\"disabled\\\": false,\\n+ \\\"downloads_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/downloads\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/events\\\",\\n+ \\\"fork\\\": false,\\n+ \\\"forks\\\": 0,\\n+ \\\"forks_count\\\": 0,\\n+ \\\"forks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/forks\\\",\\n+ \\\"full_name\\\": \\\"ursa-labs/ursabot\\\",\\n+ \\\"git_commits_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\\\",\\n+ \\\"git_refs_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\\\",\\n+ \\\"git_tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\\\",\\n+ \\\"git_url\\\": \\\"git://github.com/ursa-labs/ursabot.git\\\",\\n+ \\\"has_downloads\\\": true,\\n+ \\\"has_issues\\\": true,\\n+ \\\"has_pages\\\": false,\\n+ \\\"has_projects\\\": true,\\n+ \\\"has_wiki\\\": true,\\n+ \\\"homepage\\\": null,\\n+ \\\"hooks_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/hooks\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"id\\\": 169101701,\\n+ \\\"issue_comment_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\\\",\\n+ \\\"issue_events_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\\\",\\n+ \\\"issues_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\\\",\\n+ \\\"keys_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\\\",\\n+ \\\"labels_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\\\",\\n+ \\\"language\\\": \\\"Jupyter Notebook\\\",\\n+ \\\"languages_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/languages\\\",\\n+ \\\"license\\\": null,\\n+ \\\"merges_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/merges\\\",\\n+ \\\"milestones_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\\\",\\n+ \\\"mirror_url\\\": null,\\n+ \\\"name\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\\\",\\n+ \\\"notifications_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\\\",\\n+ \\\"open_issues\\\": 19,\\n+ \\\"open_issues_count\\\": 19,\\n+ \\\"owner\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/46514972?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursa-labs/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursa-labs/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursa-labs/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursa-labs/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursa-labs\\\",\\n+ \\\"id\\\": 46514972,\\n+ \\\"login\\\": \\\"ursa-labs\\\",\\n+ \\\"node_id\\\": \\\"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursa-labs/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursa-labs/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursa-labs/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursa-labs/subscriptions\\\",\\n+ \\\"type\\\": \\\"Organization\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursa-labs\\\"\\n+ },\\n+ \\\"private\\\": false,\\n+ \\\"pulls_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\\\",\\n+ \\\"pushed_at\\\": \\\"2019-04-05T11:22:16Z\\\",\\n+ \\\"releases_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\\\",\\n+ \\\"size\\\": 892,\\n+ \\\"ssh_url\\\": \\\"git@github.com:ursa-labs/ursabot.git\\\",\\n+ \\\"stargazers_count\\\": 1,\\n+ \\\"stargazers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/stargazers\\\",\\n+ \\\"statuses_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\\\",\\n+ \\\"subscribers_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscribers\\\",\\n+ \\\"subscription_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/subscription\\\",\\n+ \\\"svn_url\\\": \\\"https://github.com/ursa-labs/ursabot\\\",\\n+ \\\"tags_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/tags\\\",\\n+ \\\"teams_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/teams\\\",\\n+ \\\"trees_url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\\\",\\n+ \\\"updated_at\\\": \\\"2019-04-04T17:49:10Z\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/repos/ursa-labs/ursabot\\\",\\n+ \\\"watchers\\\": 1,\\n+ \\\"watchers_count\\\": 1\\n+ },\\n+ \\\"sender\\\": {\\n+ \\\"avatar_url\\\": \\\"https://avatars2.githubusercontent.com/u/49275095?v=4\\\",\\n+ \\\"events_url\\\": \\\"https://api.github.com/users/ursabot/events{/privacy}\\\",\\n+ \\\"followers_url\\\": \\\"https://api.github.com/users/ursabot/followers\\\",\\n+ \\\"following_url\\\": \\\"https://api.github.com/users/ursabot/following{/other_user}\\\",\\n+ \\\"gists_url\\\": \\\"https://api.github.com/users/ursabot/gists{/gist_id}\\\",\\n+ \\\"gravatar_id\\\": \\\"\\\",\\n+ \\\"html_url\\\": \\\"https://github.com/ursabot\\\",\\n+ \\\"id\\\": 49275095,\\n+ \\\"login\\\": \\\"ursabot\\\",\\n+ \\\"node_id\\\": \\\"MDQ6VXNlcjQ5Mjc1MDk1\\\",\\n+ \\\"organizations_url\\\": \\\"https://api.github.com/users/ursabot/orgs\\\",\\n+ \\\"received_events_url\\\": \\\"https://api.github.com/users/ursabot/received_events\\\",\\n+ \\\"repos_url\\\": \\\"https://api.github.com/users/ursabot/repos\\\",\\n+ \\\"site_admin\\\": false,\\n+ \\\"starred_url\\\": \\\"https://api.github.com/users/ursabot/starred{/owner}{/repo}\\\",\\n+ \\\"subscriptions_url\\\": \\\"https://api.github.com/users/ursabot/subscriptions\\\",\\n+ \\\"type\\\": \\\"User\\\",\\n+ \\\"url\\\": \\\"https://api.github.com/users/ursabot\\\"\\n+ }\\n+}\"\n+ },\n+ {\n+ \"sha\": \"c738bb0eb54c87ba0f23e97e827d77c2be74d0b6\",\n+ \"filename\": \"ursabot/tests/test_hooks.py\",\n+ \"status\": \"modified\",\n+ \"additions\": 4,\n+ \"deletions\": 4,\n+ \"changes\": 8,\n+ \"blob_url\": \"https://github.com/ursa-labs/ursabot/blob/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py\",\n+ \"raw_url\": \"https://github.com/ursa-labs/ursabot/raw/2705da2b616b98fa6010a25813c5a7a27456f71d/ursabot/tests/test_hooks.py\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"patch\": \"@@ -54,7 +54,7 @@ class TestGithubHook(ChangeHookTestCase):\\n await self.request('ping', {})\\n assert len(self.hook.master.data.updates.changesAdded) == 0\\n \\n- @ensure_deferred\\n- async def test_issue_comment(self):\\n- payload = {}\\n- await self.request('issue_comment', payload)\\n+ # @ensure_deferred\\n+ # async def test_issue_comment(self):\\n+ # payload = {}\\n+ # await self.request('issue_comment', payload)\"\n+ }\n+ ]\n+}" + }, + { + "sha": "ad061d7244b917e6ea3853698dc3bc2a8c9c6857", + "filename": "ursabot/tests/fixtures/pull-request-26.json", + "status": "added", + "additions": 335, + "deletions": 0, + "changes": 335, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26.json", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/fixtures/pull-request-26.json", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/fixtures/pull-request-26.json?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,335 @@\n+{\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\",\n+ \"id\": 267785552,\n+ \"node_id\": \"MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy\",\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot/pull/26\",\n+ \"diff_url\": \"https://github.com/ursa-labs/ursabot/pull/26.diff\",\n+ \"patch_url\": \"https://github.com/ursa-labs/ursabot/pull/26.patch\",\n+ \"issue_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\",\n+ \"number\": 26,\n+ \"state\": \"open\",\n+ \"locked\": false,\n+ \"title\": \"Unittests for GithubHook\",\n+ \"user\": {\n+ \"login\": \"kszucs\",\n+ \"id\": 961747,\n+ \"node_id\": \"MDQ6VXNlcjk2MTc0Nw==\",\n+ \"avatar_url\": \"https://avatars1.githubusercontent.com/u/961747?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/kszucs\",\n+ \"html_url\": \"https://github.com/kszucs\",\n+ \"followers_url\": \"https://api.github.com/users/kszucs/followers\",\n+ \"following_url\": \"https://api.github.com/users/kszucs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/kszucs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/kszucs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/kszucs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/kszucs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/kszucs/repos\",\n+ \"events_url\": \"https://api.github.com/users/kszucs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/kszucs/received_events\",\n+ \"type\": \"User\",\n+ \"site_admin\": false\n+ },\n+ \"body\": \"\",\n+ \"created_at\": \"2019-04-05T11:22:15Z\",\n+ \"updated_at\": \"2019-04-05T12:01:40Z\",\n+ \"closed_at\": null,\n+ \"merged_at\": null,\n+ \"merge_commit_sha\": \"cc5dc3606988b3824be54df779ed2028776113cb\",\n+ \"assignee\": null,\n+ \"assignees\": [\n+\n+ ],\n+ \"requested_reviewers\": [\n+\n+ ],\n+ \"requested_teams\": [\n+\n+ ],\n+ \"labels\": [\n+\n+ ],\n+ \"milestone\": null,\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits\",\n+ \"review_comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments\",\n+ \"review_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"head\": {\n+ \"label\": \"ursa-labs:test-hook\",\n+ \"ref\": \"test-hook\",\n+ \"sha\": \"2705da2b616b98fa6010a25813c5a7a27456f71d\",\n+ \"user\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"repo\": {\n+ \"id\": 169101701,\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"name\": \"ursabot\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"private\": false,\n+ \"owner\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"description\": null,\n+ \"fork\": false,\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"homepage\": null,\n+ \"size\": 898,\n+ \"stargazers_count\": 1,\n+ \"watchers_count\": 1,\n+ \"language\": \"Jupyter Notebook\",\n+ \"has_issues\": true,\n+ \"has_projects\": true,\n+ \"has_downloads\": true,\n+ \"has_wiki\": true,\n+ \"has_pages\": false,\n+ \"forks_count\": 0,\n+ \"mirror_url\": null,\n+ \"archived\": false,\n+ \"disabled\": false,\n+ \"open_issues_count\": 19,\n+ \"license\": null,\n+ \"forks\": 0,\n+ \"open_issues\": 19,\n+ \"watchers\": 1,\n+ \"default_branch\": \"master\"\n+ }\n+ },\n+ \"base\": {\n+ \"label\": \"ursa-labs:master\",\n+ \"ref\": \"master\",\n+ \"sha\": \"a162ad254b589b924db47e057791191b39613fd5\",\n+ \"user\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"repo\": {\n+ \"id\": 169101701,\n+ \"node_id\": \"MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=\",\n+ \"name\": \"ursabot\",\n+ \"full_name\": \"ursa-labs/ursabot\",\n+ \"private\": false,\n+ \"owner\": {\n+ \"login\": \"ursa-labs\",\n+ \"id\": 46514972,\n+ \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy\",\n+ \"avatar_url\": \"https://avatars2.githubusercontent.com/u/46514972?v=4\",\n+ \"gravatar_id\": \"\",\n+ \"url\": \"https://api.github.com/users/ursa-labs\",\n+ \"html_url\": \"https://github.com/ursa-labs\",\n+ \"followers_url\": \"https://api.github.com/users/ursa-labs/followers\",\n+ \"following_url\": \"https://api.github.com/users/ursa-labs/following{/other_user}\",\n+ \"gists_url\": \"https://api.github.com/users/ursa-labs/gists{/gist_id}\",\n+ \"starred_url\": \"https://api.github.com/users/ursa-labs/starred{/owner}{/repo}\",\n+ \"subscriptions_url\": \"https://api.github.com/users/ursa-labs/subscriptions\",\n+ \"organizations_url\": \"https://api.github.com/users/ursa-labs/orgs\",\n+ \"repos_url\": \"https://api.github.com/users/ursa-labs/repos\",\n+ \"events_url\": \"https://api.github.com/users/ursa-labs/events{/privacy}\",\n+ \"received_events_url\": \"https://api.github.com/users/ursa-labs/received_events\",\n+ \"type\": \"Organization\",\n+ \"site_admin\": false\n+ },\n+ \"html_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"description\": null,\n+ \"fork\": false,\n+ \"url\": \"https://api.github.com/repos/ursa-labs/ursabot\",\n+ \"forks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/forks\",\n+ \"keys_url\": \"https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}\",\n+ \"collaborators_url\": \"https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}\",\n+ \"teams_url\": \"https://api.github.com/repos/ursa-labs/ursabot/teams\",\n+ \"hooks_url\": \"https://api.github.com/repos/ursa-labs/ursabot/hooks\",\n+ \"issue_events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}\",\n+ \"events_url\": \"https://api.github.com/repos/ursa-labs/ursabot/events\",\n+ \"assignees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}\",\n+ \"branches_url\": \"https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}\",\n+ \"tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/tags\",\n+ \"blobs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}\",\n+ \"git_tags_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}\",\n+ \"git_refs_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}\",\n+ \"trees_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}\",\n+ \"statuses_url\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}\",\n+ \"languages_url\": \"https://api.github.com/repos/ursa-labs/ursabot/languages\",\n+ \"stargazers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/stargazers\",\n+ \"contributors_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contributors\",\n+ \"subscribers_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscribers\",\n+ \"subscription_url\": \"https://api.github.com/repos/ursa-labs/ursabot/subscription\",\n+ \"commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}\",\n+ \"git_commits_url\": \"https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}\",\n+ \"comments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/comments{/number}\",\n+ \"issue_comment_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}\",\n+ \"contents_url\": \"https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}\",\n+ \"compare_url\": \"https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}\",\n+ \"merges_url\": \"https://api.github.com/repos/ursa-labs/ursabot/merges\",\n+ \"archive_url\": \"https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}\",\n+ \"downloads_url\": \"https://api.github.com/repos/ursa-labs/ursabot/downloads\",\n+ \"issues_url\": \"https://api.github.com/repos/ursa-labs/ursabot/issues{/number}\",\n+ \"pulls_url\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}\",\n+ \"milestones_url\": \"https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}\",\n+ \"notifications_url\": \"https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}\",\n+ \"labels_url\": \"https://api.github.com/repos/ursa-labs/ursabot/labels{/name}\",\n+ \"releases_url\": \"https://api.github.com/repos/ursa-labs/ursabot/releases{/id}\",\n+ \"deployments_url\": \"https://api.github.com/repos/ursa-labs/ursabot/deployments\",\n+ \"created_at\": \"2019-02-04T15:40:31Z\",\n+ \"updated_at\": \"2019-04-04T17:49:10Z\",\n+ \"pushed_at\": \"2019-04-05T12:01:40Z\",\n+ \"git_url\": \"git://github.com/ursa-labs/ursabot.git\",\n+ \"ssh_url\": \"git@github.com:ursa-labs/ursabot.git\",\n+ \"clone_url\": \"https://github.com/ursa-labs/ursabot.git\",\n+ \"svn_url\": \"https://github.com/ursa-labs/ursabot\",\n+ \"homepage\": null,\n+ \"size\": 898,\n+ \"stargazers_count\": 1,\n+ \"watchers_count\": 1,\n+ \"language\": \"Jupyter Notebook\",\n+ \"has_issues\": true,\n+ \"has_projects\": true,\n+ \"has_downloads\": true,\n+ \"has_wiki\": true,\n+ \"has_pages\": false,\n+ \"forks_count\": 0,\n+ \"mirror_url\": null,\n+ \"archived\": false,\n+ \"disabled\": false,\n+ \"open_issues_count\": 19,\n+ \"license\": null,\n+ \"forks\": 0,\n+ \"open_issues\": 19,\n+ \"watchers\": 1,\n+ \"default_branch\": \"master\"\n+ }\n+ },\n+ \"_links\": {\n+ \"self\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26\"\n+ },\n+ \"html\": {\n+ \"href\": \"https://github.com/ursa-labs/ursabot/pull/26\"\n+ },\n+ \"issue\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26\"\n+ },\n+ \"comments\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments\"\n+ },\n+ \"review_comments\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments\"\n+ },\n+ \"review_comment\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}\"\n+ },\n+ \"commits\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits\"\n+ },\n+ \"statuses\": {\n+ \"href\": \"https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d\"\n+ }\n+ },\n+ \"author_association\": \"MEMBER\",\n+ \"merged\": false,\n+ \"mergeable\": true,\n+ \"rebaseable\": true,\n+ \"mergeable_state\": \"unstable\",\n+ \"merged_by\": null,\n+ \"comments\": 5,\n+ \"review_comments\": 0,\n+ \"maintainer_can_modify\": false,\n+ \"commits\": 2,\n+ \"additions\": 1124,\n+ \"deletions\": 0,\n+ \"changed_files\": 7\n+}" + }, + { + "sha": "e87b27d2d7b4956d15f7468488b96cf6a06686f4", + "filename": "ursabot/tests/test_hooks.py", + "status": "added", + "additions": 116, + "deletions": 0, + "changes": 116, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/test_hooks.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/tests/test_hooks.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/tests/test_hooks.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,116 @@\n+import json\n+from pathlib import Path\n+from twisted.trial import unittest\n+\n+from buildbot.test.util.misc import TestReactorMixin\n+from buildbot.test.fake.httpclientservice import \\\n+ HTTPClientService as FakeHTTPClientService\n+from buildbot.test.unit.test_www_hooks_github import (\n+ _prepare_request, _prepare_github_change_hook)\n+\n+from ursabot.utils import ensure_deferred\n+from ursabot.hooks import GithubHook\n+\n+\n+class ChangeHookTestCase(unittest.TestCase, TestReactorMixin):\n+\n+ klass = None\n+\n+ @ensure_deferred\n+ async def setUp(self):\n+ self.setUpTestReactor()\n+\n+ assert self.klass is not None\n+ self.hook = _prepare_github_change_hook(self, **{'class': self.klass})\n+ self.master = self.hook.master\n+ self.http = await FakeHTTPClientService.getFakeService(\n+ self.master, self, 'https://api.github.com',\n+ headers={'User-Agent': 'Buildbot'}, debug=False, verify=False)\n+\n+ await self.master.startService()\n+\n+ @ensure_deferred\n+ async def tearDown(self):\n+ await self.master.stopService()\n+\n+ async def trigger(self, event, payload, headers=None, _secret=None):\n+ payload = json.dumps(payload).encode()\n+ request = _prepare_request(event, payload, _secret=_secret,\n+ headers=headers)\n+ await request.test_render(self.hook)\n+ return request\n+\n+ def load_fixture(self, name):\n+ path = Path(__file__).parent / 'fixtures' / f'{name}.json'\n+ with path.open('r') as fp:\n+ return json.load(fp)\n+\n+\n+class TestGithubHook(ChangeHookTestCase):\n+\n+ klass = GithubHook\n+\n+ @ensure_deferred\n+ async def test_ping(self):\n+ await self.trigger('ping', {})\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_not_mentioning_ursabot(self):\n+ payload = self.load_fixture('issue-comment-not-mentioning-ursabot')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_by_ursabot(self):\n+ payload = self.load_fixture('issue-comment-by-ursabot')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_with_empty_command(self):\n+ # responds to the comment\n+ request_json = {'body': 'Unknown command \"\"'}\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/26/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-with-empty-command')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_without_pull_request(self):\n+ # responds to the comment\n+ request_json = {\n+ 'body': 'Ursabot only listens to pull request comments!'\n+ }\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/19/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-without-pull-request')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 0\n+\n+ @ensure_deferred\n+ async def test_issue_comment_build_command(self):\n+ # handle_issue_comment queries the pull request\n+ request_json = self.load_fixture('pull-request-26')\n+ self.http.expect('get', '/repos/ursa-labs/ursabot/pulls/26',\n+ content_json=request_json)\n+ # tigger handle_pull_request which fetches the commit\n+ request_json = self.load_fixture('pull-request-26-commit')\n+ commit = '2705da2b616b98fa6010a25813c5a7a27456f71d'\n+ self.http.expect('get', f'/repos/ursa-labs/ursabot/commits/{commit}',\n+ content_json=request_json)\n+\n+ # then responds to the comment\n+ request_json = {'body': \"I've successfully started builds for this PR\"}\n+ response_json = ''\n+ self.http.expect('post', '/repos/ursa-labs/ursabot/issues/26/comments',\n+ json=request_json, content_json=response_json)\n+\n+ payload = self.load_fixture('issue-comment-build-command')\n+ await self.trigger('issue_comment', payload=payload)\n+ assert len(self.hook.master.data.updates.changesAdded) == 1" + }, + { + "sha": "3ff0e88660cf186420e8bc672735e4d446963192", + "filename": "ursabot/utils.py", + "status": "added", + "additions": 10, + "deletions": 0, + "changes": 10, + "blob_url": "https://github.com/ursa-labs/ursabot/blob/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/utils.py", + "raw_url": "https://github.com/ursa-labs/ursabot/raw/70267dee34884e4b972388e1b30d57f6248c58d0/ursabot/utils.py", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/ursabot/utils.py?ref=70267dee34884e4b972388e1b30d57f6248c58d0", + "patch": "@@ -0,0 +1,10 @@\n+import functools\n+from twisted.internet import defer\n+\n+\n+def ensure_deferred(f):\n+ @functools.wraps(f)\n+ def wrapper(*args, **kwargs):\n+ result = f(*args, **kwargs)\n+ return defer.ensureDeferred(result)\n+ return wrapper" + } +]
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json new file mode 100644 index 000000000..d295afb39 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/fixtures/pull-request-26.json @@ -0,0 +1,329 @@ +{ + "url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26", + "id": 267785552, + "node_id": "MDExOlB1bGxSZXF1ZXN0MjY3Nzg1NTUy", + "html_url": "https://github.com/ursa-labs/ursabot/pull/26", + "diff_url": "https://github.com/ursa-labs/ursabot/pull/26.diff", + "patch_url": "https://github.com/ursa-labs/ursabot/pull/26.patch", + "issue_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26", + "number": 26, + "state": "open", + "locked": false, + "title": "Unittests for GithubHook", + "user": { + "login": "kszucs", + "id": 961747, + "node_id": "MDQ6VXNlcjk2MTc0Nw==", + "avatar_url": "https://avatars1.githubusercontent.com/u/961747?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/kszucs", + "html_url": "https://github.com/kszucs", + "followers_url": "https://api.github.com/users/kszucs/followers", + "following_url": "https://api.github.com/users/kszucs/following{/other_user}", + "gists_url": "https://api.github.com/users/kszucs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/kszucs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/kszucs/subscriptions", + "organizations_url": "https://api.github.com/users/kszucs/orgs", + "repos_url": "https://api.github.com/users/kszucs/repos", + "events_url": "https://api.github.com/users/kszucs/events{/privacy}", + "received_events_url": "https://api.github.com/users/kszucs/received_events", + "type": "User", + "site_admin": false + }, + "body": "", + "body_html": "", + "body_text": "", + "created_at": "2019-04-05T11:22:15Z", + "updated_at": "2019-04-05T12:01:40Z", + "closed_at": null, + "merged_at": null, + "merge_commit_sha": "cc5dc3606988b3824be54df779ed2028776113cb", + "assignee": null, + "assignees": [], + "requested_reviewers": [], + "requested_teams": [], + "labels": [], + "milestone": null, + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits", + "review_comments_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments", + "review_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d", + "head": { + "label": "ursa-labs:test-hook", + "ref": "test-hook", + "sha": "2705da2b616b98fa6010a25813c5a7a27456f71d", + "user": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 169101701, + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "name": "ursabot", + "full_name": "ursa-labs/ursabot", + "private": false, + "owner": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/ursa-labs/ursabot", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "created_at": "2019-02-04T15:40:31Z", + "updated_at": "2019-04-04T17:49:10Z", + "pushed_at": "2019-04-05T12:01:40Z", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "svn_url": "https://github.com/ursa-labs/ursabot", + "homepage": null, + "size": 898, + "stargazers_count": 1, + "watchers_count": 1, + "language": "Jupyter Notebook", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 19, + "license": null, + "forks": 0, + "open_issues": 19, + "watchers": 1, + "default_branch": "master" + } + }, + "base": { + "label": "ursa-labs:master", + "ref": "master", + "sha": "a162ad254b589b924db47e057791191b39613fd5", + "user": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "repo": { + "id": 169101701, + "node_id": "MDEwOlJlcG9zaXRvcnkxNjkxMDE3MDE=", + "name": "ursabot", + "full_name": "ursa-labs/ursabot", + "private": false, + "owner": { + "login": "ursa-labs", + "id": 46514972, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjQ2NTE0OTcy", + "avatar_url": "https://avatars2.githubusercontent.com/u/46514972?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ursa-labs", + "html_url": "https://github.com/ursa-labs", + "followers_url": "https://api.github.com/users/ursa-labs/followers", + "following_url": "https://api.github.com/users/ursa-labs/following{/other_user}", + "gists_url": "https://api.github.com/users/ursa-labs/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ursa-labs/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ursa-labs/subscriptions", + "organizations_url": "https://api.github.com/users/ursa-labs/orgs", + "repos_url": "https://api.github.com/users/ursa-labs/repos", + "events_url": "https://api.github.com/users/ursa-labs/events{/privacy}", + "received_events_url": "https://api.github.com/users/ursa-labs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/ursa-labs/ursabot", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/ursa-labs/ursabot", + "forks_url": "https://api.github.com/repos/ursa-labs/ursabot/forks", + "keys_url": "https://api.github.com/repos/ursa-labs/ursabot/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/ursa-labs/ursabot/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/ursa-labs/ursabot/teams", + "hooks_url": "https://api.github.com/repos/ursa-labs/ursabot/hooks", + "issue_events_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/events{/number}", + "events_url": "https://api.github.com/repos/ursa-labs/ursabot/events", + "assignees_url": "https://api.github.com/repos/ursa-labs/ursabot/assignees{/user}", + "branches_url": "https://api.github.com/repos/ursa-labs/ursabot/branches{/branch}", + "tags_url": "https://api.github.com/repos/ursa-labs/ursabot/tags", + "blobs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/ursa-labs/ursabot/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/ursa-labs/ursabot/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/ursa-labs/ursabot/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/ursa-labs/ursabot/statuses/{sha}", + "languages_url": "https://api.github.com/repos/ursa-labs/ursabot/languages", + "stargazers_url": "https://api.github.com/repos/ursa-labs/ursabot/stargazers", + "contributors_url": "https://api.github.com/repos/ursa-labs/ursabot/contributors", + "subscribers_url": "https://api.github.com/repos/ursa-labs/ursabot/subscribers", + "subscription_url": "https://api.github.com/repos/ursa-labs/ursabot/subscription", + "commits_url": "https://api.github.com/repos/ursa-labs/ursabot/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/ursa-labs/ursabot/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/ursa-labs/ursabot/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/ursa-labs/ursabot/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/ursa-labs/ursabot/contents/{+path}", + "compare_url": "https://api.github.com/repos/ursa-labs/ursabot/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/ursa-labs/ursabot/merges", + "archive_url": "https://api.github.com/repos/ursa-labs/ursabot/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/ursa-labs/ursabot/downloads", + "issues_url": "https://api.github.com/repos/ursa-labs/ursabot/issues{/number}", + "pulls_url": "https://api.github.com/repos/ursa-labs/ursabot/pulls{/number}", + "milestones_url": "https://api.github.com/repos/ursa-labs/ursabot/milestones{/number}", + "notifications_url": "https://api.github.com/repos/ursa-labs/ursabot/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/ursa-labs/ursabot/labels{/name}", + "releases_url": "https://api.github.com/repos/ursa-labs/ursabot/releases{/id}", + "deployments_url": "https://api.github.com/repos/ursa-labs/ursabot/deployments", + "created_at": "2019-02-04T15:40:31Z", + "updated_at": "2019-04-04T17:49:10Z", + "pushed_at": "2019-04-05T12:01:40Z", + "git_url": "git://github.com/ursa-labs/ursabot.git", + "ssh_url": "git@github.com:ursa-labs/ursabot.git", + "clone_url": "https://github.com/ursa-labs/ursabot.git", + "svn_url": "https://github.com/ursa-labs/ursabot", + "homepage": null, + "size": 898, + "stargazers_count": 1, + "watchers_count": 1, + "language": "Jupyter Notebook", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 19, + "license": null, + "forks": 0, + "open_issues": 19, + "watchers": 1, + "default_branch": "master" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26" + }, + "html": { + "href": "https://github.com/ursa-labs/ursabot/pull/26" + }, + "issue": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26" + }, + "comments": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/issues/26/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/pulls/26/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/ursa-labs/ursabot/statuses/2705da2b616b98fa6010a25813c5a7a27456f71d" + } + }, + "author_association": "MEMBER", + "merged": false, + "mergeable": true, + "rebaseable": true, + "mergeable_state": "unstable", + "merged_by": null, + "comments": 5, + "review_comments": 0, + "maintainer_can_modify": false, + "commits": 2, + "additions": 1124, + "deletions": 0, + "changed_files": 7 +}
\ No newline at end of file diff --git a/src/arrow/dev/archery/archery/tests/test_benchmarks.py b/src/arrow/dev/archery/archery/tests/test_benchmarks.py new file mode 100644 index 000000000..fab1e8d44 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/test_benchmarks.py @@ -0,0 +1,383 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from archery.benchmark.codec import JsonEncoder +from archery.benchmark.core import Benchmark, median +from archery.benchmark.compare import ( + BenchmarkComparator, RunnerComparator +) +from archery.benchmark.google import ( + GoogleBenchmark, GoogleBenchmarkObservation +) +from archery.benchmark.runner import StaticBenchmarkRunner + + +def test_benchmark_comparator(): + unit = "micros" + + assert not BenchmarkComparator( + Benchmark("contender", unit, True, [10], unit, [1]), + Benchmark("baseline", unit, True, [20], unit, [1]), + ).regression + + assert BenchmarkComparator( + Benchmark("contender", unit, False, [10], unit, [1]), + Benchmark("baseline", unit, False, [20], unit, [1]), + ).regression + + assert BenchmarkComparator( + Benchmark("contender", unit, True, [20], unit, [1]), + Benchmark("baseline", unit, True, [10], unit, [1]), + ).regression + + assert not BenchmarkComparator( + Benchmark("contender", unit, False, [20], unit, [1]), + Benchmark("baseline", unit, False, [10], unit, [1]), + ).regression + + +def test_static_runner_from_json_not_a_regression(): + archery_result = { + "suites": [ + { + "name": "arrow-value-parsing-benchmark", + "benchmarks": [ + { + "name": "FloatParsing<DoubleType>", + "unit": "items_per_second", + "less_is_better": False, + "values": [ + 109941112.87296811 + ], + "time_unit": "ns", + "times": [ + 9095.800104330105 + ] + }, + ] + } + ] + } + + contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result)) + baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result)) + [comparison] = RunnerComparator(contender, baseline).comparisons + assert not comparison.regression + + +def test_static_runner_from_json_regression(): + archery_result = { + "suites": [ + { + "name": "arrow-value-parsing-benchmark", + "benchmarks": [ + { + "name": "FloatParsing<DoubleType>", + "unit": "items_per_second", + "less_is_better": False, + "values": [ + 109941112.87296811 + ], + "time_unit": "ns", + "times": [ + 9095.800104330105 + ] + }, + ] + } + ] + } + + contender = StaticBenchmarkRunner.from_json(json.dumps(archery_result)) + + # introduce artificial regression + archery_result['suites'][0]['benchmarks'][0]['values'][0] *= 2 + baseline = StaticBenchmarkRunner.from_json(json.dumps(archery_result)) + + [comparison] = RunnerComparator(contender, baseline).comparisons + assert comparison.regression + + +def test_benchmark_median(): + assert median([10]) == 10 + assert median([1, 2, 3]) == 2 + assert median([1, 2]) == 1.5 + assert median([1, 2, 3, 4]) == 2.5 + assert median([1, 1, 1, 1]) == 1 + try: + median([]) + assert False + except ValueError: + pass + + +def assert_benchmark(name, google_result, archery_result): + observation = GoogleBenchmarkObservation(**google_result) + benchmark = GoogleBenchmark(name, [observation]) + result = json.dumps(benchmark, cls=JsonEncoder) + assert json.loads(result) == archery_result + + +def test_items_per_second(): + name = "ArrayArrayKernel<AddChecked, UInt8Type>/32768/0" + google_result = { + "cpu_time": 116292.58886653671, + "items_per_second": 281772039.9844759, + "iterations": 5964, + "name": name, + "null_percent": 0.0, + "real_time": 119811.77313729875, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "size": 32768.0, + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 5964, + "null_percent": 0.0, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "items_per_second", + "less_is_better": False, + "values": [281772039.9844759], + "time_unit": "ns", + "times": [119811.77313729875], + } + assert "items_per_second" in google_result + assert "bytes_per_second" not in google_result + assert_benchmark(name, google_result, archery_result) + + +def test_bytes_per_second(): + name = "BufferOutputStreamLargeWrites/real_time" + google_result = { + "bytes_per_second": 1890209037.3405428, + "cpu_time": 17018127.659574457, + "iterations": 47, + "name": name, + "real_time": 17458386.53190963, + "repetition_index": 1, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 47, + "repetition_index": 1, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "bytes_per_second", + "less_is_better": False, + "values": [1890209037.3405428], + "time_unit": "ns", + "times": [17458386.53190963], + } + assert "items_per_second" not in google_result + assert "bytes_per_second" in google_result + assert_benchmark(name, google_result, archery_result) + + +def test_both_items_and_bytes_per_second(): + name = "ArrayArrayKernel<AddChecked, UInt8Type>/32768/0" + google_result = { + "bytes_per_second": 281772039.9844759, + "cpu_time": 116292.58886653671, + "items_per_second": 281772039.9844759, + "iterations": 5964, + "name": name, + "null_percent": 0.0, + "real_time": 119811.77313729875, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "size": 32768.0, + "threads": 1, + "time_unit": "ns", + } + # Note that bytes_per_second trumps items_per_second + archery_result = { + "counters": {"iterations": 5964, + "null_percent": 0.0, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "bytes_per_second", + "less_is_better": False, + "values": [281772039.9844759], + "time_unit": "ns", + "times": [119811.77313729875], + } + assert "items_per_second" in google_result + assert "bytes_per_second" in google_result + assert_benchmark(name, google_result, archery_result) + + +def test_neither_items_nor_bytes_per_second(): + name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time" + google_result = { + "cpu_time": 1778.6004847419827, + "iterations": 352765, + "name": name, + "real_time": 1835.3137357788837, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 352765, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "ns", + "less_is_better": True, + "values": [1835.3137357788837], + "time_unit": "ns", + "times": [1835.3137357788837], + } + assert "items_per_second" not in google_result + assert "bytes_per_second" not in google_result + assert_benchmark(name, google_result, archery_result) + + +def test_prefer_real_time(): + name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time" + google_result = { + "cpu_time": 1778.6004847419827, + "iterations": 352765, + "name": name, + "real_time": 1835.3137357788837, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 352765, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "ns", + "less_is_better": True, + "values": [1835.3137357788837], + "time_unit": "ns", + "times": [1835.3137357788837], + } + assert name.endswith("/real_time") + assert_benchmark(name, google_result, archery_result) + + +def test_prefer_cpu_time(): + name = "AllocateDeallocate<Jemalloc>/size:1048576" + google_result = { + "cpu_time": 1778.6004847419827, + "iterations": 352765, + "name": name, + "real_time": 1835.3137357788837, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 352765, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "ns", + "less_is_better": True, + "values": [1778.6004847419827], + "time_unit": "ns", + "times": [1835.3137357788837], + } + assert not name.endswith("/real_time") + assert_benchmark(name, google_result, archery_result) + + +def test_omits_aggregates(): + name = "AllocateDeallocate<Jemalloc>/size:1048576/real_time" + google_aggregate = { + "aggregate_name": "mean", + "cpu_time": 1757.428694267678, + "iterations": 3, + "name": "AllocateDeallocate<Jemalloc>/size:1048576/real_time_mean", + "real_time": 1849.3869337041162, + "repetitions": 0, + "run_name": name, + "run_type": "aggregate", + "threads": 1, + "time_unit": "ns", + } + google_result = { + "cpu_time": 1778.6004847419827, + "iterations": 352765, + "name": name, + "real_time": 1835.3137357788837, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "run_type": "iteration", + "threads": 1, + "time_unit": "ns", + } + archery_result = { + "counters": {"iterations": 352765, + "repetition_index": 0, + "repetitions": 0, + "run_name": name, + "threads": 1}, + "name": name, + "unit": "ns", + "less_is_better": True, + "values": [1835.3137357788837], + "time_unit": "ns", + "times": [1835.3137357788837], + } + assert google_aggregate["run_type"] == "aggregate" + assert google_result["run_type"] == "iteration" + observation1 = GoogleBenchmarkObservation(**google_aggregate) + observation2 = GoogleBenchmarkObservation(**google_result) + benchmark = GoogleBenchmark(name, [observation1, observation2]) + result = json.dumps(benchmark, cls=JsonEncoder) + assert json.loads(result) == archery_result diff --git a/src/arrow/dev/archery/archery/tests/test_bot.py b/src/arrow/dev/archery/archery/tests/test_bot.py new file mode 100644 index 000000000..e84fb7e27 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/test_bot.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from unittest.mock import Mock + +import click +import pytest +import responses as rsps + +from archery.bot import CommentBot, CommandError, group + + +@pytest.fixture +def responses(): + with rsps.RequestsMock() as mock: + yield mock + + +def github_url(path): + return 'https://api.github.com:443/{}'.format(path.strip('/')) + + +@group() +def custom_handler(): + pass + + +@custom_handler.command() +@click.pass_obj +def extra(obj): + return obj + + +@custom_handler.command() +@click.option('--force', '-f', is_flag=True) +def build(force): + return force + + +@custom_handler.command() +@click.option('--name', required=True) +def benchmark(name): + return name + + +def test_click_based_commands(): + assert custom_handler('build') is False + assert custom_handler('build -f') is True + + assert custom_handler('benchmark --name strings') == 'strings' + with pytest.raises(CommandError): + assert custom_handler('benchmark') + + assert custom_handler('extra', extra='data') == {'extra': 'data'} + + +@pytest.mark.parametrize('fixture_name', [ + # the bot is not mentioned, nothing to do + 'event-issue-comment-not-mentioning-ursabot.json', + # don't respond to itself, it prevents recursive comment storms! + 'event-issue-comment-by-ursabot.json', + # non-authorized user sent the comment, do not respond + 'event-issue-comment-by-non-authorized-user.json', +]) +def test_noop_events(load_fixture, fixture_name): + payload = load_fixture(fixture_name) + + handler = Mock() + bot = CommentBot(name='ursabot', token='', handler=handler) + bot.handle('issue_comment', payload) + + handler.assert_not_called() + + +def test_issue_comment_without_pull_request(load_fixture, responses): + responses.add( + responses.GET, + github_url('/repositories/169101701/issues/19'), + json=load_fixture('issue-19.json'), + status=200 + ) + responses.add( + responses.GET, + github_url('repos/ursa-labs/ursabot/pulls/19'), + json={}, + status=404 + ) + responses.add( + responses.POST, + github_url('/repos/ursa-labs/ursabot/issues/19/comments'), + json={} + ) + + def handler(command, **kwargs): + pass + + payload = load_fixture('event-issue-comment-without-pull-request.json') + bot = CommentBot(name='ursabot', token='', handler=handler) + bot.handle('issue_comment', payload) + + post = responses.calls[2] + assert json.loads(post.request.body) == { + 'body': "The comment bot only listens to pull request comments!" + } + + +def test_respond_with_usage(load_fixture, responses): + responses.add( + responses.GET, + github_url('/repositories/169101701/issues/26'), + json=load_fixture('issue-26.json'), + status=200 + ) + responses.add( + responses.GET, + github_url('/repos/ursa-labs/ursabot/pulls/26'), + json=load_fixture('pull-request-26.json'), + status=200 + ) + responses.add( + responses.GET, + github_url('/repos/ursa-labs/ursabot/issues/comments/480243811'), + json=load_fixture('issue-comment-480243811.json') + ) + responses.add( + responses.POST, + github_url('/repos/ursa-labs/ursabot/issues/26/comments'), + json={} + ) + + def handler(command, **kwargs): + raise CommandError('test-usage') + + payload = load_fixture('event-issue-comment-with-empty-command.json') + bot = CommentBot(name='ursabot', token='', handler=handler) + bot.handle('issue_comment', payload) + + post = responses.calls[3] + assert json.loads(post.request.body) == {'body': '```\ntest-usage\n```'} + + +@pytest.mark.parametrize(('command', 'reaction'), [ + ('@ursabot build', '+1'), + ('@ursabot build\nwith a comment', '+1'), + ('@ursabot listen', '-1'), +]) +def test_issue_comment_with_commands(load_fixture, responses, command, + reaction): + responses.add( + responses.GET, + github_url('/repositories/169101701/issues/26'), + json=load_fixture('issue-26.json'), + status=200 + ) + responses.add( + responses.GET, + github_url('/repos/ursa-labs/ursabot/pulls/26'), + json=load_fixture('pull-request-26.json'), + status=200 + ) + responses.add( + responses.GET, + github_url('/repos/ursa-labs/ursabot/issues/comments/480248726'), + json=load_fixture('issue-comment-480248726.json') + ) + responses.add( + responses.POST, + github_url( + '/repos/ursa-labs/ursabot/issues/comments/480248726/reactions' + ), + json={} + ) + + def handler(command, **kwargs): + if command == 'build': + return True + else: + raise ValueError('Only `build` command is supported.') + + payload = load_fixture('event-issue-comment-build-command.json') + payload["comment"]["body"] = command + + bot = CommentBot(name='ursabot', token='', handler=handler) + bot.handle('issue_comment', payload) + + post = responses.calls[3] + assert json.loads(post.request.body) == {'content': reaction} + + +def test_issue_comment_with_commands_bot_not_first(load_fixture, responses): + # when the @-mention is not first, this is a no-op + handler = Mock() + + payload = load_fixture('event-issue-comment-build-command.json') + payload["comment"]["body"] = 'with a comment\n@ursabot build' + + bot = CommentBot(name='ursabot', token='', handler=handler) + bot.handle('issue_comment', payload) + + handler.assert_not_called() diff --git a/src/arrow/dev/archery/archery/tests/test_cli.py b/src/arrow/dev/archery/archery/tests/test_cli.py new file mode 100644 index 000000000..3891a2c28 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/test_cli.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path +from unittest.mock import patch + +from click.testing import CliRunner + +from archery.cli import archery + + +@patch("archery.linking.check_dynamic_library_dependencies") +def test_linking_check_dependencies(fn): + args = [ + "linking", + "check-dependencies", + "-a", "libarrow", + "-d", "libcurl", + "somelib.so" + ] + result = CliRunner().invoke(archery, args) + assert result.exit_code == 0 + fn.assert_called_once_with( + Path('somelib.so'), allowed={'libarrow'}, disallowed={'libcurl'} + ) diff --git a/src/arrow/dev/archery/archery/tests/test_release.py b/src/arrow/dev/archery/archery/tests/test_release.py new file mode 100644 index 000000000..75aac8921 --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/test_release.py @@ -0,0 +1,333 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from archery.release import ( + Release, MajorRelease, MinorRelease, PatchRelease, + Jira, Version, Issue, CommitTitle, Commit +) +from archery.testing import DotDict + + +# subset of issues per revision +_issues = { + "1.0.1": [ + Issue("ARROW-9684", type="Bug", summary="[C++] Title"), + Issue("ARROW-9667", type="New Feature", summary="[Crossbow] Title"), + Issue("ARROW-9659", type="Bug", summary="[C++] Title"), + Issue("ARROW-9644", type="Bug", summary="[C++][Dataset] Title"), + Issue("ARROW-9643", type="Bug", summary="[C++] Title"), + Issue("ARROW-9609", type="Bug", summary="[C++] Title"), + Issue("ARROW-9606", type="Bug", summary="[C++][Dataset] Title") + ], + "1.0.0": [ + Issue("ARROW-300", type="New Feature", summary="[Format] Title"), + Issue("ARROW-4427", type="Task", summary="[Doc] Title"), + Issue("ARROW-5035", type="Improvement", summary="[C#] Title"), + Issue("ARROW-8473", type="Bug", summary="[Rust] Title"), + Issue("ARROW-8472", type="Bug", summary="[Go][Integration] Title"), + Issue("ARROW-8471", type="Bug", summary="[C++][Integration] Title"), + Issue("ARROW-8974", type="Improvement", summary="[C++] Title"), + Issue("ARROW-8973", type="New Feature", summary="[Java] Title") + ], + "0.17.1": [ + Issue("ARROW-8684", type="Bug", summary="[Python] Title"), + Issue("ARROW-8657", type="Bug", summary="[C++][Parquet] Title"), + Issue("ARROW-8641", type="Bug", summary="[Python] Title"), + Issue("ARROW-8609", type="Bug", summary="[C++] Title"), + ], + "0.17.0": [ + Issue("ARROW-2882", type="New Feature", summary="[C++][Python] Title"), + Issue("ARROW-2587", type="Bug", summary="[Python] Title"), + Issue("ARROW-2447", type="Improvement", summary="[C++] Title"), + Issue("ARROW-2255", type="Bug", summary="[Integration] Title"), + Issue("ARROW-1907", type="Bug", summary="[C++/Python] Title"), + Issue("ARROW-1636", type="New Feature", summary="[Format] Title") + ] +} + + +class FakeJira(Jira): + + def __init__(self): + pass + + def project_versions(self, project='ARROW'): + return [ + Version.parse("3.0.0", released=False), + Version.parse("2.0.0", released=False), + Version.parse("1.1.0", released=False), + Version.parse("1.0.1", released=False), + Version.parse("1.0.0", released=True), + Version.parse("0.17.1", released=True), + Version.parse("0.17.0", released=True), + Version.parse("0.16.0", released=True), + Version.parse("0.15.2", released=True), + Version.parse("0.15.1", released=True), + Version.parse("0.15.0", released=True), + ] + + def project_issues(self, version, project='ARROW'): + return _issues[str(version)] + + +@pytest.fixture +def fake_jira(): + return FakeJira() + + +def test_version(fake_jira): + v = Version.parse("1.2.5") + assert str(v) == "1.2.5" + assert v.major == 1 + assert v.minor == 2 + assert v.patch == 5 + assert v.released is False + assert v.release_date is None + + v = Version.parse("1.0.0", released=True, release_date="2020-01-01") + assert str(v) == "1.0.0" + assert v.major == 1 + assert v.minor == 0 + assert v.patch == 0 + assert v.released is True + assert v.release_date == "2020-01-01" + + +def test_issue(fake_jira): + i = Issue("ARROW-1234", type='Bug', summary="title") + assert i.key == "ARROW-1234" + assert i.type == "Bug" + assert i.summary == "title" + assert i.project == "ARROW" + assert i.number == 1234 + + i = Issue("PARQUET-1111", type='Improvement', summary="another title") + assert i.key == "PARQUET-1111" + assert i.type == "Improvement" + assert i.summary == "another title" + assert i.project == "PARQUET" + assert i.number == 1111 + + fake_jira_issue = DotDict({ + 'key': 'ARROW-2222', + 'fields': { + 'issuetype': { + 'name': 'Feature' + }, + 'summary': 'Issue title' + } + }) + i = Issue.from_jira(fake_jira_issue) + assert i.key == "ARROW-2222" + assert i.type == "Feature" + assert i.summary == "Issue title" + assert i.project == "ARROW" + assert i.number == 2222 + + +def test_commit_title(): + t = CommitTitle.parse( + "ARROW-9598: [C++][Parquet] Fix writing nullable structs" + ) + assert t.project == "ARROW" + assert t.issue == "ARROW-9598" + assert t.components == ["C++", "Parquet"] + assert t.summary == "Fix writing nullable structs" + + t = CommitTitle.parse( + "ARROW-8002: [C++][Dataset][R] Support partitioned dataset writing" + ) + assert t.project == "ARROW" + assert t.issue == "ARROW-8002" + assert t.components == ["C++", "Dataset", "R"] + assert t.summary == "Support partitioned dataset writing" + + t = CommitTitle.parse( + "ARROW-9600: [Rust][Arrow] pin older version of proc-macro2 during " + "build" + ) + assert t.project == "ARROW" + assert t.issue == "ARROW-9600" + assert t.components == ["Rust", "Arrow"] + assert t.summary == "pin older version of proc-macro2 during build" + + t = CommitTitle.parse("[Release] Update versions for 1.0.0") + assert t.project is None + assert t.issue is None + assert t.components == ["Release"] + assert t.summary == "Update versions for 1.0.0" + + t = CommitTitle.parse("[Python][Doc] Fix rst role dataset.rst (#7725)") + assert t.project is None + assert t.issue is None + assert t.components == ["Python", "Doc"] + assert t.summary == "Fix rst role dataset.rst (#7725)" + + t = CommitTitle.parse( + "PARQUET-1882: [C++] Buffered Reads should allow for 0 length" + ) + assert t.project == 'PARQUET' + assert t.issue == 'PARQUET-1882' + assert t.components == ["C++"] + assert t.summary == "Buffered Reads should allow for 0 length" + + t = CommitTitle.parse( + "ARROW-9340 [R] Use CRAN version of decor package " + "\nsomething else\n" + "\nwhich should be truncated" + ) + assert t.project == 'ARROW' + assert t.issue == 'ARROW-9340' + assert t.components == ["R"] + assert t.summary == "Use CRAN version of decor package " + + +def test_release_basics(fake_jira): + r = Release.from_jira("1.0.0", jira=fake_jira) + assert isinstance(r, MajorRelease) + assert r.is_released is True + assert r.branch == 'master' + assert r.tag == 'apache-arrow-1.0.0' + + r = Release.from_jira("1.1.0", jira=fake_jira) + assert isinstance(r, MinorRelease) + assert r.is_released is False + assert r.branch == 'maint-1.x.x' + assert r.tag == 'apache-arrow-1.1.0' + + # minor releases before 1.0 are treated as major releases + r = Release.from_jira("0.17.0", jira=fake_jira) + assert isinstance(r, MajorRelease) + assert r.is_released is True + assert r.branch == 'master' + assert r.tag == 'apache-arrow-0.17.0' + + r = Release.from_jira("0.17.1", jira=fake_jira) + assert isinstance(r, PatchRelease) + assert r.is_released is True + assert r.branch == 'maint-0.17.x' + assert r.tag == 'apache-arrow-0.17.1' + + +def test_previous_and_next_release(fake_jira): + r = Release.from_jira("3.0.0", jira=fake_jira) + assert isinstance(r.previous, MajorRelease) + assert r.previous.version == Version.parse("2.0.0") + with pytest.raises(ValueError, match="There is no upcoming release set"): + assert r.next + + r = Release.from_jira("2.0.0", jira=fake_jira) + assert isinstance(r.previous, MajorRelease) + assert isinstance(r.next, MajorRelease) + assert r.previous.version == Version.parse("1.0.0") + assert r.next.version == Version.parse("3.0.0") + + r = Release.from_jira("1.1.0", jira=fake_jira) + assert isinstance(r.previous, MajorRelease) + assert isinstance(r.next, MajorRelease) + assert r.previous.version == Version.parse("1.0.0") + assert r.next.version == Version.parse("2.0.0") + + r = Release.from_jira("1.0.0", jira=fake_jira) + assert isinstance(r.next, MajorRelease) + assert isinstance(r.previous, MajorRelease) + assert r.previous.version == Version.parse("0.17.0") + assert r.next.version == Version.parse("2.0.0") + + r = Release.from_jira("0.17.0", jira=fake_jira) + assert isinstance(r.previous, MajorRelease) + assert r.previous.version == Version.parse("0.16.0") + + r = Release.from_jira("0.15.2", jira=fake_jira) + assert isinstance(r.previous, PatchRelease) + assert isinstance(r.next, MajorRelease) + assert r.previous.version == Version.parse("0.15.1") + assert r.next.version == Version.parse("0.16.0") + + r = Release.from_jira("0.15.1", jira=fake_jira) + assert isinstance(r.previous, MajorRelease) + assert isinstance(r.next, PatchRelease) + assert r.previous.version == Version.parse("0.15.0") + assert r.next.version == Version.parse("0.15.2") + + +def test_release_issues(fake_jira): + # major release issues + r = Release.from_jira("1.0.0", jira=fake_jira) + assert r.issues.keys() == set([ + "ARROW-300", + "ARROW-4427", + "ARROW-5035", + "ARROW-8473", + "ARROW-8472", + "ARROW-8471", + "ARROW-8974", + "ARROW-8973" + ]) + # minor release issues + r = Release.from_jira("0.17.0", jira=fake_jira) + assert r.issues.keys() == set([ + "ARROW-2882", + "ARROW-2587", + "ARROW-2447", + "ARROW-2255", + "ARROW-1907", + "ARROW-1636", + ]) + # patch release issues + r = Release.from_jira("1.0.1", jira=fake_jira) + assert r.issues.keys() == set([ + "ARROW-9684", + "ARROW-9667", + "ARROW-9659", + "ARROW-9644", + "ARROW-9643", + "ARROW-9609", + "ARROW-9606" + ]) + + +@pytest.mark.parametrize(('version', 'ncommits'), [ + ("1.0.0", 771), + ("0.17.1", 27), + ("0.17.0", 569), + ("0.15.1", 41) +]) +def test_release_commits(fake_jira, version, ncommits): + r = Release.from_jira(version, jira=fake_jira) + assert len(r.commits) == ncommits + for c in r.commits: + assert isinstance(c, Commit) + assert isinstance(c.title, CommitTitle) + assert c.url.endswith(c.hexsha) + + +def test_maintenance_patch_selection(fake_jira): + r = Release.from_jira("0.17.1", jira=fake_jira) + + shas_to_pick = [ + c.hexsha for c in r.commits_to_pick(exclude_already_applied=False) + ] + expected = [ + '8939b4bd446ee406d5225c79d563a27d30fd7d6d', + 'bcef6c95a324417e85e0140f9745d342cd8784b3', + '6002ec388840de5622e39af85abdc57a2cccc9b2', + '9123dadfd123bca7af4eaa9455f5b0d1ca8b929d', + ] + assert shas_to_pick == expected diff --git a/src/arrow/dev/archery/archery/tests/test_testing.py b/src/arrow/dev/archery/archery/tests/test_testing.py new file mode 100644 index 000000000..117b9288d --- /dev/null +++ b/src/arrow/dev/archery/archery/tests/test_testing.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess + +import pytest + +from archery.testing import PartialEnv, assert_subprocess_calls + + +def test_partial_env(): + assert PartialEnv(a=1, b=2) == {'a': 1, 'b': 2, 'c': 3} + assert PartialEnv(a=1) == {'a': 1, 'b': 2, 'c': 3} + assert PartialEnv(a=1, b=2) == {'a': 1, 'b': 2} + assert PartialEnv(a=1, b=2) != {'b': 2, 'c': 3} + assert PartialEnv(a=1, b=2) != {'a': 1, 'c': 3} + + +def test_assert_subprocess_calls(): + expected_calls = [ + "echo Hello", + ["echo", "World"] + ] + with assert_subprocess_calls(expected_calls): + subprocess.run(['echo', 'Hello']) + subprocess.run(['echo', 'World']) + + expected_env = PartialEnv( + CUSTOM_ENV_A='a', + CUSTOM_ENV_C='c' + ) + with assert_subprocess_calls(expected_calls, env=expected_env): + env = { + 'CUSTOM_ENV_A': 'a', + 'CUSTOM_ENV_B': 'b', + 'CUSTOM_ENV_C': 'c' + } + subprocess.run(['echo', 'Hello'], env=env) + subprocess.run(['echo', 'World'], env=env) + + with pytest.raises(AssertionError): + with assert_subprocess_calls(expected_calls, env=expected_env): + env = { + 'CUSTOM_ENV_B': 'b', + 'CUSTOM_ENV_C': 'c' + } + subprocess.run(['echo', 'Hello'], env=env) + subprocess.run(['echo', 'World'], env=env) diff --git a/src/arrow/dev/archery/archery/utils/__init__.py b/src/arrow/dev/archery/archery/utils/__init__.py new file mode 100644 index 000000000..13a83393a --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/arrow/dev/archery/archery/utils/cache.py b/src/arrow/dev/archery/archery/utils/cache.py new file mode 100644 index 000000000..d92c5f32e --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/cache.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path +import os +from urllib.request import urlopen + +from .logger import logger + +ARCHERY_CACHE_DIR = Path.home() / ".cache" / "archery" + + +class Cache: + """ Cache stores downloaded objects, notably apache-rat.jar. """ + + def __init__(self, path=ARCHERY_CACHE_DIR): + self.root = path + + if not path.exists(): + os.makedirs(path) + + def key_path(self, key): + """ Return the full path of a key. """ + return self.root/key + + def get(self, key): + """ Return the full path of a key if cached, None otherwise. """ + path = self.key_path(key) + return path if path.exists() else None + + def delete(self, key): + """ Remove a key (and the file) from the cache. """ + path = self.get(key) + if path: + path.unlink() + + def get_or_insert(self, key, create): + """ + Get or Insert a key from the cache. If the key is not found, the + `create` closure will be evaluated. + + The `create` closure takes a single parameter, the path where the + object should be store. The file should only be created upon success. + """ + path = self.key_path(key) + + if not path.exists(): + create(path) + + return path + + def get_or_insert_from_url(self, key, url): + """ + Get or Insert a key from the cache. If the key is not found, the file + is downloaded from `url`. + """ + def download(path): + """ Tiny wrapper that download a file and save as key. """ + logger.debug("Downloading {} as {}".format(url, path)) + conn = urlopen(url) + # Ensure the download is completed before writing to disks. + content = conn.read() + with open(path, "wb") as path_fd: + path_fd.write(content) + + return self.get_or_insert(key, download) diff --git a/src/arrow/dev/archery/archery/utils/cli.py b/src/arrow/dev/archery/archery/utils/cli.py new file mode 100644 index 000000000..701abe925 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/cli.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib + +import click + +from .source import ArrowSources, InvalidArrowSource + + +class ArrowBool(click.types.BoolParamType): + """ + ArrowBool supports the 'ON' and 'OFF' values on top of the values + supported by BoolParamType. This is convenient to port script which exports + CMake options variables. + """ + name = "boolean" + + def convert(self, value, param, ctx): + if isinstance(value, str): + lowered = value.lower() + if lowered == "on": + return True + elif lowered == "off": + return False + + return super().convert(value, param, ctx) + + +def validate_arrow_sources(ctx, param, src): + """ + Ensure a directory contains Arrow cpp sources. + """ + try: + return ArrowSources.find(src) + except InvalidArrowSource as e: + raise click.BadParameter(str(e)) + + +def add_optional_command(name, module, function, parent): + try: + module = importlib.import_module(module, package="archery") + command = getattr(module, function) + except ImportError as exc: + error_message = exc.name + + @parent.command( + name, + context_settings={ + "allow_extra_args": True, + "ignore_unknown_options": True, + } + ) + def command(): + raise click.ClickException( + f"Couldn't import command `{name}` due to {error_message}" + ) + else: + parent.add_command(command) diff --git a/src/arrow/dev/archery/archery/utils/cmake.py b/src/arrow/dev/archery/archery/utils/cmake.py new file mode 100644 index 000000000..f93895b1a --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/cmake.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +from shutil import rmtree, which + +from .command import Command, default_bin + + +class CMake(Command): + def __init__(self, cmake_bin=None): + self.bin = default_bin(cmake_bin, "cmake") + + @staticmethod + def default_generator(): + """ Infer default generator. + + Gives precedence to ninja if there exists an executable named `ninja` + in the search path. + """ + found_ninja = which("ninja") + return "Ninja" if found_ninja else "Unix Makefiles" + + +cmake = CMake() + + +class CMakeDefinition: + """ CMakeDefinition captures the cmake invocation arguments. + + It allows creating build directories with the same definition, e.g. + ``` + build_1 = cmake_def.build("/tmp/build-1") + build_2 = cmake_def.build("/tmp/build-2") + + ... + + build1.all() + build2.all() + """ + + def __init__(self, source, build_type="release", generator=None, + definitions=None, env=None): + """ Initialize a CMakeDefinition + + Parameters + ---------- + source : str + Source directory where the top-level CMakeLists.txt is + located. This is usually the root of the project. + generator : str, optional + definitions: list(str), optional + env : dict(str,str), optional + Environment to use when invoking cmake. This can be required to + work around cmake deficiencies, e.g. CC and CXX. + """ + self.source = os.path.abspath(source) + self.build_type = build_type + self.generator = generator if generator else cmake.default_generator() + self.definitions = definitions if definitions else [] + self.env = env + + @property + def arguments(self): + """" Return the arguments to cmake invocation. """ + arguments = [ + "-G{}".format(self.generator), + ] + self.definitions + [ + self.source + ] + return arguments + + def build(self, build_dir, force=False, cmd_kwargs=None, **kwargs): + """ Invoke cmake into a build directory. + + Parameters + ---------- + build_dir : str + Directory in which the CMake build will be instantiated. + force : bool + If the build folder exists, delete it before. Otherwise if it's + present, an error will be returned. + """ + if os.path.exists(build_dir): + # Extra safety to ensure we're deleting a build folder. + if not CMakeBuild.is_build_dir(build_dir): + raise FileExistsError( + "{} is not a cmake build".format(build_dir) + ) + if not force: + raise FileExistsError( + "{} exists use force=True".format(build_dir) + ) + rmtree(build_dir) + + os.mkdir(build_dir) + + cmd_kwargs = cmd_kwargs if cmd_kwargs else {} + cmake(*self.arguments, cwd=build_dir, env=self.env, **cmd_kwargs) + return CMakeBuild(build_dir, self.build_type, definition=self, + **kwargs) + + def __repr__(self): + return "CMakeDefinition[source={}]".format(self.source) + + +CMAKE_BUILD_TYPE_RE = re.compile("CMAKE_BUILD_TYPE:STRING=([a-zA-Z]+)") + + +class CMakeBuild(CMake): + """ CMakeBuild represents a build directory initialized by cmake. + + The build instance can be used to build/test/install. It alleviates the + user to know which generator is used. + """ + + def __init__(self, build_dir, build_type, definition=None): + """ Initialize a CMakeBuild. + + The caller must ensure that cmake was invoked in the build directory. + + Parameters + ---------- + definition : CMakeDefinition + The definition to build from. + build_dir : str + The build directory to setup into. + """ + assert CMakeBuild.is_build_dir(build_dir) + super().__init__() + self.build_dir = os.path.abspath(build_dir) + self.build_type = build_type + self.definition = definition + + @property + def binaries_dir(self): + return os.path.join(self.build_dir, self.build_type) + + def run(self, *argv, verbose=False, **kwargs): + cmake_args = ["--build", self.build_dir, "--"] + extra = [] + if verbose: + extra.append("-v" if self.bin.endswith("ninja") else "VERBOSE=1") + # Commands must be ran under the build directory + return super().run(*cmake_args, *extra, + *argv, **kwargs, cwd=self.build_dir) + + def all(self): + return self.run("all") + + def clean(self): + return self.run("clean") + + def install(self): + return self.run("install") + + def test(self): + return self.run("test") + + @staticmethod + def is_build_dir(path): + """ Indicate if a path is CMake build directory. + + This method only checks for the existence of paths and does not do any + validation whatsoever. + """ + cmake_cache = os.path.join(path, "CMakeCache.txt") + cmake_files = os.path.join(path, "CMakeFiles") + return os.path.exists(cmake_cache) and os.path.exists(cmake_files) + + @staticmethod + def from_path(path): + """ Instantiate a CMakeBuild from a path. + + This is used to recover from an existing physical directory (created + with or without CMakeBuild). + + Note that this method is not idempotent as the original definition will + be lost. Only build_type is recovered. + """ + if not CMakeBuild.is_build_dir(path): + raise ValueError("Not a valid CMakeBuild path: {}".format(path)) + + build_type = None + # Infer build_type by looking at CMakeCache.txt and looking for a magic + # definition + cmake_cache_path = os.path.join(path, "CMakeCache.txt") + with open(cmake_cache_path, "r") as cmake_cache: + candidates = CMAKE_BUILD_TYPE_RE.findall(cmake_cache.read()) + build_type = candidates[0].lower() if candidates else "release" + + return CMakeBuild(path, build_type) + + def __repr__(self): + return ("CMakeBuild[" + "build = {}," + "build_type = {}," + "definition = {}]".format(self.build_dir, + self.build_type, + self.definition)) diff --git a/src/arrow/dev/archery/archery/utils/command.py b/src/arrow/dev/archery/archery/utils/command.py new file mode 100644 index 000000000..f655e2ef2 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/command.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import shlex +import shutil +import subprocess + +from .logger import logger, ctx + + +def default_bin(name, default): + assert(default) + env_name = "ARCHERY_{0}_BIN".format(default.upper()) + return name if name else os.environ.get(env_name, default) + + +# Decorator running a command and returning stdout +class capture_stdout: + def __init__(self, strip=False, listify=False): + self.strip = strip + self.listify = listify + + def __call__(self, f): + def strip_it(x): + return x.strip() if self.strip else x + + def list_it(x): + return x.decode('utf-8').splitlines() if self.listify else x + + def wrapper(*argv, **kwargs): + # Ensure stdout is captured + kwargs["stdout"] = subprocess.PIPE + return list_it(strip_it(f(*argv, **kwargs).stdout)) + return wrapper + + +class Command: + """ + A runnable command. + + Class inheriting from the Command class must provide the bin + property/attribute. + """ + + def __init__(self, bin): + self.bin = bin + + def run(self, *argv, **kwargs): + assert hasattr(self, "bin") + invocation = shlex.split(self.bin) + invocation.extend(argv) + + for key in ["stdout", "stderr"]: + # Preserve caller intention, otherwise silence + if key not in kwargs and ctx.quiet: + kwargs[key] = subprocess.PIPE + + # Prefer safe by default + if "check" not in kwargs: + kwargs["check"] = True + + logger.debug("Executing `{}`".format(invocation)) + return subprocess.run(invocation, **kwargs) + + @property + def available(self): + """ + Indicate if the command binary is found in PATH. + """ + binary = shlex.split(self.bin)[0] + return shutil.which(binary) is not None + + def __call__(self, *argv, **kwargs): + return self.run(*argv, **kwargs) + + +class CommandStackMixin: + def run(self, *argv, **kwargs): + stacked_args = self.argv + argv + return super(CommandStackMixin, self).run(*stacked_args, **kwargs) + + +class Bash(Command): + def __init__(self, bash_bin=None): + self.bin = default_bin(bash_bin, "bash") diff --git a/src/arrow/dev/archery/archery/utils/git.py b/src/arrow/dev/archery/archery/utils/git.py new file mode 100644 index 000000000..798bc5d70 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/git.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .command import Command, capture_stdout, default_bin +from ..compat import _stringify_path + + +# Decorator prepending argv with the git sub-command found with the method +# name. +def git_cmd(fn): + # function name is the subcommand + sub_cmd = fn.__name__.replace("_", "-") + + def wrapper(self, *argv, **kwargs): + return fn(self, sub_cmd, *argv, **kwargs) + return wrapper + + +class Git(Command): + def __init__(self, git_bin=None): + self.bin = default_bin(git_bin, "git") + + def run_cmd(self, cmd, *argv, git_dir=None, **kwargs): + """ Inject flags before sub-command in argv. """ + opts = [] + if git_dir is not None: + opts.extend(["-C", _stringify_path(git_dir)]) + + return self.run(*opts, cmd, *argv, **kwargs) + + @capture_stdout(strip=False) + @git_cmd + def archive(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @git_cmd + def clone(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @git_cmd + def fetch(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @git_cmd + def checkout(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + def dirty(self, **kwargs): + return len(self.status("--short", **kwargs)) > 0 + + @git_cmd + def log(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @capture_stdout(strip=True, listify=True) + @git_cmd + def ls_files(self, *argv, listify=False, **kwargs): + stdout = self.run_cmd(*argv, **kwargs) + return stdout + + @capture_stdout(strip=True) + @git_cmd + def rev_parse(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @capture_stdout(strip=True) + @git_cmd + def status(self, *argv, **kwargs): + return self.run_cmd(*argv, **kwargs) + + @capture_stdout(strip=True) + def head(self, **kwargs): + """ Return commit pointed by HEAD. """ + return self.rev_parse("HEAD", **kwargs) + + @capture_stdout(strip=True) + def current_branch(self, **kwargs): + return self.rev_parse("--abbrev-ref", "HEAD", **kwargs) + + def repository_root(self, git_dir=None, **kwargs): + """ Locates the repository's root path from a subdirectory. """ + stdout = self.rev_parse("--show-toplevel", git_dir=git_dir, **kwargs) + return stdout.decode('utf-8') + + +git = Git() diff --git a/src/arrow/dev/archery/archery/utils/lint.py b/src/arrow/dev/archery/archery/utils/lint.py new file mode 100644 index 000000000..d95bfeea3 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/lint.py @@ -0,0 +1,429 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fnmatch +import gzip +import os +from pathlib import Path + +import click + +from .command import Bash, Command, default_bin +from .cmake import CMake +from .git import git +from .logger import logger +from ..lang.cpp import CppCMakeDefinition, CppConfiguration +from ..lang.python import Autopep8, Flake8, NumpyDoc +from .rat import Rat, exclusion_from_globs +from .tmpdir import tmpdir + + +_archery_install_msg = ( + "Please install archery using: `pip install -e dev/archery[lint]`. " +) + + +class LintValidationException(Exception): + pass + + +class LintResult: + def __init__(self, success, reason=None): + self.success = success + + def ok(self): + if not self.success: + raise LintValidationException + + @staticmethod + def from_cmd(command_result): + return LintResult(command_result.returncode == 0) + + +def cpp_linter(src, build_dir, clang_format=True, cpplint=True, + clang_tidy=False, iwyu=False, iwyu_all=False, + fix=False): + """ Run clang-format, cpplint and clang-tidy on cpp/ codebase. """ + logger.info("Running C++ linters") + + cmake = CMake() + if not cmake.available: + logger.error("cpp linter requested but cmake binary not found.") + return + + # A cmake build directory is required to populate `compile_commands.json` + # which in turn is required by clang-tidy. It also provides a convenient + # way to hide clang-format/clang-tidy invocation via the Generate + # (ninja/make) targets. + + # ARROW_LINT_ONLY exits early but ignore building compile_command.json + lint_only = not (iwyu or clang_tidy) + cmake_args = {"with_python": False, "with_lint_only": lint_only} + cmake_def = CppCMakeDefinition(src.cpp, CppConfiguration(**cmake_args)) + + build = cmake_def.build(build_dir) + if clang_format: + target = "format" if fix else "check-format" + yield LintResult.from_cmd(build.run(target, check=False)) + + if cpplint: + yield LintResult.from_cmd(build.run("lint", check=False)) + yield LintResult.from_cmd(build.run("lint_cpp_cli", check=False)) + + if clang_tidy: + yield LintResult.from_cmd(build.run("check-clang-tidy", check=False)) + + if iwyu: + if iwyu_all: + iwyu_cmd = "iwyu-all" + else: + iwyu_cmd = "iwyu" + yield LintResult.from_cmd(build.run(iwyu_cmd, check=False)) + + +class CMakeFormat(Command): + + def __init__(self, paths, cmake_format_bin=None): + self.check_version() + self.bin = default_bin(cmake_format_bin, "cmake-format") + self.paths = paths + + @classmethod + def from_patterns(cls, base_path, include_patterns, exclude_patterns): + paths = { + str(path.as_posix()) + for pattern in include_patterns + for path in base_path.glob(pattern) + } + for pattern in exclude_patterns: + pattern = (base_path / pattern).as_posix() + paths -= set(fnmatch.filter(paths, str(pattern))) + return cls(paths) + + @staticmethod + def check_version(): + try: + # cmake_format is part of the cmakelang package + import cmakelang + except ImportError: + raise ImportError( + + ) + # pin a specific version of cmake_format, must be updated in setup.py + if cmakelang.__version__ != "0.6.13": + raise LintValidationException( + f"Wrong version of cmake_format is detected. " + f"{_archery_install_msg}" + ) + + def check(self): + return self.run("-l", "error", "--check", *self.paths, check=False) + + def fix(self): + return self.run("--in-place", *self.paths, check=False) + + +def cmake_linter(src, fix=False): + """ + Run cmake-format on all CMakeFiles.txt + """ + logger.info("Running cmake-format linters") + + cmake_format = CMakeFormat.from_patterns( + src.path, + include_patterns=[ + 'ci/**/*.cmake', + 'cpp/CMakeLists.txt', + 'cpp/src/**/CMakeLists.txt', + 'cpp/cmake_modules/*.cmake', + 'go/**/CMakeLists.txt', + 'java/**/CMakeLists.txt', + 'matlab/**/CMakeLists.txt', + 'python/CMakeLists.txt', + ], + exclude_patterns=[ + 'cpp/cmake_modules/FindNumPy.cmake', + 'cpp/cmake_modules/FindPythonLibsNew.cmake', + 'cpp/cmake_modules/UseCython.cmake', + 'cpp/src/arrow/util/config.h.cmake', + ] + ) + method = cmake_format.fix if fix else cmake_format.check + + yield LintResult.from_cmd(method()) + + +def python_linter(src, fix=False): + """Run Python linters on python/pyarrow, python/examples, setup.py + and dev/. """ + setup_py = os.path.join(src.python, "setup.py") + setup_cfg = os.path.join(src.python, "setup.cfg") + + logger.info("Running Python formatter (autopep8)") + + autopep8 = Autopep8() + if not autopep8.available: + logger.error( + "Python formatter requested but autopep8 binary not found. " + f"{_archery_install_msg}") + return + + # Gather files for autopep8 + patterns = ["python/pyarrow/**/*.py", + "python/pyarrow/**/*.pyx", + "python/pyarrow/**/*.pxd", + "python/pyarrow/**/*.pxi", + "python/examples/**/*.py", + "dev/archery/**/*.py"] + files = [setup_py] + for pattern in patterns: + files += list(map(str, Path(src.path).glob(pattern))) + + args = ['--global-config', setup_cfg, '--ignore-local-config'] + if fix: + args += ['-j0', '--in-place'] + args += sorted(files) + yield LintResult.from_cmd(autopep8(*args)) + else: + # XXX `-j0` doesn't work well with `--exit-code`, so instead + # we capture the diff and check whether it's empty + # (https://github.com/hhatto/autopep8/issues/543) + args += ['-j0', '--diff'] + args += sorted(files) + diff = autopep8.run_captured(*args) + if diff: + print(diff.decode('utf8')) + yield LintResult(success=False) + else: + yield LintResult(success=True) + + # Run flake8 after autopep8 (the latter may have modified some files) + logger.info("Running Python linter (flake8)") + + flake8 = Flake8() + if not flake8.available: + logger.error( + "Python linter requested but flake8 binary not found. " + f"{_archery_install_msg}") + return + + flake8_exclude = ['.venv*'] + + yield LintResult.from_cmd( + flake8("--extend-exclude=" + ','.join(flake8_exclude), + setup_py, src.pyarrow, os.path.join(src.python, "examples"), + src.dev, check=False)) + config = os.path.join(src.python, ".flake8.cython") + yield LintResult.from_cmd( + flake8("--config=" + config, src.pyarrow, check=False)) + + +def python_numpydoc(symbols=None, allow_rules=None, disallow_rules=None): + """Run numpydoc linter on python. + + Pyarrow must be available for import. + """ + logger.info("Running Python docstring linters") + # by default try to run on all pyarrow package + symbols = symbols or { + 'pyarrow', + 'pyarrow.compute', + 'pyarrow.csv', + 'pyarrow.dataset', + 'pyarrow.feather', + 'pyarrow.flight', + 'pyarrow.fs', + 'pyarrow.gandiva', + 'pyarrow.ipc', + 'pyarrow.json', + 'pyarrow.orc', + 'pyarrow.parquet', + 'pyarrow.plasma', + 'pyarrow.types', + } + try: + numpydoc = NumpyDoc(symbols) + except RuntimeError as e: + logger.error(str(e)) + yield LintResult(success=False) + return + + results = numpydoc.validate( + # limit the validation scope to the pyarrow package + from_package='pyarrow', + allow_rules=allow_rules, + disallow_rules=disallow_rules + ) + + if len(results) == 0: + yield LintResult(success=True) + return + + number_of_violations = 0 + for obj, result in results: + errors = result['errors'] + + # inspect doesn't play nice with cython generated source code, + # to use a hacky way to represent a proper __qualname__ + doc = getattr(obj, '__doc__', '') + name = getattr(obj, '__name__', '') + qualname = getattr(obj, '__qualname__', '') + module = getattr(obj, '__module__', '') + instance = getattr(obj, '__self__', '') + if instance: + klass = instance.__class__.__name__ + else: + klass = '' + + try: + cython_signature = doc.splitlines()[0] + except Exception: + cython_signature = '' + + desc = '.'.join(filter(None, [module, klass, qualname or name])) + + click.echo() + click.echo(click.style(desc, bold=True, fg='yellow')) + if cython_signature: + qualname_with_signature = '.'.join([module, cython_signature]) + click.echo( + click.style( + '-> {}'.format(qualname_with_signature), + fg='yellow' + ) + ) + + for error in errors: + number_of_violations += 1 + click.echo('{}: {}'.format(*error)) + + msg = 'Total number of docstring violations: {}'.format( + number_of_violations + ) + click.echo() + click.echo(click.style(msg, fg='red')) + + yield LintResult(success=False) + + +def rat_linter(src, root): + """Run apache-rat license linter.""" + logger.info("Running apache-rat linter") + + if src.git_dirty: + logger.warn("Due to the usage of git-archive, uncommitted files will" + " not be checked for rat violations. ") + + exclusion = exclusion_from_globs( + os.path.join(src.dev, "release", "rat_exclude_files.txt")) + + # Creates a git-archive of ArrowSources, apache-rat expects a gzip + # compressed tar archive. + archive_path = os.path.join(root, "apache-arrow.tar.gz") + src.archive(archive_path, compressor=gzip.compress) + report = Rat().report(archive_path) + + violations = list(report.validate(exclusion=exclusion)) + for violation in violations: + print("apache-rat license violation: {}".format(violation)) + + yield LintResult(len(violations) == 0) + + +def r_linter(src): + """Run R linter.""" + logger.info("Running R linter") + r_lint_sh = os.path.join(src.r, "lint.sh") + yield LintResult.from_cmd(Bash().run(r_lint_sh, check=False)) + + +class Hadolint(Command): + def __init__(self, hadolint_bin=None): + self.bin = default_bin(hadolint_bin, "hadolint") + + +def is_docker_image(path): + dirname = os.path.dirname(path) + filename = os.path.basename(path) + + excluded = dirname.startswith( + "dev") or dirname.startswith("python/manylinux") + + return filename.startswith("Dockerfile") and not excluded + + +def docker_linter(src): + """Run Hadolint docker linter.""" + logger.info("Running Docker linter") + + hadolint = Hadolint() + + if not hadolint.available: + logger.error( + "hadolint linter requested but hadolint binary not found.") + return + + for path in git.ls_files(git_dir=src.path): + if is_docker_image(path): + yield LintResult.from_cmd(hadolint.run(path, check=False, + cwd=src.path)) + + +def linter(src, fix=False, *, clang_format=False, cpplint=False, + clang_tidy=False, iwyu=False, iwyu_all=False, + python=False, numpydoc=False, cmake_format=False, rat=False, + r=False, docker=False): + """Run all linters.""" + with tmpdir(prefix="arrow-lint-") as root: + build_dir = os.path.join(root, "cpp-build") + + # Linters yield LintResult without raising exceptions on failure. + # This allows running all linters in one pass and exposing all + # errors to the user. + results = [] + + if clang_format or cpplint or clang_tidy or iwyu: + results.extend(cpp_linter(src, build_dir, + clang_format=clang_format, + cpplint=cpplint, + clang_tidy=clang_tidy, + iwyu=iwyu, + iwyu_all=iwyu_all, + fix=fix)) + + if python: + results.extend(python_linter(src, fix=fix)) + + if numpydoc: + results.extend(python_numpydoc()) + + if cmake_format: + results.extend(cmake_linter(src, fix=fix)) + + if rat: + results.extend(rat_linter(src, root)) + + if r: + results.extend(r_linter(src)) + + if docker: + results.extend(docker_linter(src)) + + # Raise error if one linter failed, ensuring calling code can exit with + # non-zero. + for result in results: + result.ok() diff --git a/src/arrow/dev/archery/archery/utils/logger.py b/src/arrow/dev/archery/archery/utils/logger.py new file mode 100644 index 000000000..9d0feda88 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/logger.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging + +""" Global logger. """ +logger = logging.getLogger("archery") + + +class LoggingContext: + def __init__(self, quiet=False): + self.quiet = quiet + + +ctx = LoggingContext() diff --git a/src/arrow/dev/archery/archery/utils/maven.py b/src/arrow/dev/archery/archery/utils/maven.py new file mode 100644 index 000000000..96a3bf5bd --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/maven.py @@ -0,0 +1,204 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from .command import Command, default_bin + + +class Maven(Command): + def __init__(self, maven_bin=None): + self.bin = default_bin(maven_bin, "mvn") + + +maven = Maven() + + +class MavenDefinition: + """ MavenDefinition captures the maven invocation arguments. + + It allows creating build directories with the same definition, e.g. + ``` + build_1 = maven_def.build("/tmp/build-1") + build_2 = maven_def.build("/tmp/build-2") + + ... + + build1.install() + build2.install() + """ + + def __init__(self, source, build_definitions=None, + benchmark_definitions=None, env=None): + """ Initialize a MavenDefinition + + Parameters + ---------- + source : str + Source directory where the top-level pom.xml is + located. This is usually the root of the project. + build_definitions: list(str), optional + benchmark_definitions: list(str), optional + """ + self.source = os.path.abspath(source) + self.build_definitions = build_definitions if build_definitions else [] + self.benchmark_definitions =\ + benchmark_definitions if benchmark_definitions else [] + self.env = env + + @property + def build_arguments(self): + """" Return the arguments to maven invocation for build. """ + arguments = self.build_definitions + [ + "-B", "-DskipTests", "-Drat.skip=true", + "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer." + "Slf4jMavenTransferListener=warn", + "-T", "2C", "install" + ] + return arguments + + def build(self, build_dir, force=False, cmd_kwargs=None, **kwargs): + """ Invoke maven into a build directory. + + Parameters + ---------- + build_dir : str + Directory in which the Maven build will be instantiated. + force : bool + not used now + """ + if os.path.exists(build_dir): + # Extra safety to ensure we're deleting a build folder. + if not MavenBuild.is_build_dir(build_dir): + raise FileExistsError( + "{} is not a maven build".format(build_dir) + ) + + cmd_kwargs = cmd_kwargs if cmd_kwargs else {} + assert MavenBuild.is_build_dir(build_dir) + maven(*self.build_arguments, cwd=build_dir, env=self.env, **cmd_kwargs) + return MavenBuild(build_dir, definition=self, **kwargs) + + @property + def list_arguments(self): + """" Return the arguments to maven invocation for list """ + arguments = [ + "-Dskip.perf.benchmarks=false", "-Dbenchmark.list=-lp", "install" + ] + return arguments + + @property + def benchmark_arguments(self): + """" Return the arguments to maven invocation for benchmark """ + arguments = self.benchmark_definitions + [ + "-Dskip.perf.benchmarks=false", "-Dbenchmark.fork=1", + "-Dbenchmark.jvmargs=\"-Darrow.enable_null_check_for_get=false " + "-Darrow.enable_unsafe_memory_access=true\"", + "install" + ] + return arguments + + def __repr__(self): + return "MavenDefinition[source={}]".format(self.source) + + +class MavenBuild(Maven): + """ MavenBuild represents a build directory initialized by maven. + + The build instance can be used to build/test/install. It alleviates the + user to know which generator is used. + """ + + def __init__(self, build_dir, definition=None): + """ Initialize a MavenBuild. + + The caller must ensure that maven was invoked in the build directory. + + Parameters + ---------- + definition : MavenDefinition + The definition to build from. + build_dir : str + The build directory to setup into. + """ + assert MavenBuild.is_build_dir(build_dir) + super().__init__() + self.build_dir = os.path.abspath(build_dir) + self.definition = definition + + @property + def binaries_dir(self): + return self.build_dir + + def run(self, *argv, verbose=False, cwd=None, **kwargs): + extra = [] + if verbose: + extra.append("-X") + if cwd is None: + cwd = self.build_dir + # Commands must be ran under the directory where pom.xml exists + return super().run(*extra, *argv, **kwargs, cwd=cwd) + + def build(self, *argv, verbose=False, **kwargs): + definition_args = self.definition.build_arguments + cwd = self.binaries_dir + return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd, + env=self.definition.env, **kwargs) + + def list(self, *argv, verbose=False, **kwargs): + definition_args = self.definition.list_arguments + cwd = self.binaries_dir + "/performance" + return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd, + env=self.definition.env, **kwargs) + + def benchmark(self, *argv, verbose=False, **kwargs): + definition_args = self.definition.benchmark_arguments + cwd = self.binaries_dir + "/performance" + return self.run(*argv, *definition_args, verbose=verbose, cwd=cwd, + env=self.definition.env, **kwargs) + + @staticmethod + def is_build_dir(path): + """ Indicate if a path is Maven top directory. + + This method only checks for the existence of paths and does not do any + validation whatsoever. + """ + pom_xml = os.path.join(path, "pom.xml") + performance_dir = os.path.join(path, "performance") + return os.path.exists(pom_xml) and os.path.isdir(performance_dir) + + @staticmethod + def from_path(path): + """ Instantiate a Maven from a path. + + This is used to recover from an existing physical directory (created + with or without Maven). + + Note that this method is not idempotent as the original definition will + be lost. + """ + if not MavenBuild.is_build_dir(path): + raise ValueError("Not a valid MavenBuild path: {}".format(path)) + + return MavenBuild(path, definition=None) + + def __repr__(self): + return ("MavenBuild[" + "build = {}," + "definition = {}]".format(self.build_dir, + self.definition)) diff --git a/src/arrow/dev/archery/archery/utils/rat.py b/src/arrow/dev/archery/archery/utils/rat.py new file mode 100644 index 000000000..e7fe19a7e --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/rat.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fnmatch +import re +from xml.etree import ElementTree + +from ..lang.java import Jar +from .cache import Cache +from .command import capture_stdout + +RAT_VERSION = 0.13 +RAT_JAR_FILENAME = "apache-rat-{}.jar".format(RAT_VERSION) +RAT_URL_ = "https://repo1.maven.org/maven2/org/apache/rat/apache-rat" +RAT_URL = "/".join([RAT_URL_, str(RAT_VERSION), RAT_JAR_FILENAME]) + + +class Rat(Jar): + def __init__(self): + jar = Cache().get_or_insert_from_url(RAT_JAR_FILENAME, RAT_URL) + Jar.__init__(self, jar) + + @capture_stdout(strip=False) + def run_report(self, archive_path, **kwargs): + return self.run("--xml", archive_path, **kwargs) + + def report(self, archive_path, **kwargs): + return RatReport(self.run_report(archive_path, **kwargs)) + + +def exclusion_from_globs(exclusions_path): + with open(exclusions_path, 'r') as exclusions_fd: + exclusions = [e.strip() for e in exclusions_fd] + return lambda path: any([fnmatch.fnmatch(path, e) for e in exclusions]) + + +class RatReport: + def __init__(self, xml): + self.xml = xml + self.tree = ElementTree.fromstring(xml) + + def __repr__(self): + return "RatReport({})".format(self.xml) + + def validate(self, exclusion=None): + for r in self.tree.findall('resource'): + approvals = r.findall('license-approval') + if not approvals or approvals[0].attrib['name'] == 'true': + continue + + clean_name = re.sub('^[^/]+/', '', r.attrib['name']) + + if exclusion and exclusion(clean_name): + continue + + yield clean_name diff --git a/src/arrow/dev/archery/archery/utils/report.py b/src/arrow/dev/archery/archery/utils/report.py new file mode 100644 index 000000000..6c7587ddd --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/report.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABCMeta, abstractmethod +import datetime + +import jinja2 + + +def markdown_escape(s): + for char in ('*', '#', '_', '~', '`', '>'): + s = s.replace(char, '\\' + char) + return s + + +class Report(metaclass=ABCMeta): + + def __init__(self, **kwargs): + for field in self.fields: + if field not in kwargs: + raise ValueError('Missing keyword argument {}'.format(field)) + self._data = kwargs + + def __getattr__(self, key): + return self._data[key] + + @abstractmethod + def fields(self): + pass + + @property + @abstractmethod + def templates(self): + pass + + +class JinjaReport(Report): + + def __init__(self, **kwargs): + self.env = jinja2.Environment( + loader=jinja2.PackageLoader('archery', 'templates') + ) + self.env.filters['md'] = markdown_escape + self.env.globals['today'] = datetime.date.today + super().__init__(**kwargs) + + def render(self, template_name): + template_path = self.templates[template_name] + template = self.env.get_template(template_path) + return template.render(**self._data) diff --git a/src/arrow/dev/archery/archery/utils/source.py b/src/arrow/dev/archery/archery/utils/source.py new file mode 100644 index 000000000..1080cb75d --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/source.py @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from pathlib import Path +import subprocess + +from .git import git + + +class InvalidArrowSource(Exception): + pass + + +class ArrowSources: + """ ArrowSources is a companion class representing a directory containing + Apache Arrow's sources. + """ + # Note that WORKSPACE is a reserved git revision name by this module to + # reference the current git workspace. In other words, this indicates to + # ArrowSources.at_revision that no cloning/checkout is required. + WORKSPACE = "WORKSPACE" + + def __init__(self, path): + """ Initialize an ArrowSources + + The caller must ensure that path is valid arrow source directory (can + be checked with ArrowSources.valid) + + Parameters + ---------- + path : src + """ + path = Path(path) + # validate by checking a specific path in the arrow source tree + if not (path / 'cpp' / 'CMakeLists.txt').exists(): + raise InvalidArrowSource( + "No Arrow C++ sources found in {}.".format(path) + ) + self.path = path + + @property + def archery(self): + """ Returns the archery directory of an Arrow sources. """ + return self.dev / "archery" + + @property + def cpp(self): + """ Returns the cpp directory of an Arrow sources. """ + return self.path / "cpp" + + @property + def dev(self): + """ Returns the dev directory of an Arrow sources. """ + return self.path / "dev" + + @property + def java(self): + """ Returns the java directory of an Arrow sources. """ + return self.path / "java" + + @property + def python(self): + """ Returns the python directory of an Arrow sources. """ + return self.path / "python" + + @property + def pyarrow(self): + """ Returns the python/pyarrow directory of an Arrow sources. """ + return self.python / "pyarrow" + + @property + def r(self): + """ Returns the r directory of an Arrow sources. """ + return self.path / "r" + + @property + def git_backed(self): + """ Indicate if the sources are backed by git. """ + return (self.path / ".git").exists() + + @property + def git_dirty(self): + """ Indicate if the sources is a dirty git directory. """ + return self.git_backed and git.dirty(git_dir=self.path) + + def archive(self, path, dereference=False, compressor=None, revision=None): + """ Saves a git archive at path. """ + if not self.git_backed: + raise ValueError("{} is not backed by git".format(self)) + + rev = revision if revision else "HEAD" + archive = git.archive("--prefix=apache-arrow/", rev, + git_dir=self.path) + + # TODO(fsaintjacques): fix dereference for + + if compressor: + archive = compressor(archive) + + with open(path, "wb") as archive_fd: + archive_fd.write(archive) + + def at_revision(self, revision, clone_dir): + """ Return a copy of the current sources for a specified git revision. + + This method may return the current object if no checkout is required. + The caller is responsible to remove the cloned repository directory. + + The user can use the special WORKSPACE token to mean the current git + workspace (no checkout performed). + + The second value of the returned tuple indicates if a clone was + performed. + + Parameters + ---------- + revision : str + Revision to checkout sources at. + clone_dir : str + Path to checkout the local clone. + """ + if not self.git_backed: + raise ValueError("{} is not backed by git".format(self)) + + if revision == ArrowSources.WORKSPACE: + return self, False + + # A local clone is required to leave the current sources intact such + # that builds depending on said sources are not invalidated (or worse + # slightly affected when re-invoking the generator). + # "--local" only works when dest dir is on same volume of source dir. + # "--shared" works even if dest dir is on different volume. + git.clone("--shared", self.path, clone_dir) + + # Revision can reference "origin/" (or any remotes) that are not found + # in the local clone. Thus, revisions are dereferenced in the source + # repository. + original_revision = git.rev_parse(revision) + + git.checkout(original_revision, git_dir=clone_dir) + + return ArrowSources(clone_dir), True + + @staticmethod + def find(path=None): + """ Infer Arrow sources directory from various method. + + The following guesses are done in order until a valid match is found: + + 1. Checks the given optional parameter. + + 2. Checks if the environment variable `ARROW_SRC` is defined and use + this. + + 3. Checks if the current working directory (cwd) is an Arrow source + directory. + + 4. Checks if this file (cli.py) is still in the original source + repository. If so, returns the relative path to the source + directory. + """ + + # Explicit via environment + env = os.environ.get("ARROW_SRC") + + # Implicit via cwd + cwd = Path.cwd() + + # Implicit via current file + try: + this = Path(__file__).parents[4] + except IndexError: + this = None + + # Implicit via git repository (if archery is installed system wide) + try: + repo = git.repository_root(git_dir=cwd) + except subprocess.CalledProcessError: + # We're not inside a git repository. + repo = None + + paths = list(filter(None, [path, env, cwd, this, repo])) + for p in paths: + try: + return ArrowSources(p) + except InvalidArrowSource: + pass + + searched_paths = "\n".join([" - {}".format(p) for p in paths]) + raise InvalidArrowSource( + "Unable to locate Arrow's source directory. " + "Searched paths are:\n{}".format(searched_paths) + ) + + def __repr__(self): + return self.path diff --git a/src/arrow/dev/archery/archery/utils/tmpdir.py b/src/arrow/dev/archery/archery/utils/tmpdir.py new file mode 100644 index 000000000..07d7355c8 --- /dev/null +++ b/src/arrow/dev/archery/archery/utils/tmpdir.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from contextlib import contextmanager +from tempfile import mkdtemp, TemporaryDirectory + + +@contextmanager +def tmpdir(preserve=False, prefix="arrow-archery-"): + if preserve: + yield mkdtemp(prefix=prefix) + else: + with TemporaryDirectory(prefix=prefix) as tmp: + yield tmp diff --git a/src/arrow/dev/archery/conftest.py b/src/arrow/dev/archery/conftest.py new file mode 100644 index 000000000..06a643bea --- /dev/null +++ b/src/arrow/dev/archery/conftest.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib + +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--enable-integration", + action="store_true", + default=False, + help="run slow tests" + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + ( + "integration: mark test as integration tests involving more " + "extensive setup (only used for crossbow at the moment)" + ) + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--enable-integration"): + return + marker = pytest.mark.skip(reason="need --enable-integration option to run") + for item in items: + if "integration" in item.keywords: + item.add_marker(marker) + + +@pytest.fixture +def load_fixture(request): + current_test_directory = pathlib.Path(request.node.fspath).parent + + def decoder(path): + with path.open('r') as fp: + if path.suffix == '.json': + import json + return json.load(fp) + elif path.suffix == '.yaml': + import yaml + return yaml.load(fp) + else: + return fp.read() + + def loader(name, decoder=decoder): + path = current_test_directory / 'fixtures' / name + return decoder(path) + + return loader diff --git a/src/arrow/dev/archery/generate_files_for_endian_test.sh b/src/arrow/dev/archery/generate_files_for_endian_test.sh new file mode 100755 index 000000000..ba3ce9f16 --- /dev/null +++ b/src/arrow/dev/archery/generate_files_for_endian_test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script generates json and arrow files of each type (e.g. primitive) for integration endian test +# Usage: generate_files_for_endian_test.sh +# ARROW_CPP_EXE_PATH : where Arrow C++ binaries can be found +# TMP_DIR : where files will be generated + +set -e + +: ${ARROW_CPP_EXE_PATH:=/arrow/cpp/build/debug/} +: ${TMP_DIR:=/tmp/arrow} + +json_dir=$TMP_DIR/arrow.$$ +mkdir -p $json_dir + +archery integration --stop-on-error --with-cpp=1 --tempdir=$json_dir + +for f in $json_dir/*.json ; do + $ARROW_CPP_EXE_PATH/arrow-json-integration-test -mode JSON_TO_ARROW -json $f -arrow ${f%.*}.arrow_file -integration true ; +done +for f in $json_dir/*.arrow_file ; do + $ARROW_CPP_EXE_PATH/arrow-file-to-stream $f > ${f%.*}.stream; +done +for f in $json_dir/*.json ; do + gzip $f ; +done +echo "The files are under $json_dir" diff --git a/src/arrow/dev/archery/requirements.txt b/src/arrow/dev/archery/requirements.txt new file mode 100644 index 000000000..0e1258adb --- /dev/null +++ b/src/arrow/dev/archery/requirements.txt @@ -0,0 +1,4 @@ +click +pygithub +python-dotenv +ruamel.yaml diff --git a/src/arrow/dev/archery/setup.py b/src/arrow/dev/archery/setup.py new file mode 100755 index 000000000..664807375 --- /dev/null +++ b/src/arrow/dev/archery/setup.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import functools +import operator +import sys +from setuptools import setup, find_packages + +if sys.version_info < (3, 6): + sys.exit('Python < 3.6 is not supported') + +# For pathlib.Path compatibility +jinja_req = 'jinja2>=2.11' + +extras = { + 'lint': ['numpydoc==1.1.0', 'autopep8', 'flake8', 'cmake_format==0.6.13'], + 'benchmark': ['pandas'], + 'docker': ['ruamel.yaml', 'python-dotenv'], + 'release': [jinja_req, 'jira', 'semver', 'gitpython'], + 'crossbow': ['github3.py', jinja_req, 'pygit2>=1.6.0', 'ruamel.yaml', + 'setuptools_scm'], + 'crossbow-upload': ['github3.py', jinja_req, 'ruamel.yaml', + 'setuptools_scm'], +} +extras['bot'] = extras['crossbow'] + ['pygithub', 'jira'] +extras['all'] = list(set(functools.reduce(operator.add, extras.values()))) + +setup( + name='archery', + version="0.1.0", + description='Apache Arrow Developers Tools', + url='http://github.com/apache/arrow', + maintainer='Arrow Developers', + maintainer_email='dev@arrow.apache.org', + packages=find_packages(), + include_package_data=True, + install_requires=['click>=7'], + tests_require=['pytest', 'responses'], + extras_require=extras, + entry_points=''' + [console_scripts] + archery=archery.cli:archery + ''' +) |