summaryrefslogtreecommitdiffstats
path: root/tools/tryselect/util/fzf.py
blob: 66894ffb67bf04c52c277979d28b669850a2b2b7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


import os
import platform
import shutil
import subprocess
import sys
from distutils.spawn import find_executable

import mozfile
import six
from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks
from mach.util import get_state_dir
from mozboot.util import http_download_and_save
from mozbuild.base import MozbuildObject
from mozterm import Terminal
from packaging.version import Version

from ..push import check_working_directory
from ..tasks import generate_tasks
from ..util.manage_estimates import (
    download_task_history_data,
    make_trimmed_taskgraph_cache,
)

terminal = Terminal()

here = os.path.abspath(os.path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here)

PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py")

FZF_MIN_VERSION = "0.20.0"
FZF_CURRENT_VERSION = "0.29.0"

# It would make more sense to have the full filename be the key; but that makes
# the line too long and ./mach lint and black can't agree about what to about that.
# You can get these from the github release, e.g.
#          https://github.com/junegunn/fzf/releases/download/0.24.1/fzf_0.24.1_checksums.txt
# However the darwin releases may not be included, so double check you have everything
FZF_CHECKSUMS = {
    "linux_armv5.tar.gz": "61d3c2aa77b977ba694836fd1134da9272bd97ee490ececaf87959b985820111",
    "linux_armv6.tar.gz": "db6b30fcbbd99ac4cf7e3ff6c5db1d3c0afcbe37d10ec3961bdc43e8c4f2e4f9",
    "linux_armv7.tar.gz": "ed86f0e91e41d2cea7960a78e3eb175dc2a5fc1510380c195d0c3559bfdc701c",
    "linux_arm64.tar.gz": "47988d8b68905541cbc26587db3ed1cfa8bc3aa8da535120abb4229b988f259e",
    "linux_amd64.tar.gz": "0106f458b933be65edb0e8f0edb9a16291a79167836fd26a77ff5496269b5e9a",
    "windows_armv5.zip": "08eaac45b3600d82608d292c23e7312696e7e11b6278b292feba25e8eb91c712",
    "windows_armv6.zip": "8b6618726a9d591a45120fddebc29f4164e01ce6639ed9aa8fc79ab03eefcfed",
    "windows_armv7.zip": "c167117b4c08f4f098446291115871ce5f14a8a8b22f0ca70e1b4342452ab5d7",
    "windows_arm64.zip": "0cda7bf68850a3e867224a05949612405e63a4421d52396c1a6c9427d4304d72",
    "windows_amd64.zip": "f0797ceee089017108c80b09086c71b8eec43d4af11ce939b78b1d5cfd202540",
    "darwin_arm64.zip": "2571b4d381f1fc691e7603bbc8113a67116da2404751ebb844818d512dd62b4b",
    "darwin_amd64.zip": "bc541e8ae0feb94efa96424bfe0b944f746db04e22f5cccfe00709925839a57f",
    "openbsd_amd64.tar.gz": "b62343827ff83949c09d5e2c8ca0c1198d05f733c9a779ec37edd840541ccdab",
    "freebsd_amd64.tar.gz": "f0367f2321c070d103589c7c7eb6a771bc7520820337a6c2fbb75be37ff783a9",
}

FZF_INSTALL_MANUALLY = """
The `mach try fuzzy` command depends on fzf. Please install it following the
appropriate instructions for your platform:

    https://github.com/junegunn/fzf#installation

Only the binary is required, if you do not wish to install the shell and
editor integrations, download the appropriate binary and put it on your $PATH:

    https://github.com/junegunn/fzf/releases
""".lstrip()

FZF_COULD_NOT_DETERMINE_PLATFORM = (
    """
Could not automatically obtain the `fzf` binary because we could not determine
your Operating System.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_COULD_NOT_DETERMINE_MACHINE = (
    """
Could not automatically obtain the `fzf` binary because we could not determine
your machine type.  It's reported as '%s' and we don't handle that case; but fzf
may still be available as a prebuilt binary.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_NOT_SUPPORTED_X86 = (
    """
We don't believe that a prebuilt binary for `fzf` if available on %s, but we
could be wrong.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_NOT_FOUND = (
    """
Could not find the `fzf` binary.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_VERSION_FAILED = (
    """
Could not obtain the 'fzf' version; we require version > 0.20.0 for some of
the features.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_INSTALL_FAILED = (
    """
Failed to install fzf.

""".lstrip()
    + FZF_INSTALL_MANUALLY
)

FZF_HEADER = """
For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
{shortcuts}
""".strip()

fzf_shortcuts = {
    "ctrl-a": "select-all",
    "ctrl-d": "deselect-all",
    "ctrl-t": "toggle-all",
    "alt-bspace": "beginning-of-line+kill-line",
    "?": "toggle-preview",
}

fzf_header_shortcuts = [
    ("select", "tab"),
    ("accept", "enter"),
    ("cancel", "ctrl-c"),
    ("select-all", "ctrl-a"),
    ("cursor-up", "up"),
    ("cursor-down", "down"),
]


def get_fzf_platform():
    if platform.machine() in ["i386", "i686"]:
        print(FZF_NOT_SUPPORTED_X86 % platform.machine())
        sys.exit(1)

    if platform.system().lower() == "windows":
        if platform.machine().lower() in ["x86_64", "amd64"]:
            return "windows_amd64.zip"
        elif platform.machine().lower() == "arm64":
            return "windows_arm64.zip"
        else:
            print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
            sys.exit(1)
    elif platform.system().lower() == "darwin":
        if platform.machine().lower() in ["x86_64", "amd64"]:
            return "darwin_amd64.zip"
        elif platform.machine().lower() == "arm64":
            return "darwin_arm64.zip"
        else:
            print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
            sys.exit(1)
    elif platform.system().lower() == "linux":
        if platform.machine().lower() in ["x86_64", "amd64"]:
            return "linux_amd64.tar.gz"
        elif platform.machine().lower() == "arm64":
            return "linux_arm64.tar.gz"
        else:
            print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
            sys.exit(1)
    else:
        print(FZF_COULD_NOT_DETERMINE_PLATFORM)
        sys.exit(1)


def get_fzf_state_dir():
    return os.path.join(get_state_dir(), "fzf")


def get_fzf_filename():
    return "fzf-%s-%s" % (FZF_CURRENT_VERSION, get_fzf_platform())


def get_fzf_download_link():
    return "https://github.com/junegunn/fzf/releases/download/%s/%s" % (
        FZF_CURRENT_VERSION,
        get_fzf_filename(),
    )


def clean_up_state_dir():
    """
    We used to have a checkout of fzf that we would update.
    Now we only download the bin and cpin the hash; so if
    we find the old git checkout, wipe it
    """

    fzf_path = os.path.join(get_state_dir(), "fzf")
    git_path = os.path.join(fzf_path, ".git")
    if os.path.isdir(git_path):
        shutil.rmtree(fzf_path, ignore_errors=True)

    # Also delete any existing fzf binary
    fzf_bin = find_executable("fzf", fzf_path)
    if fzf_bin:
        mozfile.remove(fzf_bin)

    # Make sure the state dir is present
    if not os.path.isdir(fzf_path):
        os.makedirs(fzf_path)


def download_and_install_fzf():
    clean_up_state_dir()
    download_link = get_fzf_download_link()
    download_file = get_fzf_filename()
    download_destination_path = get_fzf_state_dir()
    download_destination_file = os.path.join(download_destination_path, download_file)
    http_download_and_save(
        download_link, download_destination_file, FZF_CHECKSUMS[get_fzf_platform()]
    )

    mozfile.extract(download_destination_file, download_destination_path)
    mozfile.remove(download_destination_file)


def get_fzf_version(fzf_bin):
    cmd = [fzf_bin, "--version"]
    try:
        fzf_version = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        print(FZF_VERSION_FAILED)
        sys.exit(1)

    # Some fzf versions have extra, e.g 0.18.0 (ff95134)
    fzf_version = six.ensure_text(fzf_version.split()[0])

    return fzf_version


def should_force_fzf_update(fzf_bin):
    fzf_version = get_fzf_version(fzf_bin)

    # 0.20.0 introduced passing selections through a temporary file,
    # which is good for large ctrl-a actions.
    if Version(fzf_version) < Version(FZF_MIN_VERSION):
        print("fzf version is old, you must update to use ./mach try fuzzy.")
        return True
    return False


def fzf_bootstrap(update=False):
    """
    Bootstrap fzf if necessary and return path to the executable.

    This function is a bit complicated. We fetch a new version of fzf if:
    1) an existing fzf is too outdated
    2) the user says --update and we are behind the recommended version
    3) no fzf can be found and
      3a) user passes --update
      3b) user agrees to a prompt

    """
    fzf_path = get_fzf_state_dir()

    fzf_bin = find_executable("fzf")
    if not fzf_bin:
        fzf_bin = find_executable("fzf", fzf_path)

    if fzf_bin and should_force_fzf_update(fzf_bin):  # Case (1)
        update = True

    if fzf_bin and not update:
        return fzf_bin

    elif fzf_bin and update:
        # Case 2
        fzf_version = get_fzf_version(fzf_bin)
        if Version(fzf_version) < Version(FZF_CURRENT_VERSION) and update:
            # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH
            # Swap to os.path.commonpath when we're not on Py2
            if fzf_bin and update and not fzf_bin.startswith(fzf_path):
                print(
                    "fzf installed somewhere other than {}, please update manually".format(
                        fzf_path
                    )
                )
                sys.exit(1)

            download_and_install_fzf()
            print("Updated fzf to {}".format(FZF_CURRENT_VERSION))
        else:
            print("fzf is the recommended version and does not need an update")

    else:  # not fzf_bin:
        if not update:
            # Case 3b
            install = input("Could not detect fzf, install it now? [y/n]: ")
            if install.lower() != "y":
                return

        # Case 3a and 3b-fall-through
        download_and_install_fzf()
        fzf_bin = find_executable("fzf", fzf_path)
        print("Installed fzf to {}".format(fzf_path))

    return fzf_bin


def format_header():
    shortcuts = []
    for action, key in fzf_header_shortcuts:
        shortcuts.append(
            "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format(
                t=terminal, action=action, key=key
            )
        )
    return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal)


def run_fzf(cmd, tasks):
    env = dict(os.environ)
    env.update(
        {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])}
    )
    # Make sure fzf uses Windows' shell rather than MozillaBuild bash or
    # whatever our caller uses, since it doesn't quote the arguments properly
    # and thus windows paths like: C:\moz\foo end up as C:mozfoo...
    if platform.system() == "Windows":
        env["SHELL"] = env["COMSPEC"]
    proc = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stdin=subprocess.PIPE,
        env=env,
        universal_newlines=True,
    )
    out = proc.communicate("\n".join(tasks))[0].splitlines()

    selected = []
    query = None
    if out:
        query = out[0]
        selected = out[1:]
    return query, selected


def setup_tasks_for_fzf(
    push,
    parameters,
    full=False,
    disable_target_task_filter=False,
    show_estimates=True,
):
    check_working_directory(push)
    tg = generate_tasks(
        parameters, full=full, disable_target_task_filter=disable_target_task_filter
    )
    all_tasks = sorted(tg.tasks.keys())

    # graph_Cache created by generate_tasks, recreate the path to that file.
    cache_dir = os.path.join(
        get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph"
    )
    if full:
        graph_cache = os.path.join(cache_dir, "full_task_graph")
        dep_cache = os.path.join(cache_dir, "full_task_dependencies")
        target_set = os.path.join(cache_dir, "full_task_set")
    else:
        graph_cache = os.path.join(cache_dir, "target_task_graph")
        dep_cache = os.path.join(cache_dir, "target_task_dependencies")
        target_set = os.path.join(cache_dir, "target_task_set")

    if show_estimates:
        download_task_history_data(cache_dir=cache_dir)
        make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set)

    if not full and not disable_target_task_filter:
        # Put all_tasks into a list because it's used multiple times, and "filter()"
        # returns a consumable iterator.
        all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks))

    return all_tasks, dep_cache, cache_dir


def build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=True):
    key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()]
    base_cmd = [
        fzf,
        "-m",
        "--bind",
        ",".join(key_shortcuts),
        "--header",
        format_header(),
        "--preview-window=right:30%",
        "--print-query",
    ]

    if show_estimates:
        base_cmd.extend(
            [
                "--preview",
                '{} {} -g {} -s -c {} -t "{{+f}}"'.format(
                    sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir
                ),
            ]
        )
    else:
        base_cmd.extend(
            [
                "--preview",
                '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT),
            ]
        )

    return base_cmd