diff options
Diffstat (limited to '')
| -rw-r--r-- | mesonbuild/wrap/__init__.py | 59 | ||||
| -rw-r--r-- | mesonbuild/wrap/wrap.py | 833 | ||||
| -rw-r--r-- | mesonbuild/wrap/wraptool.py | 231 |
3 files changed, 1123 insertions, 0 deletions
diff --git a/mesonbuild/wrap/__init__.py b/mesonbuild/wrap/__init__.py new file mode 100644 index 0000000..653f42a --- /dev/null +++ b/mesonbuild/wrap/__init__.py @@ -0,0 +1,59 @@ +from enum import Enum + +# Used for the --wrap-mode command-line argument +# +# Special wrap modes: +# nofallback: Don't download wraps for dependency() fallbacks +# nodownload: Don't download wraps for all subproject() calls +# +# subprojects are used for two purposes: +# 1. To download and build dependencies by using .wrap +# files if they are not provided by the system. This is +# usually expressed via dependency(..., fallback: ...). +# 2. To download and build 'copylibs' which are meant to be +# used by copying into your project. This is always done +# with an explicit subproject() call. +# +# --wrap-mode=nofallback will never do (1) +# --wrap-mode=nodownload will do neither (1) nor (2) +# +# If you are building from a release tarball, you should be +# able to safely use 'nodownload' since upstream is +# expected to ship all required sources with the tarball. +# +# If you are building from a git repository, you will want +# to use 'nofallback' so that any 'copylib' wraps will be +# download as subprojects. +# +# --wrap-mode=forcefallback will ignore external dependencies, +# even if they match the version requirements, and automatically +# use the fallback if one was provided. This is useful for example +# to make sure a project builds when using the fallbacks. +# +# Note that these options do not affect subprojects that +# are git submodules since those are only usable in git +# repositories, and you almost always want to download them. + +# This did _not_ work when inside the WrapMode class. +# I don't know why. If you can fix this, patches welcome. +string_to_value = {'default': 1, + 'nofallback': 2, + 'nodownload': 3, + 'forcefallback': 4, + 'nopromote': 5, + } + +class WrapMode(Enum): + default = 1 + nofallback = 2 + nodownload = 3 + forcefallback = 4 + nopromote = 5 + + def __str__(self) -> str: + return self.name + + @staticmethod + def from_string(mode_name: str) -> 'WrapMode': + g = string_to_value[mode_name] + return WrapMode(g) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py new file mode 100644 index 0000000..9b1795b --- /dev/null +++ b/mesonbuild/wrap/wrap.py @@ -0,0 +1,833 @@ +# Copyright 2015 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from .. import mlog +import contextlib +from dataclasses import dataclass +import urllib.request +import urllib.error +import urllib.parse +import os +import hashlib +import shutil +import tempfile +import stat +import subprocess +import sys +import configparser +import time +import typing as T +import textwrap +import json + +from base64 import b64encode +from netrc import netrc +from pathlib import Path + +from . import WrapMode +from .. import coredata +from ..mesonlib import quiet_git, GIT, ProgressBar, MesonException, windows_proof_rmtree, Popen_safe +from ..interpreterbase import FeatureNew +from ..interpreterbase import SubProject +from .. import mesonlib + +if T.TYPE_CHECKING: + import http.client + +try: + # Importing is just done to check if SSL exists, so all warnings + # regarding 'imported but unused' can be safely ignored + import ssl # noqa + has_ssl = True +except ImportError: + has_ssl = False + +REQ_TIMEOUT = 600.0 +SSL_WARNING_PRINTED = False +WHITELIST_SUBDOMAIN = 'wrapdb.mesonbuild.com' + +ALL_TYPES = ['file', 'git', 'hg', 'svn'] + +PATCH = shutil.which('patch') + +def whitelist_wrapdb(urlstr: str) -> urllib.parse.ParseResult: + """ raises WrapException if not whitelisted subdomain """ + url = urllib.parse.urlparse(urlstr) + if not url.hostname: + raise WrapException(f'{urlstr} is not a valid URL') + if not url.hostname.endswith(WHITELIST_SUBDOMAIN): + raise WrapException(f'{urlstr} is not a whitelisted WrapDB URL') + if has_ssl and not url.scheme == 'https': + raise WrapException(f'WrapDB did not have expected SSL https url, instead got {urlstr}') + return url + +def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool = False) -> 'http.client.HTTPResponse': + if have_opt: + insecure_msg = '\n\n To allow connecting anyway, pass `--allow-insecure`.' + else: + insecure_msg = '' + + url = whitelist_wrapdb(urlstring) + if has_ssl: + try: + return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(url), timeout=REQ_TIMEOUT)) + except urllib.error.URLError as excp: + msg = f'WrapDB connection failed to {urlstring} with error {excp}.' + if isinstance(excp.reason, ssl.SSLCertVerificationError): + if allow_insecure: + mlog.warning(f'{msg}\n\n Proceeding without authentication.') + else: + raise WrapException(f'{msg}{insecure_msg}') + else: + raise WrapException(msg) + elif not allow_insecure: + raise WrapException(f'SSL module not available in {sys.executable}: Cannot contact the WrapDB.{insecure_msg}') + else: + # following code is only for those without Python SSL + global SSL_WARNING_PRINTED # pylint: disable=global-statement + if not SSL_WARNING_PRINTED: + mlog.warning(f'SSL module not available in {sys.executable}: WrapDB traffic not authenticated.') + SSL_WARNING_PRINTED = True + + # If we got this far, allow_insecure was manually passed + nossl_url = url._replace(scheme='http') + try: + return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(nossl_url), timeout=REQ_TIMEOUT)) + except urllib.error.URLError as excp: + raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}') + +def get_releases_data(allow_insecure: bool) -> bytes: + url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) + return url.read() + +def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + data = get_releases_data(allow_insecure) + return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) + +def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None: + url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap', + allow_insecure, True) + with open(wrapfile, 'wb') as f: + f.write(url.read()) + +def parse_patch_url(patch_url: str) -> T.Tuple[str, str]: + u = urllib.parse.urlparse(patch_url) + if u.netloc != 'wrapdb.mesonbuild.com': + raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch') + arr = u.path.strip('/').split('/') + if arr[0] == 'v1': + # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip + return arr[-3], arr[-2] + elif arr[0] == 'v2': + # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch + tag = arr[-2] + _, version = tag.rsplit('_', 1) + version, revision = version.rsplit('-', 1) + return version, revision + else: + raise WrapException(f'Invalid wrapdb URL {patch_url}') + +class WrapException(MesonException): + pass + +class WrapNotFoundException(WrapException): + pass + +class PackageDefinition: + def __init__(self, fname: str, subproject: str = ''): + self.filename = fname + self.subproject = SubProject(subproject) + self.type = None # type: T.Optional[str] + self.values = {} # type: T.Dict[str, str] + self.provided_deps = {} # type: T.Dict[str, T.Optional[str]] + self.provided_programs = [] # type: T.List[str] + self.diff_files = [] # type: T.List[Path] + self.basename = os.path.basename(fname) + self.has_wrap = self.basename.endswith('.wrap') + self.name = self.basename[:-5] if self.has_wrap else self.basename + # must be lowercase for consistency with dep=variable assignment + self.provided_deps[self.name.lower()] = None + # What the original file name was before redirection + self.original_filename = fname + self.redirected = False + if self.has_wrap: + self.parse_wrap() + with open(fname, 'r', encoding='utf-8') as file: + self.wrapfile_hash = hashlib.sha256(file.read().encode('utf-8')).hexdigest() + self.directory = self.values.get('directory', self.name) + if os.path.dirname(self.directory): + raise WrapException('Directory key must be a name and not a path') + if self.type and self.type not in ALL_TYPES: + raise WrapException(f'Unknown wrap type {self.type!r}') + self.filesdir = os.path.join(os.path.dirname(self.filename), 'packagefiles') + + def parse_wrap(self) -> None: + try: + config = configparser.ConfigParser(interpolation=None) + config.read(self.filename, encoding='utf-8') + except configparser.Error as e: + raise WrapException(f'Failed to parse {self.basename}: {e!s}') + self.parse_wrap_section(config) + if self.type == 'redirect': + # [wrap-redirect] have a `filename` value pointing to the real wrap + # file we should parse instead. It must be relative to the current + # wrap file location and must be in the form foo/subprojects/bar.wrap. + dirname = Path(self.filename).parent + fname = Path(self.values['filename']) + for i, p in enumerate(fname.parts): + if i % 2 == 0: + if p == '..': + raise WrapException('wrap-redirect filename cannot contain ".."') + else: + if p != 'subprojects': + raise WrapException('wrap-redirect filename must be in the form foo/subprojects/bar.wrap') + if fname.suffix != '.wrap': + raise WrapException('wrap-redirect filename must be a .wrap file') + fname = dirname / fname + if not fname.is_file(): + raise WrapException(f'wrap-redirect {fname} filename does not exist') + self.filename = str(fname) + self.parse_wrap() + self.redirected = True + else: + self.parse_provide_section(config) + if 'patch_directory' in self.values: + FeatureNew('Wrap files with patch_directory', '0.55.0').use(self.subproject) + for what in ['patch', 'source']: + if f'{what}_filename' in self.values and f'{what}_url' not in self.values: + FeatureNew(f'Local wrap patch files without {what}_url', '0.55.0').use(self.subproject) + + def parse_wrap_section(self, config: configparser.ConfigParser) -> None: + if len(config.sections()) < 1: + raise WrapException(f'Missing sections in {self.basename}') + self.wrap_section = config.sections()[0] + if not self.wrap_section.startswith('wrap-'): + raise WrapException(f'{self.wrap_section!r} is not a valid first section in {self.basename}') + self.type = self.wrap_section[5:] + self.values = dict(config[self.wrap_section]) + if 'diff_files' in self.values: + FeatureNew('Wrap files with diff_files', '0.63.0').use(self.subproject) + for s in self.values['diff_files'].split(','): + path = Path(s.strip()) + if path.is_absolute(): + raise WrapException('diff_files paths cannot be absolute') + if '..' in path.parts: + raise WrapException('diff_files paths cannot contain ".."') + self.diff_files.append(path) + + def parse_provide_section(self, config: configparser.ConfigParser) -> None: + if config.has_section('provide'): + for k, v in config['provide'].items(): + if k == 'dependency_names': + # A comma separated list of dependency names that does not + # need a variable name; must be lowercase for consistency with + # dep=variable assignment + names_dict = {n.strip().lower(): None for n in v.split(',')} + self.provided_deps.update(names_dict) + continue + if k == 'program_names': + # A comma separated list of program names + names_list = [n.strip() for n in v.split(',')] + self.provided_programs += names_list + continue + if not v: + m = (f'Empty dependency variable name for {k!r} in {self.basename}. ' + 'If the subproject uses meson.override_dependency() ' + 'it can be added in the "dependency_names" special key.') + raise WrapException(m) + self.provided_deps[k] = v + + def get(self, key: str) -> str: + try: + return self.values[key] + except KeyError: + raise WrapException(f'Missing key {key!r} in {self.basename}') + + def get_hashfile(self, subproject_directory: str) -> str: + return os.path.join(subproject_directory, '.meson-subproject-wrap-hash.txt') + + def update_hash_cache(self, subproject_directory: str) -> None: + if self.has_wrap: + with open(self.get_hashfile(subproject_directory), 'w', encoding='utf-8') as file: + file.write(self.wrapfile_hash + '\n') + +def get_directory(subdir_root: str, packagename: str) -> str: + fname = os.path.join(subdir_root, packagename + '.wrap') + if os.path.isfile(fname): + wrap = PackageDefinition(fname) + return wrap.directory + return packagename + +def verbose_git(cmd: T.List[str], workingdir: str, check: bool = False) -> bool: + ''' + Wrapper to convert GitException to WrapException caught in interpreter. + ''' + try: + return mesonlib.verbose_git(cmd, workingdir, check=check) + except mesonlib.GitException as e: + raise WrapException(str(e)) + +@dataclass(eq=False) +class Resolver: + source_dir: str + subdir: str + subproject: str = '' + wrap_mode: WrapMode = WrapMode.default + wrap_frontend: bool = False + allow_insecure: bool = False + + def __post_init__(self) -> None: + self.subdir_root = os.path.join(self.source_dir, self.subdir) + self.cachedir = os.path.join(self.subdir_root, 'packagecache') + self.wraps = {} # type: T.Dict[str, PackageDefinition] + self.netrc: T.Optional[netrc] = None + self.provided_deps = {} # type: T.Dict[str, PackageDefinition] + self.provided_programs = {} # type: T.Dict[str, PackageDefinition] + self.wrapdb: T.Dict[str, T.Any] = {} + self.wrapdb_provided_deps: T.Dict[str, str] = {} + self.wrapdb_provided_programs: T.Dict[str, str] = {} + self.load_wraps() + self.load_netrc() + self.load_wrapdb() + + def load_netrc(self) -> None: + try: + self.netrc = netrc() + except FileNotFoundError: + return + except Exception as e: + mlog.warning(f'failed to process netrc file: {e}.', fatal=False) + + def load_wraps(self) -> None: + if not os.path.isdir(self.subdir_root): + return + root, dirs, files = next(os.walk(self.subdir_root)) + ignore_dirs = {'packagecache', 'packagefiles'} + for i in files: + if not i.endswith('.wrap'): + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition(fname, self.subproject) + self.wraps[wrap.name] = wrap + ignore_dirs |= {wrap.directory, wrap.name} + # Add dummy package definition for directories not associated with a wrap file. + for i in dirs: + if i in ignore_dirs: + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition(fname, self.subproject) + self.wraps[wrap.name] = wrap + + for wrap in self.wraps.values(): + self.add_wrap(wrap) + + def add_wrap(self, wrap: PackageDefinition) -> None: + for k in wrap.provided_deps.keys(): + if k in self.provided_deps: + prev_wrap = self.provided_deps[k] + m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_deps[k] = wrap + for k in wrap.provided_programs: + if k in self.provided_programs: + prev_wrap = self.provided_programs[k] + m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_programs[k] = wrap + + def load_wrapdb(self) -> None: + try: + with Path(self.subdir_root, 'wrapdb.json').open('r', encoding='utf-8') as f: + self.wrapdb = json.load(f) + except FileNotFoundError: + return + for name, info in self.wrapdb.items(): + self.wrapdb_provided_deps.update({i: name for i in info.get('dependency_names', [])}) + self.wrapdb_provided_programs.update({i: name for i in info.get('program_names', [])}) + + def get_from_wrapdb(self, subp_name: str) -> PackageDefinition: + info = self.wrapdb.get(subp_name) + if not info: + return None + self.check_can_download() + latest_version = info['versions'][0] + version, revision = latest_version.rsplit('-', 1) + url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap') + fname = Path(self.subdir_root, f'{subp_name}.wrap') + with fname.open('wb') as f: + f.write(url.read()) + mlog.log(f'Installed {subp_name} version {version} revision {revision}') + wrap = PackageDefinition(str(fname)) + self.wraps[wrap.name] = wrap + self.add_wrap(wrap) + return wrap + + def merge_wraps(self, other_resolver: 'Resolver') -> None: + for k, v in other_resolver.wraps.items(): + self.wraps.setdefault(k, v) + for k, v in other_resolver.provided_deps.items(): + self.provided_deps.setdefault(k, v) + for k, v in other_resolver.provided_programs.items(): + self.provided_programs.setdefault(k, v) + + def find_dep_provider(self, packagename: str) -> T.Tuple[T.Optional[str], T.Optional[str]]: + # Python's ini parser converts all key values to lowercase. + # Thus the query name must also be in lower case. + packagename = packagename.lower() + wrap = self.provided_deps.get(packagename) + if wrap: + dep_var = wrap.provided_deps.get(packagename) + return wrap.name, dep_var + wrap_name = self.wrapdb_provided_deps.get(packagename) + return wrap_name, None + + def get_varname(self, subp_name: str, depname: str) -> T.Optional[str]: + wrap = self.wraps.get(subp_name) + return wrap.provided_deps.get(depname) if wrap else None + + def find_program_provider(self, names: T.List[str]) -> T.Optional[str]: + for name in names: + wrap = self.provided_programs.get(name) + if wrap: + return wrap.name + wrap_name = self.wrapdb_provided_programs.get(name) + if wrap_name: + return wrap_name + return None + + def resolve(self, packagename: str, method: str) -> str: + self.packagename = packagename + self.directory = packagename + self.wrap = self.wraps.get(packagename) + if not self.wrap: + self.wrap = self.get_from_wrapdb(packagename) + if not self.wrap: + m = f'Neither a subproject directory nor a {self.packagename}.wrap file was found.' + raise WrapNotFoundException(m) + self.directory = self.wrap.directory + + if self.wrap.has_wrap: + # We have a .wrap file, use directory relative to the location of + # the wrap file if it exists, otherwise source code will be placed + # into main project's subproject_dir even if the wrap file comes + # from another subproject. + self.dirname = os.path.join(os.path.dirname(self.wrap.filename), self.wrap.directory) + if not os.path.exists(self.dirname): + self.dirname = os.path.join(self.subdir_root, self.directory) + # Check if the wrap comes from the main project. + main_fname = os.path.join(self.subdir_root, self.wrap.basename) + if self.wrap.filename != main_fname: + rel = os.path.relpath(self.wrap.filename, self.source_dir) + mlog.log('Using', mlog.bold(rel)) + # Write a dummy wrap file in main project that redirect to the + # wrap we picked. + with open(main_fname, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + [wrap-redirect] + filename = {} + '''.format(os.path.relpath(self.wrap.filename, self.subdir_root)))) + else: + # No wrap file, it's a dummy package definition for an existing + # directory. Use the source code in place. + self.dirname = self.wrap.filename + rel_path = os.path.relpath(self.dirname, self.source_dir) + + if method == 'meson': + buildfile = os.path.join(self.dirname, 'meson.build') + elif method == 'cmake': + buildfile = os.path.join(self.dirname, 'CMakeLists.txt') + else: + raise WrapException('Only the methods "meson" and "cmake" are supported') + + # The directory is there and has meson.build? Great, use it. + if os.path.exists(buildfile): + self.validate() + return rel_path + + # Check if the subproject is a git submodule + self.resolve_git_submodule() + + if os.path.exists(self.dirname): + if not os.path.isdir(self.dirname): + raise WrapException('Path already exists but is not a directory') + else: + if self.wrap.type == 'file': + self.get_file() + else: + self.check_can_download() + if self.wrap.type == 'git': + self.get_git() + elif self.wrap.type == "hg": + self.get_hg() + elif self.wrap.type == "svn": + self.get_svn() + else: + raise WrapException(f'Unknown wrap type {self.wrap.type!r}') + try: + self.apply_patch() + self.apply_diff_files() + except Exception: + windows_proof_rmtree(self.dirname) + raise + + # A meson.build or CMakeLists.txt file is required in the directory + if not os.path.exists(buildfile): + raise WrapException(f'Subproject exists but has no {os.path.basename(buildfile)} file') + + # At this point, the subproject has been successfully resolved for the + # first time so save off the hash of the entire wrap file for future + # reference. + self.wrap.update_hash_cache(self.dirname) + + return rel_path + + def check_can_download(self) -> None: + # Don't download subproject data based on wrap file if requested. + # Git submodules are ok (see above)! + if self.wrap_mode is WrapMode.nodownload: + m = 'Automatic wrap-based subproject downloading is disabled' + raise WrapException(m) + + def resolve_git_submodule(self) -> bool: + # Is git installed? If not, we're probably not in a git repository and + # definitely cannot try to conveniently set up a submodule. + if not GIT: + return False + # Does the directory exist? Even uninitialised submodules checkout an + # empty directory to work in + if not os.path.isdir(self.dirname): + return False + # Are we in a git repository? + ret, out = quiet_git(['rev-parse'], Path(self.dirname).parent) + if not ret: + return False + # Is `dirname` a submodule? + ret, out = quiet_git(['submodule', 'status', '.'], self.dirname) + if not ret: + return False + # Submodule has not been added, add it + if out.startswith('+'): + mlog.warning('git submodule might be out of date') + return True + elif out.startswith('U'): + raise WrapException('git submodule has merge conflicts') + # Submodule exists, but is deinitialized or wasn't initialized + elif out.startswith('-'): + if verbose_git(['submodule', 'update', '--init', '.'], self.dirname): + return True + raise WrapException('git submodule failed to init') + # Submodule looks fine, but maybe it wasn't populated properly. Do a checkout. + elif out.startswith(' '): + verbose_git(['submodule', 'update', '.'], self.dirname) + verbose_git(['checkout', '.'], self.dirname) + # Even if checkout failed, try building it anyway and let the user + # handle any problems manually. + return True + elif out == '': + # It is not a submodule, just a folder that exists in the main repository. + return False + raise WrapException(f'Unknown git submodule output: {out!r}') + + def get_file(self) -> None: + path = self.get_file_internal('source') + extract_dir = self.subdir_root + # Some upstreams ship packages that do not have a leading directory. + # Create one for them. + if 'lead_directory_missing' in self.wrap.values: + os.mkdir(self.dirname) + extract_dir = self.dirname + shutil.unpack_archive(path, extract_dir) + + def get_git(self) -> None: + if not GIT: + raise WrapException(f'Git program not found, cannot download {self.packagename}.wrap via git.') + revno = self.wrap.get('revision') + checkout_cmd = ['-c', 'advice.detachedHead=false', 'checkout', revno, '--'] + is_shallow = False + depth_option = [] # type: T.List[str] + if self.wrap.values.get('depth', '') != '': + is_shallow = True + depth_option = ['--depth', self.wrap.values.get('depth')] + # for some reason git only allows commit ids to be shallowly fetched by fetch not with clone + if is_shallow and self.is_git_full_commit_id(revno): + # git doesn't support directly cloning shallowly for commits, + # so we follow https://stackoverflow.com/a/43136160 + verbose_git(['-c', 'init.defaultBranch=meson-dummy-branch', 'init', self.directory], self.subdir_root, check=True) + verbose_git(['remote', 'add', 'origin', self.wrap.get('url')], self.dirname, check=True) + revno = self.wrap.get('revision') + verbose_git(['fetch', *depth_option, 'origin', revno], self.dirname, check=True) + verbose_git(checkout_cmd, self.dirname, check=True) + if self.wrap.values.get('clone-recursive', '').lower() == 'true': + verbose_git(['submodule', 'update', '--init', '--checkout', + '--recursive', *depth_option], self.dirname, check=True) + push_url = self.wrap.values.get('push-url') + if push_url: + verbose_git(['remote', 'set-url', '--push', 'origin', push_url], self.dirname, check=True) + else: + if not is_shallow: + verbose_git(['clone', self.wrap.get('url'), self.directory], self.subdir_root, check=True) + if revno.lower() != 'head': + if not verbose_git(checkout_cmd, self.dirname): + verbose_git(['fetch', self.wrap.get('url'), revno], self.dirname, check=True) + verbose_git(checkout_cmd, self.dirname, check=True) + else: + args = ['-c', 'advice.detachedHead=false', 'clone', *depth_option] + if revno.lower() != 'head': + args += ['--branch', revno] + args += [self.wrap.get('url'), self.directory] + verbose_git(args, self.subdir_root, check=True) + if self.wrap.values.get('clone-recursive', '').lower() == 'true': + verbose_git(['submodule', 'update', '--init', '--checkout', '--recursive', *depth_option], + self.dirname, check=True) + push_url = self.wrap.values.get('push-url') + if push_url: + verbose_git(['remote', 'set-url', '--push', 'origin', push_url], self.dirname, check=True) + + def validate(self) -> None: + # This check is only for subprojects with wraps. + if not self.wrap.has_wrap: + return + + # Retrieve original hash, if it exists. + hashfile = self.wrap.get_hashfile(self.dirname) + if os.path.isfile(hashfile): + with open(hashfile, 'r', encoding='utf-8') as file: + expected_hash = file.read().strip() + else: + # If stored hash doesn't exist then don't warn. + return + + actual_hash = self.wrap.wrapfile_hash + + # Compare hashes and warn the user if they don't match. + if expected_hash != actual_hash: + mlog.warning(f'Subproject {self.wrap.name}\'s revision may be out of date; its wrap file has changed since it was first configured') + + def is_git_full_commit_id(self, revno: str) -> bool: + result = False + if len(revno) in {40, 64}: # 40 for sha1, 64 for upcoming sha256 + result = all(ch in '0123456789AaBbCcDdEeFf' for ch in revno) + return result + + def get_hg(self) -> None: + revno = self.wrap.get('revision') + hg = shutil.which('hg') + if not hg: + raise WrapException('Mercurial program not found.') + subprocess.check_call([hg, 'clone', self.wrap.get('url'), + self.directory], cwd=self.subdir_root) + if revno.lower() != 'tip': + subprocess.check_call([hg, 'checkout', revno], + cwd=self.dirname) + + def get_svn(self) -> None: + revno = self.wrap.get('revision') + svn = shutil.which('svn') + if not svn: + raise WrapException('SVN program not found.') + subprocess.check_call([svn, 'checkout', '-r', revno, self.wrap.get('url'), + self.directory], cwd=self.subdir_root) + + def get_netrc_credentials(self, netloc: str) -> T.Optional[T.Tuple[str, str]]: + if self.netrc is None or netloc not in self.netrc.hosts: + return None + + login, account, password = self.netrc.authenticators(netloc) + if account is not None: + login = account + + return login, password + + def get_data(self, urlstring: str) -> T.Tuple[str, str]: + blocksize = 10 * 1024 + h = hashlib.sha256() + tmpfile = tempfile.NamedTemporaryFile(mode='wb', dir=self.cachedir, delete=False) + url = urllib.parse.urlparse(urlstring) + if url.hostname and url.hostname.endswith(WHITELIST_SUBDOMAIN): + resp = open_wrapdburl(urlstring, allow_insecure=self.allow_insecure, have_opt=self.wrap_frontend) + elif WHITELIST_SUBDOMAIN in urlstring: + raise WrapException(f'{urlstring} may be a WrapDB-impersonating URL') + else: + headers = {'User-Agent': f'mesonbuild/{coredata.version}'} + creds = self.get_netrc_credentials(url.netloc) + + if creds is not None and '@' not in url.netloc: + login, password = creds + if url.scheme == 'https': + enc_creds = b64encode(f'{login}:{password}'.encode()).decode() + headers.update({'Authorization': f'Basic {enc_creds}'}) + elif url.scheme == 'ftp': + urlstring = urllib.parse.urlunparse(url._replace(netloc=f'{login}:{password}@{url.netloc}')) + else: + mlog.warning('Meson is not going to use netrc credentials for protocols other than https/ftp', + fatal=False) + + try: + req = urllib.request.Request(urlstring, headers=headers) + resp = urllib.request.urlopen(req, timeout=REQ_TIMEOUT) + except urllib.error.URLError as e: + mlog.log(str(e)) + raise WrapException(f'could not get {urlstring} is the internet available?') + with contextlib.closing(resp) as resp, tmpfile as tmpfile: + try: + dlsize = int(resp.info()['Content-Length']) + except TypeError: + dlsize = None + if dlsize is None: + print('Downloading file of unknown size.') + while True: + block = resp.read(blocksize) + if block == b'': + break + h.update(block) + tmpfile.write(block) + hashvalue = h.hexdigest() + return hashvalue, tmpfile.name + sys.stdout.flush() + progress_bar = ProgressBar(bar_type='download', total=dlsize, + desc='Downloading') + while True: + block = resp.read(blocksize) + if block == b'': + break + h.update(block) + tmpfile.write(block) + progress_bar.update(len(block)) + progress_bar.close() + hashvalue = h.hexdigest() + return hashvalue, tmpfile.name + + def check_hash(self, what: str, path: str, hash_required: bool = True) -> None: + if what + '_hash' not in self.wrap.values and not hash_required: + return + expected = self.wrap.get(what + '_hash').lower() + h = hashlib.sha256() + with open(path, 'rb') as f: + h.update(f.read()) + dhash = h.hexdigest() + if dhash != expected: + raise WrapException(f'Incorrect hash for {what}:\n {expected} expected\n {dhash} actual.') + + def get_data_with_backoff(self, urlstring: str) -> T.Tuple[str, str]: + delays = [1, 2, 4, 8, 16] + for d in delays: + try: + return self.get_data(urlstring) + except Exception as e: + mlog.warning(f'failed to download with error: {e}. Trying after a delay...', fatal=False) + time.sleep(d) + return self.get_data(urlstring) + + def download(self, what: str, ofname: str, fallback: bool = False) -> None: + self.check_can_download() + srcurl = self.wrap.get(what + ('_fallback_url' if fallback else '_url')) + mlog.log('Downloading', mlog.bold(self.packagename), what, 'from', mlog.bold(srcurl)) + try: + dhash, tmpfile = self.get_data_with_backoff(srcurl) + expected = self.wrap.get(what + '_hash').lower() + if dhash != expected: + os.remove(tmpfile) + raise WrapException(f'Incorrect hash for {what}:\n {expected} expected\n {dhash} actual.') + except WrapException: + if not fallback: + if what + '_fallback_url' in self.wrap.values: + return self.download(what, ofname, fallback=True) + mlog.log('A fallback URL could be specified using', + mlog.bold(what + '_fallback_url'), 'key in the wrap file') + raise + os.rename(tmpfile, ofname) + + def get_file_internal(self, what: str) -> str: + filename = self.wrap.get(what + '_filename') + if what + '_url' in self.wrap.values: + cache_path = os.path.join(self.cachedir, filename) + + if os.path.exists(cache_path): + self.check_hash(what, cache_path) + mlog.log('Using', mlog.bold(self.packagename), what, 'from cache.') + return cache_path + + os.makedirs(self.cachedir, exist_ok=True) + self.download(what, cache_path) + return cache_path + else: + path = Path(self.wrap.filesdir) / filename + + if not path.exists(): + raise WrapException(f'File "{path}" does not exist') + self.check_hash(what, path.as_posix(), hash_required=False) + + return path.as_posix() + + def apply_patch(self) -> None: + if 'patch_filename' in self.wrap.values and 'patch_directory' in self.wrap.values: + m = f'Wrap file {self.wrap.basename!r} must not have both "patch_filename" and "patch_directory"' + raise WrapException(m) + if 'patch_filename' in self.wrap.values: + path = self.get_file_internal('patch') + try: + shutil.unpack_archive(path, self.subdir_root) + except Exception: + with tempfile.TemporaryDirectory() as workdir: + shutil.unpack_archive(path, workdir) + self.copy_tree(workdir, self.subdir_root) + elif 'patch_directory' in self.wrap.values: + patch_dir = self.wrap.values['patch_directory'] + src_dir = os.path.join(self.wrap.filesdir, patch_dir) + if not os.path.isdir(src_dir): + raise WrapException(f'patch directory does not exist: {patch_dir}') + self.copy_tree(src_dir, self.dirname) + + def apply_diff_files(self) -> None: + for filename in self.wrap.diff_files: + mlog.log(f'Applying diff file "{filename}"') + path = Path(self.wrap.filesdir) / filename + if not path.exists(): + raise WrapException(f'Diff file "{path}" does not exist') + relpath = os.path.relpath(str(path), self.dirname) + if PATCH: + cmd = [PATCH, '-f', '-p1', '-i', relpath] + elif GIT: + # If the `patch` command is not available, fall back to `git + # apply`. The `--work-tree` is necessary in case we're inside a + # Git repository: by default, Git will try to apply the patch to + # the repository root. + cmd = [GIT, '--work-tree', '.', 'apply', '-p1', relpath] + else: + raise WrapException('Missing "patch" or "git" commands to apply diff files') + + p, out, _ = Popen_safe(cmd, cwd=self.dirname, stderr=subprocess.STDOUT) + if p.returncode != 0: + mlog.log(out.strip()) + raise WrapException(f'Failed to apply diff file "{filename}"') + + def copy_tree(self, root_src_dir: str, root_dst_dir: str) -> None: + """ + Copy directory tree. Overwrites also read only files. + """ + for src_dir, _, files in os.walk(root_src_dir): + dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) + if not os.path.exists(dst_dir): + os.makedirs(dst_dir) + for file_ in files: + src_file = os.path.join(src_dir, file_) + dst_file = os.path.join(dst_dir, file_) + if os.path.exists(dst_file): + try: + os.remove(dst_file) + except PermissionError: + os.chmod(dst_file, stat.S_IWUSR) + os.remove(dst_file) + shutil.copy2(src_file, dst_dir) diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py new file mode 100644 index 0000000..c009aa1 --- /dev/null +++ b/mesonbuild/wrap/wraptool.py @@ -0,0 +1,231 @@ +# Copyright 2015-2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import sys, os +import configparser +import shutil +import typing as T + +from glob import glob +from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data, + update_wrap_file, parse_patch_url) +from pathlib import Path + +from .. import mesonlib, msubprojects + +if T.TYPE_CHECKING: + import argparse + +def add_arguments(parser: 'argparse.ArgumentParser') -> None: + subparsers = parser.add_subparsers(title='Commands', dest='command') + subparsers.required = True + + p = subparsers.add_parser('list', help='show all available projects') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.set_defaults(wrap_func=list_projects) + + p = subparsers.add_parser('search', help='search the db by name') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.add_argument('name') + p.set_defaults(wrap_func=search) + + p = subparsers.add_parser('install', help='install the specified project') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.add_argument('name') + p.set_defaults(wrap_func=install) + + p = msubprojects.add_wrap_update_parser(subparsers) + p.set_defaults(wrap_func=msubprojects.run) + + p = subparsers.add_parser('info', help='show available versions of a project') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.add_argument('name') + p.set_defaults(wrap_func=info) + + p = subparsers.add_parser('status', help='show installed and available versions of your projects') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.set_defaults(wrap_func=status) + + p = subparsers.add_parser('promote', help='bring a subsubproject up to the master project') + p.add_argument('project_path') + p.set_defaults(wrap_func=promote) + + p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.set_defaults(wrap_func=update_db) + +def list_projects(options: 'argparse.Namespace') -> None: + releases = get_releases(options.allow_insecure) + for p in releases.keys(): + print(p) + +def search(options: 'argparse.Namespace') -> None: + name = options.name + releases = get_releases(options.allow_insecure) + for p, info in releases.items(): + if p.find(name) != -1: + print(p) + else: + for dep in info.get('dependency_names', []): + if dep.find(name) != -1: + print(f'Dependency {dep} found in wrap {p}') + +def get_latest_version(name: str, allow_insecure: bool) -> T.Tuple[str, str]: + releases = get_releases(allow_insecure) + info = releases.get(name) + if not info: + raise WrapException(f'Wrap {name} not found in wrapdb') + latest_version = info['versions'][0] + version, revision = latest_version.rsplit('-', 1) + return version, revision + +def install(options: 'argparse.Namespace') -> None: + name = options.name + if not os.path.isdir('subprojects'): + raise SystemExit('Subprojects dir not found. Run this script in your source root directory.') + if os.path.isdir(os.path.join('subprojects', name)): + raise SystemExit('Subproject directory for this project already exists.') + wrapfile = os.path.join('subprojects', name + '.wrap') + if os.path.exists(wrapfile): + raise SystemExit('Wrap file already exists.') + (version, revision) = get_latest_version(name, options.allow_insecure) + url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{version}-{revision}/{name}.wrap', options.allow_insecure, True) + with open(wrapfile, 'wb') as f: + f.write(url.read()) + print(f'Installed {name} version {version} revision {revision}') + +def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]: + cp = configparser.ConfigParser(interpolation=None) + cp.read(wrapfile) + try: + wrap_data = cp['wrap-file'] + except KeyError: + raise WrapException('Not a wrap-file, cannot have come from the wrapdb') + try: + patch_url = wrap_data['patch_url'] + except KeyError: + # We assume a wrap without a patch_url is probably just an pointer to upstream's + # build files. The version should be in the tarball filename, even if it isn't + # purely guaranteed. The wrapdb revision should be 1 because it just needs uploading once. + branch = mesonlib.search_version(wrap_data['source_filename']) + revision, patch_filename = '1', None + else: + branch, revision = parse_patch_url(patch_url) + patch_filename = wrap_data['patch_filename'] + return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename + +def update(options: 'argparse.Namespace') -> None: + name = options.name + if not os.path.isdir('subprojects'): + raise SystemExit('Subprojects dir not found. Run this command in your source root directory.') + wrapfile = os.path.join('subprojects', name + '.wrap') + if not os.path.exists(wrapfile): + raise SystemExit('Project ' + name + ' is not in use.') + (branch, revision, subdir, src_file, patch_file) = get_current_version(wrapfile) + (new_branch, new_revision) = get_latest_version(name, options.allow_insecure) + if new_branch == branch and new_revision == revision: + print('Project ' + name + ' is already up to date.') + raise SystemExit + update_wrap_file(wrapfile, name, new_branch, new_revision, options.allow_insecure) + shutil.rmtree(os.path.join('subprojects', subdir), ignore_errors=True) + try: + os.unlink(os.path.join('subprojects/packagecache', src_file)) + except FileNotFoundError: + pass + if patch_file is not None: + try: + os.unlink(os.path.join('subprojects/packagecache', patch_file)) + except FileNotFoundError: + pass + print(f'Updated {name} version {new_branch} revision {new_revision}') + +def info(options: 'argparse.Namespace') -> None: + name = options.name + releases = get_releases(options.allow_insecure) + info = releases.get(name) + if not info: + raise WrapException(f'Wrap {name} not found in wrapdb') + print(f'Available versions of {name}:') + for v in info['versions']: + print(' ', v) + +def do_promotion(from_path: str, spdir_name: str) -> None: + if os.path.isfile(from_path): + assert from_path.endswith('.wrap') + shutil.copy(from_path, spdir_name) + elif os.path.isdir(from_path): + sproj_name = os.path.basename(from_path) + outputdir = os.path.join(spdir_name, sproj_name) + if os.path.exists(outputdir): + raise SystemExit(f'Output dir {outputdir} already exists. Will not overwrite.') + shutil.copytree(from_path, outputdir, ignore=shutil.ignore_patterns('subprojects')) + +def promote(options: 'argparse.Namespace') -> None: + argument = options.project_path + spdir_name = 'subprojects' + sprojs = mesonlib.detect_subprojects(spdir_name) + + # check if the argument is a full path to a subproject directory or wrap file + system_native_path_argument = argument.replace('/', os.sep) + for matches in sprojs.values(): + if system_native_path_argument in matches: + do_promotion(system_native_path_argument, spdir_name) + return + + # otherwise the argument is just a subproject basename which must be unambiguous + if argument not in sprojs: + raise SystemExit(f'Subproject {argument} not found in directory tree.') + matches = sprojs[argument] + if len(matches) > 1: + print(f'There is more than one version of {argument} in tree. Please specify which one to promote:\n', file=sys.stderr) + for s in matches: + print(s, file=sys.stderr) + raise SystemExit(1) + do_promotion(matches[0], spdir_name) + +def status(options: 'argparse.Namespace') -> None: + print('Subproject status') + for w in glob('subprojects/*.wrap'): + name = os.path.basename(w)[:-5] + try: + (latest_branch, latest_revision) = get_latest_version(name, options.allow_insecure) + except Exception: + print('', name, 'not available in wrapdb.', file=sys.stderr) + continue + try: + (current_branch, current_revision, _, _, _) = get_current_version(w) + except Exception: + print('', name, 'Wrap file not from wrapdb.', file=sys.stderr) + continue + if current_branch == latest_branch and current_revision == latest_revision: + print('', name, f'up to date. Branch {current_branch}, revision {current_revision}.') + else: + print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.') + +def update_db(options: 'argparse.Namespace') -> None: + data = get_releases_data(options.allow_insecure) + Path('subprojects').mkdir(exist_ok=True) + with Path('subprojects/wrapdb.json').open('wb') as f: + f.write(data) + +def run(options: 'argparse.Namespace') -> int: + options.wrap_func(options) + return 0 |