diff options
Diffstat (limited to '')
| -rwxr-xr-x | debian/bin/gencontrol.py | 691 |
1 files changed, 691 insertions, 0 deletions
diff --git a/debian/bin/gencontrol.py b/debian/bin/gencontrol.py new file mode 100755 index 000000000..3df5549cc --- /dev/null +++ b/debian/bin/gencontrol.py @@ -0,0 +1,691 @@ +#!/usr/bin/python3 + +import sys +import locale +import io +import os +import os.path +import subprocess +import re + +from debian_linux import config +from debian_linux.debian import PackageDescription, PackageRelation, \ + PackageRelationEntry, PackageRelationGroup, VersionLinux +from debian_linux.gencontrol import Gencontrol as Base, merge_packages, \ + iter_featuresets +from debian_linux.utils import Templates, read_control + +locale.setlocale(locale.LC_CTYPE, "C.UTF-8") + + +class Gencontrol(Base): + config_schema = { + 'abi': { + 'ignore-changes': config.SchemaItemList(), + }, + 'build': { + 'debug-info': config.SchemaItemBoolean(), + 'signed-code': config.SchemaItemBoolean(), + 'vdso': config.SchemaItemBoolean(), + }, + 'description': { + 'parts': config.SchemaItemList(), + }, + 'image': { + 'bootloaders': config.SchemaItemList(), + 'configs': config.SchemaItemList(), + 'initramfs-generators': config.SchemaItemList(), + 'check-size': config.SchemaItemInteger(), + 'check-size-with-dtb': config.SchemaItemBoolean(), + 'check-uncompressed-size': config.SchemaItemInteger(), + 'depends': config.SchemaItemList(','), + 'provides': config.SchemaItemList(','), + 'suggests': config.SchemaItemList(','), + 'recommends': config.SchemaItemList(','), + 'conflicts': config.SchemaItemList(','), + 'breaks': config.SchemaItemList(','), + }, + 'relations': { + }, + 'packages': { + 'docs': config.SchemaItemBoolean(), + 'headers-all': config.SchemaItemBoolean(), + 'installer': config.SchemaItemBoolean(), + 'libc-dev': config.SchemaItemBoolean(), + 'tools-unversioned': config.SchemaItemBoolean(), + 'tools-versioned': config.SchemaItemBoolean(), + 'source': config.SchemaItemBoolean(), + } + } + + env_flags = [ + ('DEBIAN_KERNEL_DISABLE_DEBUG', 'disable_debug', 'debug infos'), + ('DEBIAN_KERNEL_DISABLE_INSTALLER', 'disable_installer', 'installer modules'), + ('DEBIAN_KERNEL_DISABLE_SIGNED', 'disable_signed', 'signed code'), + ] + + def __init__(self, config_dirs=["debian/config"], + template_dirs=["debian/templates"]): + super(Gencontrol, self).__init__( + config.ConfigCoreHierarchy(self.config_schema, config_dirs), + Templates(template_dirs), + VersionLinux) + self.process_changelog() + self.config_dirs = config_dirs + + for env, attr, desc in self.env_flags: + setattr(self, attr, False) + if os.getenv(env): + if self.changelog[0].distribution == 'UNRELEASED': + import warnings + warnings.warn(f'Disable {desc} on request ({env} set)') + setattr(self, attr, True) + else: + raise RuntimeError( + f'Unable to disable {desc} in release build ({env} set)') + + def _setup_makeflags(self, names, makeflags, data): + for src, dst, optional in names: + if src in data or not optional: + makeflags[dst] = data[src] + + def _substitute_file(self, template, vars, target, append=False): + with open(target, 'a' if append else 'w') as f: + f.write(self.substitute(self.templates[template], vars)) + + def do_main_setup(self, vars, makeflags, extra): + super(Gencontrol, self).do_main_setup(vars, makeflags, extra) + makeflags.update({ + 'VERSION': self.version.linux_version, + 'UPSTREAMVERSION': self.version.linux_upstream, + 'ABINAME': self.abiname_version + self.abiname_part, + 'SOURCEVERSION': self.version.complete, + }) + makeflags['SOURCE_BASENAME'] = self.vars['source_basename'] + + # Prepare to generate debian/tests/control + self.tests_control = self.process_packages( + self.templates['tests-control.main'], vars) + self.tests_control_image = None + + self.installer_packages = {} + + if not self.disable_installer and self.config.merge('packages').get('installer', True): + # Add udebs using kernel-wedge + kw_env = os.environ.copy() + kw_env['KW_DEFCONFIG_DIR'] = 'debian/installer' + kw_env['KW_CONFIG_DIR'] = 'debian/installer' + kw_proc = subprocess.Popen( + ['kernel-wedge', 'gen-control', vars['abiname']], + stdout=subprocess.PIPE, + env=kw_env) + if not isinstance(kw_proc.stdout, io.IOBase): + udeb_packages = read_control(io.open(kw_proc.stdout.fileno(), + closefd=False)) + else: + udeb_packages = read_control(io.TextIOWrapper(kw_proc.stdout)) + kw_proc.wait() + if kw_proc.returncode != 0: + raise RuntimeError('kernel-wedge exited with code %d' % + kw_proc.returncode) + + # All architectures that have some installer udebs + arches = set() + for package in udeb_packages: + arches.update(package['Architecture']) + + # Code-signing status for those architectures + # If we're going to build signed udebs later, don't actually + # generate udebs. Just test that we *can* build, so we find + # configuration errors before building linux-signed. + build_signed = {} + for arch in arches: + if not self.disable_signed: + build_signed[arch] = self.config.merge('build', arch) \ + .get('signed-code', False) + else: + build_signed[arch] = False + + for package in udeb_packages: + # kernel-wedge currently chokes on Build-Profiles so add it now + if any(build_signed[arch] for arch in package['Architecture']): + assert all(build_signed[arch] + for arch in package['Architecture']) + # XXX This is a hack to exclude the udebs from + # the package list while still being able to + # convince debhelper and kernel-wedge to go + # part way to building them. + package['Build-Profiles'] = ( + '<pkg.linux.udeb-unsigned-test-build>') + else: + package['Build-Profiles'] = '<!stage1 !pkg.linux.nokernel>' + + for arch in package['Architecture']: + self.installer_packages.setdefault(arch, []) \ + .append(package) + + def do_main_makefile(self, makefile, makeflags, extra): + for featureset in iter_featuresets(self.config): + makeflags_featureset = makeflags.copy() + makeflags_featureset['FEATURESET'] = featureset + cmds_source = ["$(MAKE) -f debian/rules.real source-featureset %s" + % makeflags_featureset] + makefile.add('source_%s_real' % featureset, cmds=cmds_source) + makefile.add('source_%s' % featureset, + ['source_%s_real' % featureset]) + makefile.add('source', ['source_%s' % featureset]) + + makeflags = makeflags.copy() + makeflags['ALL_FEATURESETS'] = ' '.join(iter_featuresets(self.config)) + super(Gencontrol, self).do_main_makefile(makefile, makeflags, extra) + + def do_main_packages(self, packages, vars, makeflags, extra): + packages.extend(self.process_packages( + self.templates["control.main"], self.vars)) + if self.config.merge('packages').get('docs', True): + packages.extend(self.process_packages( + self.templates["control.docs"], self.vars)) + if self.config.merge('packages').get('tools-unversioned', True): + packages.extend(self.process_packages( + self.templates["control.tools-unversioned"], self.vars)) + if self.config.merge('packages').get('tools-versioned', True): + packages.extend(self.process_packages( + self.templates["control.tools-versioned"], self.vars)) + if self.config.merge('packages').get('source', True): + packages.extend(self.process_packages( + self.templates["control.sourcebin"], self.vars)) + + self._substitute_file('perf.lintian-overrides', self.vars, + 'debian/linux-perf-%s.lintian-overrides' % + self.vars['version']) + + def do_indep_featureset_setup(self, vars, makeflags, featureset, extra): + makeflags['LOCALVERSION'] = vars['localversion'] + kernel_arches = set() + for arch in iter(self.config['base', ]['arches']): + if self.config.get_merge('base', arch, featureset, None, + 'flavours'): + kernel_arches.add(self.config['base', arch]['kernel-arch']) + makeflags['ALL_KERNEL_ARCHES'] = ' '.join(sorted(list(kernel_arches))) + + vars['featureset_desc'] = '' + if featureset != 'none': + desc = self.config[('description', None, featureset)] + desc_parts = desc['parts'] + vars['featureset_desc'] = (' with the %s featureset' % + desc['part-short-%s' % desc_parts[0]]) + + def do_indep_featureset_packages(self, packages, makefile, featureset, + vars, makeflags, extra): + headers_featureset = self.templates["control.headers.featureset"] + packages.extend(self.process_packages(headers_featureset, vars)) + + cmds_binary_arch = ["$(MAKE) -f debian/rules.real " + "binary-indep-featureset %s" % + makeflags] + makefile.add('binary-indep_%s_real' % featureset, + cmds=cmds_binary_arch) + + arch_makeflags = ( + ('kernel-arch', 'KERNEL_ARCH', False), + ) + + def do_arch_setup(self, vars, makeflags, arch, extra): + config_base = self.config.merge('base', arch) + + self._setup_makeflags(self.arch_makeflags, makeflags, config_base) + + try: + gnu_type_bytes = subprocess.check_output( + ['dpkg-architecture', '-f', '-a', arch, + '-q', 'DEB_HOST_GNU_TYPE'], + stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + # This sometimes happens for the newest ports :-/ + print('W: Unable to get GNU type for %s' % arch, file=sys.stderr) + else: + vars['gnu-type-package'] = ( + gnu_type_bytes.decode('utf-8').strip().replace('_', '-')) + + def do_arch_packages(self, packages, makefile, arch, vars, makeflags, + extra): + if self.version.linux_modifier is None: + try: + abiname_part = '-%s' % self.config['abi', arch]['abiname'] + except KeyError: + abiname_part = self.abiname_part + makeflags['ABINAME'] = vars['abiname'] = \ + self.abiname_version + abiname_part + + if not self.disable_signed: + build_signed = self.config.merge('build', arch) \ + .get('signed-code', False) + else: + build_signed = False + + # Some userland architectures require kernels from another + # (Debian) architecture, e.g. x32/amd64. + # And some derivatives don't need the headers-all packages + # for other reasons. + if self.config['base', arch].get('featuresets') and \ + self.config.merge('packages').get('headers-all', True): + headers_arch = self.templates["control.headers.arch"] + packages_headers_arch = self.process_packages(headers_arch, vars) + packages_headers_arch[-1]['Depends'].extend(PackageRelation()) + extra['headers_arch_depends'] = ( + packages_headers_arch[-1]['Depends']) + else: + packages_headers_arch = [] + + if self.config.merge('packages').get('libc-dev', True): + libc_dev = self.templates["control.libc-dev"] + packages_headers_arch[0:0] = self.process_packages(libc_dev, {}) + + merge_packages(packages, packages_headers_arch, arch) + + if self.config['base', arch].get('featuresets') and \ + self.config.merge('packages').get('source', True): + merge_packages(packages, + self.process_packages( + self.templates["control.config"], vars), + arch) + + cmds_build_arch = ["$(MAKE) -f debian/rules.real build-arch-arch %s" % + makeflags] + makefile.add('build-arch_%s_real' % arch, cmds=cmds_build_arch) + + cmds_binary_arch = ["$(MAKE) -f debian/rules.real binary-arch-arch %s" + % makeflags] + makefile.add('binary-arch_%s_real' % arch, cmds=cmds_binary_arch, + deps=['setup_%s' % arch]) + + udeb_packages = self.installer_packages.get(arch, []) + if udeb_packages: + merge_packages(packages, udeb_packages, arch) + + # These packages must be built after the per-flavour/ + # per-featureset packages. Also, this won't work + # correctly with an empty package list. + makefile.add( + 'binary-arch_%s' % arch, + cmds=["$(MAKE) -f debian/rules.real install-udeb_%s %s " + "PACKAGE_NAMES='%s' UDEB_UNSIGNED_TEST_BUILD=%s" % + (arch, makeflags, + ' '.join(p['Package'] for p in udeb_packages), + build_signed)]) + + # This also needs to be built after the per-flavour/per-featureset + # packages. + if build_signed: + merge_packages(packages, + self.process_packages( + self.templates['control.signed-template'], + vars), + arch) + makefile.add( + 'binary-arch_%s' % arch, + cmds=["$(MAKE) -f debian/rules.real " + "install-signed-template_%s %s" % + (arch, makeflags)]) + + def do_featureset_setup(self, vars, makeflags, arch, featureset, extra): + vars['localversion_headers'] = vars['localversion'] + makeflags['LOCALVERSION_HEADERS'] = vars['localversion_headers'] + + flavour_makeflags_base = ( + ('compiler', 'COMPILER', False), + ('compiler-filename', 'COMPILER', True), + ('kernel-arch', 'KERNEL_ARCH', False), + ('cflags', 'CFLAGS_KERNEL', True), + ('override-host-type', 'OVERRIDE_HOST_TYPE', True), + ) + + flavour_makeflags_build = ( + ('image-file', 'IMAGE_FILE', True), + ) + + flavour_makeflags_image = ( + ('install-stem', 'IMAGE_INSTALL_STEM', True), + ) + + flavour_makeflags_other = ( + ('localversion', 'LOCALVERSION', False), + ('localversion-image', 'LOCALVERSION_IMAGE', True), + ) + + def do_flavour_setup(self, vars, makeflags, arch, featureset, flavour, + extra): + config_base = self.config.merge('base', arch, featureset, flavour) + config_build = self.config.merge('build', arch, featureset, flavour) + config_description = self.config.merge('description', arch, featureset, + flavour) + config_image = self.config.merge('image', arch, featureset, flavour) + + vars['class'] = config_description['hardware'] + vars['longclass'] = (config_description.get('hardware-long') + or vars['class']) + + vars['localversion-image'] = vars['localversion'] + override_localversion = config_image.get('override-localversion', None) + if override_localversion is not None: + vars['localversion-image'] = (vars['localversion_headers'] + '-' + + override_localversion) + vars['image-stem'] = config_image.get('install-stem') + + self._setup_makeflags(self.flavour_makeflags_base, makeflags, + config_base) + self._setup_makeflags(self.flavour_makeflags_build, makeflags, + config_build) + self._setup_makeflags(self.flavour_makeflags_image, makeflags, + config_image) + self._setup_makeflags(self.flavour_makeflags_other, makeflags, vars) + + def do_flavour_packages(self, packages, makefile, arch, featureset, + flavour, vars, makeflags, extra): + headers = self.templates["control.headers"] + assert len(headers) == 1 + + config_entry_base = self.config.merge('base', arch, featureset, + flavour) + config_entry_build = self.config.merge('build', arch, featureset, + flavour) + config_entry_description = self.config.merge('description', arch, + featureset, flavour) + config_entry_relations = self.config.merge('relations', arch, + featureset, flavour) + + def config_entry_image(key, *args, **kwargs): + return self.config.get_merge( + 'image', arch, featureset, flavour, key, *args, **kwargs) + + compiler = config_entry_base.get('compiler', 'gcc') + + # Work out dependency from linux-headers to compiler. Drop + # dependencies for cross-builds. Strip any remaining + # restrictions, as they don't apply to binary Depends. + relations_compiler_headers = PackageRelation( + self.substitute(config_entry_relations.get('headers%' + compiler) + or config_entry_relations.get(compiler), vars)) + relations_compiler_headers = PackageRelation( + PackageRelationGroup(entry for entry in group + if 'cross' not in entry.restrictions) + for group in relations_compiler_headers) + for group in relations_compiler_headers: + for entry in group: + entry.restrictions = [] + + relations_compiler_build_dep = PackageRelation( + self.substitute(config_entry_relations[compiler], vars)) + for group in relations_compiler_build_dep: + for item in group: + item.arches = [arch] + packages['source']['Build-Depends-Arch'].extend( + relations_compiler_build_dep) + + image_fields = {'Description': PackageDescription()} + for field in ('Depends', 'Provides', 'Suggests', 'Recommends', + 'Conflicts', 'Breaks'): + image_fields[field] = PackageRelation( + config_entry_image(field.lower(), None), + override_arches=(arch,)) + + generators = config_entry_image('initramfs-generators') + group = PackageRelationGroup() + for i in generators: + i = config_entry_relations.get(i, i) + group.append(i) + a = PackageRelationEntry(i) + if a.operator is not None: + a.operator = -a.operator + image_fields['Breaks'].append(PackageRelationGroup([a])) + for item in group: + item.arches = [arch] + image_fields['Depends'].append(group) + + bootloaders = config_entry_image('bootloaders', None) + if bootloaders: + group = PackageRelationGroup() + for i in bootloaders: + i = config_entry_relations.get(i, i) + group.append(i) + a = PackageRelationEntry(i) + if a.operator is not None: + a.operator = -a.operator + image_fields['Breaks'].append(PackageRelationGroup([a])) + for item in group: + item.arches = [arch] + image_fields['Suggests'].append(group) + + desc_parts = self.config.get_merge('description', arch, featureset, + flavour, 'parts') + if desc_parts: + # XXX: Workaround, we need to support multiple entries of the same + # name + parts = list(set(desc_parts)) + parts.sort() + desc = image_fields['Description'] + for part in parts: + desc.append(config_entry_description['part-long-' + part]) + desc.append_short(config_entry_description + .get('part-short-' + part, '')) + + packages_own = [] + + if not self.disable_signed: + build_signed = config_entry_build.get('signed-code') + else: + build_signed = False + + image = self.templates[build_signed and "control.image-unsigned" + or "control.image"] + assert len(image) == 1 + + vars.setdefault('desc', None) + + image_main = self.process_real_image(image[0], image_fields, vars) + packages_own.append(image_main) + makeflags['IMAGE_PACKAGE_NAME'] = image_main['Package'] + + package_headers = self.process_package(headers[0], vars) + package_headers['Depends'].extend(relations_compiler_headers) + packages_own.append(package_headers) + if extra.get('headers_arch_depends'): + extra['headers_arch_depends'].append('%s (= ${binary:Version})' % + packages_own[-1]['Package']) + + if config_entry_build.get('vdso', False): + makeflags['VDSO'] = True + + if not self.disable_debug: + build_debug = config_entry_build.get('debug-info') + else: + build_debug = False + + if build_debug: + makeflags['DEBUG'] = True + packages_own.extend(self.process_packages( + self.templates['control.image-dbg'], vars)) + + merge_packages(packages, packages_own, arch) + + tests_control = self.process_package( + self.templates['tests-control.image'][0], vars) + tests_control['Depends'].append( + PackageRelationGroup(image_main['Package'], + override_arches=(arch,))) + if self.tests_control_image: + self.tests_control_image['Depends'].extend( + tests_control['Depends']) + else: + self.tests_control_image = tests_control + self.tests_control.append(tests_control) + + def get_config(*entry_name): + entry_real = ('image',) + entry_name + entry = self.config.get(entry_real, None) + if entry is None: + return None + return entry.get('configs', None) + + def check_config_default(fail, f): + for d in self.config_dirs[::-1]: + f1 = d + '/' + f + if os.path.exists(f1): + return [f1] + if fail: + raise RuntimeError("%s unavailable" % f) + return [] + + def check_config_files(files): + ret = [] + for f in files: + for d in self.config_dirs[::-1]: + f1 = d + '/' + f + if os.path.exists(f1): + ret.append(f1) + break + else: + raise RuntimeError("%s unavailable" % f) + return ret + + def check_config(default, fail, *entry_name): + configs = get_config(*entry_name) + if configs is None: + return check_config_default(fail, default) + return check_config_files(configs) + + kconfig = check_config('config', True) + kconfig.extend(check_config("kernelarch-%s/config" % + config_entry_base['kernel-arch'], + False)) + kconfig.extend(check_config("%s/config" % arch, True, arch)) + kconfig.extend(check_config("%s/config.%s" % (arch, flavour), False, + arch, None, flavour)) + kconfig.extend(check_config("featureset-%s/config" % featureset, False, + None, featureset)) + kconfig.extend(check_config("%s/%s/config" % (arch, featureset), False, + arch, featureset)) + kconfig.extend(check_config("%s/%s/config.%s" % + (arch, featureset, flavour), False, + arch, featureset, flavour)) + makeflags['KCONFIG'] = ' '.join(kconfig) + makeflags['KCONFIG_OPTIONS'] = '' + if build_debug: + makeflags['KCONFIG_OPTIONS'] += ' -o DEBUG_INFO=y' + if build_signed: + makeflags['KCONFIG_OPTIONS'] += ' -o MODULE_SIG=y' + # Add "salt" to fix #872263 + makeflags['KCONFIG_OPTIONS'] += (' -o "BUILD_SALT=\\"%s%s\\""' % + (vars['abiname'], + vars['localversion'])) + + cmds_binary_arch = ["$(MAKE) -f debian/rules.real binary-arch-flavour " + "%s" % + makeflags] + cmds_build = ["$(MAKE) -f debian/rules.real build-arch-flavour %s" % + makeflags] + cmds_setup = ["$(MAKE) -f debian/rules.real setup-arch-flavour %s" % + makeflags] + makefile.add('binary-arch_%s_%s_%s_real' % (arch, featureset, flavour), + cmds=cmds_binary_arch) + makefile.add('build-arch_%s_%s_%s_real' % (arch, featureset, flavour), + cmds=cmds_build) + makefile.add('setup_%s_%s_%s_real' % (arch, featureset, flavour), + cmds=cmds_setup) + + merged_config = ('debian/build/config.%s_%s_%s' % + (arch, featureset, flavour)) + makefile.add(merged_config, + cmds=["$(MAKE) -f debian/rules.real %s %s" % + (merged_config, makeflags)]) + + # Substitute kernel version etc. into maintainer scripts, + # translations and lintian overrides + self._substitute_file('headers.postinst', vars, + 'debian/linux-headers-%s%s.postinst' % + (vars['abiname'], vars['localversion'])) + for name in ['postinst', 'postrm', 'preinst', 'prerm']: + self._substitute_file('image.%s' % name, vars, + 'debian/%s.%s' % + (image_main['Package'], name)) + if build_debug: + debug_lintian_over = ( + 'debian/linux-image-%s%s-dbg.lintian-overrides' % + (vars['abiname'], vars['localversion'])) + self._substitute_file('image-dbg.lintian-overrides', vars, + debug_lintian_over) + os.chmod(debug_lintian_over, 0o755) + + def process_changelog(self): + version = self.version = self.changelog[0].version + if self.version.linux_modifier is not None: + self.abiname_part = '' + else: + self.abiname_part = '-%s' % self.config['abi', ]['abiname'] + # We need to keep at least three version components to avoid + # userland breakage (e.g. #742226, #745984). + self.abiname_version = re.sub(r'^(\d+\.\d+)(?=-|$)', r'\1.0', + self.version.linux_upstream) + self.vars = { + 'upstreamversion': self.version.linux_upstream, + 'version': self.version.linux_version, + 'source_basename': re.sub(r'-[\d.]+$', '', + self.changelog[0].source), + 'source_upstream': self.version.upstream, + 'source_package': self.changelog[0].source, + 'abiname': self.abiname_version + self.abiname_part, + } + self.config['version', ] = {'source': self.version.complete, + 'upstream': self.version.linux_upstream, + 'abiname_base': self.abiname_version, + 'abiname': (self.abiname_version + + self.abiname_part)} + + distribution = self.changelog[0].distribution + if distribution in ('unstable', ): + if version.linux_revision_experimental or \ + version.linux_revision_backports or \ + version.linux_revision_other: + raise RuntimeError("Can't upload to %s with a version of %s" % + (distribution, version)) + if distribution in ('experimental', ): + if not version.linux_revision_experimental: + raise RuntimeError("Can't upload to %s with a version of %s" % + (distribution, version)) + if distribution.endswith('-security') or distribution.endswith('-lts'): + if version.linux_revision_backports or \ + version.linux_revision_other: + raise RuntimeError("Can't upload to %s with a version of %s" % + (distribution, version)) + if distribution.endswith('-backports'): + if not version.linux_revision_backports: + raise RuntimeError("Can't upload to %s with a version of %s" % + (distribution, version)) + + def process_real_image(self, entry, fields, vars): + entry = self.process_package(entry, vars) + for key, value in fields.items(): + if key in entry: + real = entry[key] + real.extend(value) + elif value: + entry[key] = value + return entry + + def write(self, packages, makefile): + self.write_config() + super(Gencontrol, self).write(packages, makefile) + self.write_tests_control() + + def write_config(self): + f = open("debian/config.defines.dump", 'wb') + self.config.dump(f) + f.close() + + def write_tests_control(self): + self.write_rfc822(open("debian/tests/control", 'w'), + self.tests_control) + + +if __name__ == '__main__': + Gencontrol()() |