diff options
Diffstat (limited to 'debian/lib')
-rw-r--r-- | debian/lib/python/debian_linux/__init__.py | 1 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/abi.py | 42 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/config.py | 256 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/debian.py | 700 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/firmware.py | 90 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/gencontrol.py | 418 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/kconfig.py | 93 | ||||
-rw-r--r-- | debian/lib/python/debian_linux/utils.py | 97 |
8 files changed, 1697 insertions, 0 deletions
diff --git a/debian/lib/python/debian_linux/__init__.py b/debian/lib/python/debian_linux/__init__.py new file mode 100644 index 000000000..b785cebf7 --- /dev/null +++ b/debian/lib/python/debian_linux/__init__.py @@ -0,0 +1 @@ +# Module diff --git a/debian/lib/python/debian_linux/abi.py b/debian/lib/python/debian_linux/abi.py new file mode 100644 index 000000000..8532db16d --- /dev/null +++ b/debian/lib/python/debian_linux/abi.py @@ -0,0 +1,42 @@ +class Symbol(object): + def __init__(self, name, module, version, export): + self.name, self.module = name, module + self.version, self.export = version, export + + def __eq__(self, other): + if not isinstance(other, Symbol): + return NotImplemented + + # Symbols are resolved to modules by depmod at installation/ + # upgrade time, not compile time, so moving a symbol between + # modules is not an ABI change. Compare everything else. + if self.name != other.name: + return False + if self.version != other.version: + return False + if self.export != other.export: + return False + + return True + + def __ne__(self, other): + ret = self.__eq__(other) + if ret is NotImplemented: + return ret + return not ret + + +class Symbols(dict): + def __init__(self, file=None): + if file: + self.read(file) + + def read(self, file): + for line in file: + version, name, module, export = line.strip().split() + self[name] = Symbol(name, module, version, export) + + def write(self, file): + for s in sorted(self.values(), key=lambda i: i.name): + file.write("%s %s %s %s\n" % + (s.version, s.name, s.module, s.export)) diff --git a/debian/lib/python/debian_linux/config.py b/debian/lib/python/debian_linux/config.py new file mode 100644 index 000000000..37e3764ab --- /dev/null +++ b/debian/lib/python/debian_linux/config.py @@ -0,0 +1,256 @@ +import collections +import os +import os.path +import pickle +import re +import sys + +from configparser import RawConfigParser + +__all__ = [ + 'ConfigCoreDump', + 'ConfigCoreHierarchy', + 'ConfigParser', +] + + +class SchemaItemBoolean(object): + def __call__(self, i): + i = i.strip().lower() + if i in ("true", "1"): + return True + if i in ("false", "0"): + return False + raise ValueError + + +class SchemaItemInteger(object): + def __call__(self, i): + return int(i.strip(), 0) + + +class SchemaItemList(object): + def __init__(self, type=r"\s+"): + self.type = type + + def __call__(self, i): + i = i.strip() + if not i: + return [] + return [j.strip() for j in re.split(self.type, i)] + + +# Using OrderedDict instead of dict makes the pickled config reproducible +class ConfigCore(collections.OrderedDict): + def get_merge(self, section, arch, featureset, flavour, key, default=None): + temp = [] + + if arch and featureset and flavour: + temp.append(self.get((section, arch, featureset, flavour), {}) + .get(key)) + temp.append(self.get((section, arch, None, flavour), {}).get(key)) + if arch and featureset: + temp.append(self.get((section, arch, featureset), {}).get(key)) + if arch: + temp.append(self.get((section, arch), {}).get(key)) + if featureset: + temp.append(self.get((section, None, featureset), {}).get(key)) + temp.append(self.get((section,), {}).get(key)) + + ret = [] + + for i in temp: + if i is None: + continue + elif isinstance(i, (list, tuple)): + ret.extend(i) + elif ret: + # TODO + return ret + else: + return i + + return ret or default + + def merge(self, section, arch=None, featureset=None, flavour=None): + ret = {} + ret.update(self.get((section,), {})) + if featureset: + ret.update(self.get((section, None, featureset), {})) + if arch: + ret.update(self.get((section, arch), {})) + if arch and featureset: + ret.update(self.get((section, arch, featureset), {})) + if arch and featureset and flavour: + ret.update(self.get((section, arch, None, flavour), {})) + ret.update(self.get((section, arch, featureset, flavour), {})) + return ret + + def dump(self, fp): + pickle.dump(self, fp, 0) + + +class ConfigCoreDump(object): + def __new__(self, fp): + return pickle.load(fp) + + +class ConfigCoreHierarchy(object): + schema_base = { + 'base': { + 'arches': SchemaItemList(), + 'enabled': SchemaItemBoolean(), + 'featuresets': SchemaItemList(), + 'flavours': SchemaItemList(), + }, + } + + def __new__(cls, schema, dirs=[]): + schema_complete = cls.schema_base.copy() + for key, value in schema.items(): + schema_complete.setdefault(key, {}).update(value) + return cls.Reader(dirs, schema_complete)() + + class Reader(object): + config_name = "defines" + + def __init__(self, dirs, schema): + self.dirs, self.schema = dirs, schema + + def __call__(self): + ret = ConfigCore() + self.read(ret) + return ret + + def get_files(self, *dirs): + dirs = list(dirs) + dirs.append(self.config_name) + return (os.path.join(i, *dirs) for i in self.dirs if i) + + def read_arch(self, ret, arch): + config = ConfigParser(self.schema) + config.read(self.get_files(arch)) + + featuresets = config['base', ].get('featuresets', []) + flavours = config['base', ].get('flavours', []) + + for section in iter(config): + if section[0] in featuresets: + real = (section[-1], arch, section[0]) + elif len(section) > 1: + real = (section[-1], arch, None) + section[:-1] + else: + real = (section[-1], arch) + section[:-1] + s = ret.get(real, {}) + s.update(config[section]) + ret[tuple(real)] = s + + for featureset in featuresets: + self.read_arch_featureset(ret, arch, featureset) + + if flavours: + base = ret['base', arch] + featuresets.insert(0, 'none') + base['featuresets'] = featuresets + del base['flavours'] + ret['base', arch] = base + ret['base', arch, 'none'] = {'flavours': flavours, + 'implicit-flavour': True} + + def read_arch_featureset(self, ret, arch, featureset): + config = ConfigParser(self.schema) + config.read(self.get_files(arch, featureset)) + + for section in iter(config): + real = (section[-1], arch, featureset) + section[:-1] + s = ret.get(real, {}) + s.update(config[section]) + ret[tuple(real)] = s + + def read(self, ret): + config = ConfigParser(self.schema) + config.read(self.get_files()) + + arches = config['base', ]['arches'] + featuresets = config['base', ].get('featuresets', []) + + for section in iter(config): + if section[0].startswith('featureset-'): + real = (section[-1], None, section[0][11:]) + else: + real = (section[-1],) + section[1:] + ret[real] = config[section] + + for arch in arches: + self.read_arch(ret, arch) + for featureset in featuresets: + self.read_featureset(ret, featureset) + + def read_featureset(self, ret, featureset): + config = ConfigParser(self.schema) + config.read(self.get_files('featureset-%s' % featureset)) + + for section in iter(config): + real = (section[-1], None, featureset) + s = ret.get(real, {}) + s.update(config[section]) + ret[real] = s + + +class ConfigParser(object): + __slots__ = '_config', 'schemas' + + def __init__(self, schemas): + self.schemas = schemas + + self._config = RawConfigParser() + + def __getitem__(self, key): + return self._convert()[key] + + def __iter__(self): + return iter(self._convert()) + + def __str__(self): + return '<%s(%s)>' % (self.__class__.__name__, self._convert()) + + def _convert(self): + ret = {} + for section in self._config.sections(): + data = {} + for key, value in self._config.items(section): + data[key] = value + section_list = section.split('_') + section_base = section_list[-1] + if section_base in self.schemas: + section_ret = tuple(section_list) + data = self._convert_one(self.schemas[section_base], data) + else: + section_ret = (section, ) + ret[section_ret] = data + return ret + + def _convert_one(self, schema, data): + ret = {} + for key, value in data.items(): + if key in schema: + value = schema[key](value) + ret[key] = value + return ret + + def keys(self): + return self._convert().keys() + + def read(self, data): + return self._config.read(data) + + +if __name__ == '__main__': + sys.path.append('debian/lib/python') + config = ConfigCoreDump(open('debian/config.defines.dump', 'rb')) + for section, items in sorted(config.items(), + key=(lambda a: tuple(i or '' for i in a[0]))): + print(u"[%s]" % (section,)) + for item, value in sorted(items.items()): + print(u"%s: %s" % (item, value)) + print() diff --git a/debian/lib/python/debian_linux/debian.py b/debian/lib/python/debian_linux/debian.py new file mode 100644 index 000000000..6fb26185d --- /dev/null +++ b/debian/lib/python/debian_linux/debian.py @@ -0,0 +1,700 @@ +import collections +import collections.abc +import os.path +import re +import unittest + +from . import utils + + +class Changelog(list): + _top_rules = r""" +^ +(?P<source> + \w[-+0-9a-z.]+ +) +\ +\( +(?P<version> + [^\(\)\ \t]+ +) +\) +\s+ +(?P<distribution> + [-+0-9a-zA-Z.]+ +) +\;\s+urgency= +(?P<urgency> + \w+ +) +(?:,|\n) +""" + _top_re = re.compile(_top_rules, re.X) + _bottom_rules = r""" +^ +\ --\ +(?P<maintainer> + \S(?:\ ?\S)* +) +\ \ +(?P<date> + (.*) +) +\n +""" + _bottom_re = re.compile(_bottom_rules, re.X) + _ignore_re = re.compile(r'^(?: |\s*\n)') + + class Entry(object): + __slot__ = ('distribution', 'source', 'version', 'urgency', + 'maintainer', 'date') + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + def __init__(self, dir='', version=None, file=None): + if version is None: + version = Version + if file: + self._parse(version, file) + else: + with open(os.path.join(dir, "debian/changelog"), + encoding="UTF-8") as f: + self._parse(version, f) + + def _parse(self, version, f): + top_match = None + line_no = 0 + + for line in f: + line_no += 1 + + if self._ignore_re.match(line): + pass + elif top_match is None: + top_match = self._top_re.match(line) + if not top_match: + raise Exception('invalid top line %d in changelog' % + line_no) + try: + v = version(top_match.group('version')) + except Exception: + if not len(self): + raise + v = Version(top_match.group('version')) + else: + bottom_match = self._bottom_re.match(line) + if not bottom_match: + raise Exception('invalid bottom line %d in changelog' % + line_no) + + self.append(self.Entry( + distribution=top_match.group('distribution'), + source=top_match.group('source'), + version=v, + urgency=top_match.group('urgency'), + maintainer=bottom_match.group('maintainer'), + date=bottom_match.group('date'))) + top_match = bottom_match = None + + +class Version(object): + _epoch_re = re.compile(r'\d+$') + _upstream_re = re.compile(r'[0-9][A-Za-z0-9.+\-:~]*$') + _revision_re = re.compile(r'[A-Za-z0-9+.~]+$') + + def __init__(self, version): + try: + split = version.index(':') + except ValueError: + epoch, rest = None, version + else: + epoch, rest = version[0:split], version[split+1:] + try: + split = rest.rindex('-') + except ValueError: + upstream, revision = rest, None + else: + upstream, revision = rest[0:split], rest[split+1:] + if (epoch is not None and not self._epoch_re.match(epoch)) or \ + not self._upstream_re.match(upstream) or \ + (revision is not None and not self._revision_re.match(revision)): + raise RuntimeError(u"Invalid debian version") + self.epoch = epoch and int(epoch) + self.upstream = upstream + self.revision = revision + + def __str__(self): + return self.complete + + @property + def complete(self): + if self.epoch is not None: + return u"%d:%s" % (self.epoch, self.complete_noepoch) + return self.complete_noepoch + + @property + def complete_noepoch(self): + if self.revision is not None: + return u"%s-%s" % (self.upstream, self.revision) + return self.upstream + + @property + def debian(self): + from warnings import warn + warn(u"debian argument was replaced by revision", DeprecationWarning, + stacklevel=2) + return self.revision + + +class _VersionTest(unittest.TestCase): + def test_native(self): + v = Version('1.2+c~4') + self.assertEqual(v.epoch, None) + self.assertEqual(v.upstream, '1.2+c~4') + self.assertEqual(v.revision, None) + self.assertEqual(v.complete, '1.2+c~4') + self.assertEqual(v.complete_noepoch, '1.2+c~4') + + def test_nonnative(self): + v = Version('1-2+d~3') + self.assertEqual(v.epoch, None) + self.assertEqual(v.upstream, '1') + self.assertEqual(v.revision, '2+d~3') + self.assertEqual(v.complete, '1-2+d~3') + self.assertEqual(v.complete_noepoch, '1-2+d~3') + + def test_native_epoch(self): + v = Version('5:1.2.3') + self.assertEqual(v.epoch, 5) + self.assertEqual(v.upstream, '1.2.3') + self.assertEqual(v.revision, None) + self.assertEqual(v.complete, '5:1.2.3') + self.assertEqual(v.complete_noepoch, '1.2.3') + + def test_nonnative_epoch(self): + v = Version('5:1.2.3-4') + self.assertEqual(v.epoch, 5) + self.assertEqual(v.upstream, '1.2.3') + self.assertEqual(v.revision, '4') + self.assertEqual(v.complete, '5:1.2.3-4') + self.assertEqual(v.complete_noepoch, '1.2.3-4') + + def test_multi_hyphen(self): + v = Version('1-2-3') + self.assertEqual(v.epoch, None) + self.assertEqual(v.upstream, '1-2') + self.assertEqual(v.revision, '3') + self.assertEqual(v.complete, '1-2-3') + + def test_multi_colon(self): + v = Version('1:2:3') + self.assertEqual(v.epoch, 1) + self.assertEqual(v.upstream, '2:3') + self.assertEqual(v.revision, None) + + def test_invalid_epoch(self): + with self.assertRaises(RuntimeError): + Version('a:1') + with self.assertRaises(RuntimeError): + Version('-1:1') + with self.assertRaises(RuntimeError): + Version('1a:1') + + def test_invalid_upstream(self): + with self.assertRaises(RuntimeError): + Version('1_2') + with self.assertRaises(RuntimeError): + Version('1/2') + with self.assertRaises(RuntimeError): + Version('a1') + with self.assertRaises(RuntimeError): + Version('1 2') + + def test_invalid_revision(self): + with self.assertRaises(RuntimeError): + Version('1-2_3') + with self.assertRaises(RuntimeError): + Version('1-2/3') + with self.assertRaises(RuntimeError): + Version('1-2:3') + + +class VersionLinux(Version): + _upstream_re = re.compile(r""" +(?P<version> + \d+\.\d+ +) +(?P<update> + (?:\.\d+)? + (?:-[a-z]+\d+)? +) +(?: + ~ + (?P<modifier> + .+? + ) +)? +(?: + \.dfsg\. + (?P<dfsg> + \d+ + ) +)? +$ + """, re.X) + _revision_re = re.compile(r""" +\d+ +(\.\d+)? +(?: + (?P<revision_experimental> + ~exp\d+ + ) + | + (?P<revision_security> + (?:[~+]deb\d+u\d+)+ + )? + (?P<revision_backports> + ~bpo\d+\+\d+ + )? + | + (?P<revision_other> + .+? + ) +) +(?:\+b\d+)? +$ + """, re.X) + + def __init__(self, version): + super(VersionLinux, self).__init__(version) + up_match = self._upstream_re.match(self.upstream) + rev_match = self._revision_re.match(self.revision) + if up_match is None or rev_match is None: + raise RuntimeError(u"Invalid debian linux version") + d = up_match.groupdict() + self.linux_modifier = d['modifier'] + self.linux_version = d['version'] + if d['modifier'] is not None: + assert not d['update'] + self.linux_upstream = '-'.join((d['version'], d['modifier'])) + else: + self.linux_upstream = d['version'] + self.linux_upstream_full = self.linux_upstream + d['update'] + self.linux_dfsg = d['dfsg'] + d = rev_match.groupdict() + self.linux_revision_experimental = d['revision_experimental'] and True + self.linux_revision_security = d['revision_security'] and True + self.linux_revision_backports = d['revision_backports'] and True + self.linux_revision_other = d['revision_other'] and True + + +class _VersionLinuxTest(unittest.TestCase): + def test_stable(self): + v = VersionLinux('1.2.3-4') + self.assertEqual(v.linux_version, '1.2') + self.assertEqual(v.linux_upstream, '1.2') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertEqual(v.linux_modifier, None) + self.assertEqual(v.linux_dfsg, None) + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_rc(self): + v = VersionLinux('1.2~rc3-4') + self.assertEqual(v.linux_version, '1.2') + self.assertEqual(v.linux_upstream, '1.2-rc3') + self.assertEqual(v.linux_upstream_full, '1.2-rc3') + self.assertEqual(v.linux_modifier, 'rc3') + self.assertEqual(v.linux_dfsg, None) + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_dfsg(self): + v = VersionLinux('1.2~rc3.dfsg.1-4') + self.assertEqual(v.linux_version, '1.2') + self.assertEqual(v.linux_upstream, '1.2-rc3') + self.assertEqual(v.linux_upstream_full, '1.2-rc3') + self.assertEqual(v.linux_modifier, 'rc3') + self.assertEqual(v.linux_dfsg, '1') + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_experimental(self): + v = VersionLinux('1.2~rc3-4~exp5') + self.assertEqual(v.linux_upstream_full, '1.2-rc3') + self.assertTrue(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_security(self): + v = VersionLinux('1.2.3-4+deb10u1') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertFalse(v.linux_revision_experimental) + self.assertTrue(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_backports(self): + v = VersionLinux('1.2.3-4~bpo9+10') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertTrue(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_security_backports(self): + v = VersionLinux('1.2.3-4+deb10u1~bpo9+10') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertFalse(v.linux_revision_experimental) + self.assertTrue(v.linux_revision_security) + self.assertTrue(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_lts_backports(self): + # Backport during LTS, as an extra package in the -security + # suite. Since this is not part of a -backports suite it + # shouldn't get the linux_revision_backports flag. + v = VersionLinux('1.2.3-4~deb9u10') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertFalse(v.linux_revision_experimental) + self.assertTrue(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_lts_backports_2(self): + # Same but with two security extensions in the revision. + v = VersionLinux('1.2.3-4+deb10u1~deb9u10') + self.assertEqual(v.linux_upstream_full, '1.2.3') + self.assertFalse(v.linux_revision_experimental) + self.assertTrue(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_binnmu(self): + v = VersionLinux('1.2.3-4+b1') + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertFalse(v.linux_revision_other) + + def test_other_revision(self): + v = VersionLinux('4.16.5-1+revert+crng+ready') # from #898087 + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertTrue(v.linux_revision_other) + + def test_other_revision_binnmu(self): + v = VersionLinux('4.16.5-1+revert+crng+ready+b1') + self.assertFalse(v.linux_revision_experimental) + self.assertFalse(v.linux_revision_security) + self.assertFalse(v.linux_revision_backports) + self.assertTrue(v.linux_revision_other) + + +class PackageArchitecture(collections.abc.MutableSet): + __slots__ = '_data' + + def __init__(self, value=None): + self._data = set() + if value: + self.extend(value) + + def __contains__(self, value): + return self._data.__contains__(value) + + def __iter__(self): + return self._data.__iter__() + + def __len__(self): + return self._data.__len__() + + def __str__(self): + return ' '.join(sorted(self)) + + def add(self, value): + self._data.add(value) + + def discard(self, value): + self._data.discard(value) + + def extend(self, value): + if isinstance(value, str): + for i in re.split(r'\s', value.strip()): + self.add(i) + else: + raise RuntimeError + + +class PackageDescription(object): + __slots__ = "short", "long" + + def __init__(self, value=None): + self.short = [] + self.long = [] + if value is not None: + desc_split = value.split("\n", 1) + self.append_short(desc_split[0]) + if len(desc_split) == 2: + self.append(desc_split[1]) + + def __str__(self): + wrap = utils.TextWrapper(width=74, fix_sentence_endings=True).wrap + short = ', '.join(self.short) + long_pars = [] + for i in self.long: + long_pars.append(wrap(i)) + long = '\n .\n '.join(['\n '.join(i) for i in long_pars]) + return short + '\n ' + long if long else short + + def append(self, str): + str = str.strip() + if str: + self.long.extend(str.split(u"\n.\n")) + + def append_short(self, str): + for i in [i.strip() for i in str.split(u",")]: + if i: + self.short.append(i) + + def extend(self, desc): + if isinstance(desc, PackageDescription): + self.short.extend(desc.short) + self.long.extend(desc.long) + else: + raise TypeError + + +class PackageRelation(list): + def __init__(self, value=None, override_arches=None): + if value: + self.extend(value, override_arches) + + def __str__(self): + return ', '.join(str(i) for i in self) + + def _search_value(self, value): + for i in self: + if i._search_value(value): + return i + return None + + def append(self, value, override_arches=None): + if isinstance(value, str): + value = PackageRelationGroup(value, override_arches) + elif not isinstance(value, PackageRelationGroup): + raise ValueError(u"got %s" % type(value)) + j = self._search_value(value) + if j: + j._update_arches(value) + else: + super(PackageRelation, self).append(value) + + def extend(self, value, override_arches=None): + if isinstance(value, str): + value = (j.strip() for j in re.split(r',', value.strip())) + for i in value: + self.append(i, override_arches) + + +class PackageRelationGroup(list): + def __init__(self, value=None, override_arches=None): + if value: + self.extend(value, override_arches) + + def __str__(self): + return ' | '.join(str(i) for i in self) + + def _search_value(self, value): + for i, j in zip(self, value): + if i.name != j.name or i.operator != j.operator or \ + i.version != j.version or i.restrictions != j.restrictions: + return None + return self + + def _update_arches(self, value): + for i, j in zip(self, value): + if i.arches: + for arch in j.arches: + if arch not in i.arches: + i.arches.append(arch) + + def append(self, value, override_arches=None): + if isinstance(value, str): + value = PackageRelationEntry(value, override_arches) + elif not isinstance(value, PackageRelationEntry): + raise ValueError + super(PackageRelationGroup, self).append(value) + + def extend(self, value, override_arches=None): + if isinstance(value, str): + value = (j.strip() for j in re.split(r'\|', value.strip())) + for i in value: + self.append(i, override_arches) + + +class PackageRelationEntry(object): + __slots__ = "name", "operator", "version", "arches", "restrictions" + + _re = re.compile(r'^(\S+)(?: \((<<|<=|=|!=|>=|>>)\s*([^)]+)\))?' + r'(?: \[([^]]+)\])?(?: <([^>]+)>)?$') + + class _operator(object): + OP_LT = 1 + OP_LE = 2 + OP_EQ = 3 + OP_NE = 4 + OP_GE = 5 + OP_GT = 6 + + operators = { + '<<': OP_LT, + '<=': OP_LE, + '=': OP_EQ, + '!=': OP_NE, + '>=': OP_GE, + '>>': OP_GT, + } + + operators_neg = { + OP_LT: OP_GE, + OP_LE: OP_GT, + OP_EQ: OP_NE, + OP_NE: OP_EQ, + OP_GE: OP_LT, + OP_GT: OP_LE, + } + + operators_text = dict((b, a) for a, b in operators.items()) + + __slots__ = '_op', + + def __init__(self, value): + self._op = self.operators[value] + + def __neg__(self): + return self.__class__( + self.operators_text[self.operators_neg[self._op]]) + + def __str__(self): + return self.operators_text[self._op] + + def __eq__(self, other): + return type(other) == type(self) and self._op == other._op + + def __init__(self, value=None, override_arches=None): + if not isinstance(value, str): + raise ValueError + + self.parse(value) + + if override_arches: + self.arches = list(override_arches) + + def __str__(self): + ret = [self.name] + if self.operator is not None and self.version is not None: + ret.extend((' (', str(self.operator), ' ', self.version, ')')) + if self.arches: + ret.extend((' [', ' '.join(self.arches), ']')) + if self.restrictions: + ret.extend((' <', ' '.join(self.restrictions), '>')) + return ''.join(ret) + + def parse(self, value): + match = self._re.match(value) + if match is None: + raise RuntimeError(u"Can't parse dependency %s" % value) + match = match.groups() + self.name = match[0] + if match[1] is not None: + self.operator = self._operator(match[1]) + else: + self.operator = None + self.version = match[2] + if match[3] is not None: + self.arches = re.split(r'\s+', match[3]) + else: + self.arches = [] + if match[4] is not None: + self.restrictions = re.split(r'\s+', match[4]) + else: + self.restrictions = [] + + +class _ControlFileDict(dict): + def __setitem__(self, key, value): + try: + cls = self._fields[key] + if not isinstance(value, cls): + value = cls(value) + except KeyError: + pass + super(_ControlFileDict, self).__setitem__(key, value) + + def keys(self): + keys = set(super(_ControlFileDict, self).keys()) + for i in self._fields.keys(): + if i in self: + keys.remove(i) + yield i + for i in sorted(list(keys)): + yield i + + def items(self): + for i in self.keys(): + yield (i, self[i]) + + def values(self): + for i in self.keys(): + yield self[i] + + +class Package(_ControlFileDict): + _fields = collections.OrderedDict(( + ('Package', str), + ('Source', str), + ('Architecture', PackageArchitecture), + ('Section', str), + ('Priority', str), + ('Maintainer', str), + ('Uploaders', str), + ('Standards-Version', str), + ('Build-Depends', PackageRelation), + ('Build-Depends-Arch', PackageRelation), + ('Build-Depends-Indep', PackageRelation), + ('Provides', PackageRelation), + ('Pre-Depends', PackageRelation), + ('Depends', PackageRelation), + ('Recommends', PackageRelation), + ('Suggests', PackageRelation), + ('Replaces', PackageRelation), + ('Breaks', PackageRelation), + ('Conflicts', PackageRelation), + ('Description', PackageDescription), + )) + + +class TestsControl(_ControlFileDict): + _fields = collections.OrderedDict(( + ('Tests', str), + ('Test-Command', str), + ('Restrictions', str), + ('Features', str), + ('Depends', PackageRelation), + ('Tests-Directory', str), + ('Classes', str), + )) + + +if __name__ == '__main__': + unittest.main() diff --git a/debian/lib/python/debian_linux/firmware.py b/debian/lib/python/debian_linux/firmware.py new file mode 100644 index 000000000..592a66a45 --- /dev/null +++ b/debian/lib/python/debian_linux/firmware.py @@ -0,0 +1,90 @@ +import re + + +class FirmwareFile(object): + def __init__(self, binary, desc=None, source=None, version=None): + self.binary = binary + self.desc = desc + self.source = source + self.version = version + + +class FirmwareSection(object): + def __init__(self, driver, files, licence): + self.driver = driver + self.files = files + self.licence = licence + + +class FirmwareWhence(list): + def __init__(self, file): + self.read(file) + + def read(self, file): + in_header = True + driver = None + files = {} + licence = None + binary = [] + desc = None + source = [] + version = None + + for line in file: + if line.startswith('----------'): + if in_header: + in_header = False + else: + # Finish old section + if driver: + self.append(FirmwareSection(driver, files, licence)) + driver = None + files = {} + licence = None + continue + + if in_header: + continue + + if line == '\n': + # End of field; end of file fields + for b in binary: + # XXX The WHENCE file isn't yet consistent in its + # association of binaries and their sources and + # metadata. This associates all sources and + # metadata in a group with each binary. + files[b] = FirmwareFile(b, desc, source, version) + binary = [] + desc = None + source = [] + version = None + continue + + match = re.match( + r'(Driver|File|Info|Licen[cs]e|Source|Version' + r'|Original licen[cs]e info(?:rmation)?):\s*(.*)\n', + line) + if match: + keyword, value = match.group(1, 2) + if keyword == 'Driver': + driver = value.split(' ')[0].lower() + elif keyword == 'File': + match = re.match(r'(\S+)(?:\s+--\s+(.*))?', value) + binary.append(match.group(1)) + desc = match.group(2) + elif keyword in ['Info', 'Version']: + version = value + elif keyword == 'Source': + source.append(value) + else: + licence = value + elif licence is not None: + licence = (licence + '\n' + + re.sub(r'^(?:[/ ]\*| \*/)?\s*(.*?)\s*$', r'\1', + line)) + + # Finish last section if non-empty + for b in binary: + files[b] = FirmwareFile(b, desc, source, version) + if driver: + self.append(FirmwareSection(driver, files, licence)) diff --git a/debian/lib/python/debian_linux/gencontrol.py b/debian/lib/python/debian_linux/gencontrol.py new file mode 100644 index 000000000..69619da3e --- /dev/null +++ b/debian/lib/python/debian_linux/gencontrol.py @@ -0,0 +1,418 @@ +import codecs +import re +from collections import OrderedDict + +from .debian import Changelog, PackageArchitecture, PackageDescription, \ + PackageRelation, Version + + +class PackagesList(OrderedDict): + def append(self, package): + self[package['Package']] = package + + def extend(self, packages): + for package in packages: + self[package['Package']] = package + + +class Makefile(object): + def __init__(self): + self.rules = {} + self.add('.NOTPARALLEL') + + def add(self, name, deps=None, cmds=None): + if name in self.rules: + self.rules[name].add(deps, cmds) + else: + self.rules[name] = self.Rule(name, deps, cmds) + if deps is not None: + for i in deps: + if i not in self.rules: + self.rules[i] = self.Rule(i) + + def write(self, out): + for i in sorted(self.rules.keys()): + self.rules[i].write(out) + + class Rule(object): + def __init__(self, name, deps=None, cmds=None): + self.name = name + self.deps, self.cmds = set(), [] + self.add(deps, cmds) + + def add(self, deps=None, cmds=None): + if deps is not None: + self.deps.update(deps) + if cmds is not None: + self.cmds.append(cmds) + + def write(self, out): + deps_string = '' + if self.deps: + deps = list(self.deps) + deps.sort() + deps_string = ' ' + ' '.join(deps) + + if self.cmds: + if deps_string: + out.write('%s::%s\n' % (self.name, deps_string)) + for c in self.cmds: + out.write('%s::\n' % self.name) + for i in c: + out.write('\t%s\n' % i) + else: + out.write('%s:%s\n' % (self.name, deps_string)) + + +class MakeFlags(dict): + def __str__(self): + return ' '.join("%s='%s'" % i for i in sorted(self.items())) + + def copy(self): + return self.__class__(super(MakeFlags, self).copy()) + + +def iter_featuresets(config): + for featureset in config['base', ]['featuresets']: + if config.merge('base', None, featureset).get('enabled', True): + yield featureset + + +def iter_arches(config): + return iter(config['base', ]['arches']) + + +def iter_arch_featuresets(config, arch): + for featureset in config['base', arch].get('featuresets', []): + if config.merge('base', arch, featureset).get('enabled', True): + yield featureset + + +def iter_flavours(config, arch, featureset): + return iter(config['base', arch, featureset]['flavours']) + + +class Gencontrol(object): + makefile_targets = ('binary-arch', 'build-arch', 'setup') + makefile_targets_indep = ('binary-indep', 'build-indep', 'setup') + + def __init__(self, config, templates, version=Version): + self.config, self.templates = config, templates + self.changelog = Changelog(version=version) + self.vars = {} + + def __call__(self): + packages = PackagesList() + makefile = Makefile() + + self.do_source(packages) + self.do_main(packages, makefile) + self.do_extra(packages, makefile) + + self.merge_build_depends(packages) + self.write(packages, makefile) + + def do_source(self, packages): + source = self.templates["control.source"][0] + if not source.get('Source'): + source['Source'] = self.changelog[0].source + packages['source'] = self.process_package(source, self.vars) + + def do_main(self, packages, makefile): + vars = self.vars.copy() + + makeflags = MakeFlags() + extra = {} + + self.do_main_setup(vars, makeflags, extra) + self.do_main_makefile(makefile, makeflags, extra) + self.do_main_packages(packages, vars, makeflags, extra) + self.do_main_recurse(packages, makefile, vars, makeflags, extra) + + def do_main_setup(self, vars, makeflags, extra): + pass + + def do_main_makefile(self, makefile, makeflags, extra): + makefile.add('build-indep', + cmds=["$(MAKE) -f debian/rules.real build-indep %s" % + makeflags]) + makefile.add('binary-indep', + cmds=["$(MAKE) -f debian/rules.real binary-indep %s" % + makeflags]) + + def do_main_packages(self, packages, vars, makeflags, extra): + pass + + def do_main_recurse(self, packages, makefile, vars, makeflags, extra): + for featureset in iter_featuresets(self.config): + self.do_indep_featureset(packages, makefile, featureset, + vars.copy(), makeflags.copy(), extra) + for arch in iter_arches(self.config): + self.do_arch(packages, makefile, arch, vars.copy(), + makeflags.copy(), extra) + + def do_extra(self, packages, makefile): + templates_extra = self.templates.get("control.extra", None) + if templates_extra is None: + return + + packages_extra = self.process_packages(templates_extra, self.vars) + packages.extend(packages_extra) + extra_arches = {} + for package in packages_extra: + arches = package['Architecture'] + for arch in arches: + i = extra_arches.get(arch, []) + i.append(package) + extra_arches[arch] = i + for arch in sorted(extra_arches.keys()): + cmds = [] + for i in extra_arches[arch]: + cmds.append("$(MAKE) -f debian/rules.real install-dummy " + "ARCH='%s' DH_OPTIONS='-p%s'" % + (arch, i['Package'])) + makefile.add('binary-arch_%s' % arch, + ['binary-arch_%s_extra' % arch]) + makefile.add("binary-arch_%s_extra" % arch, cmds=cmds) + + def do_indep_featureset(self, packages, makefile, featureset, vars, + makeflags, extra): + vars['localversion'] = '' + if featureset != 'none': + vars['localversion'] = '-' + featureset + + self.do_indep_featureset_setup(vars, makeflags, featureset, extra) + self.do_indep_featureset_makefile(makefile, featureset, makeflags, + extra) + self.do_indep_featureset_packages(packages, makefile, featureset, + vars, makeflags, extra) + + def do_indep_featureset_setup(self, vars, makeflags, featureset, extra): + pass + + def do_indep_featureset_makefile(self, makefile, featureset, makeflags, + extra): + makeflags['FEATURESET'] = featureset + + for i in self.makefile_targets_indep: + target1 = i + target2 = '_'.join((target1, featureset)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_indep_featureset_packages(self, packages, makefile, featureset, + vars, makeflags, extra): + pass + + def do_arch(self, packages, makefile, arch, vars, makeflags, extra): + vars['arch'] = arch + + self.do_arch_setup(vars, makeflags, arch, extra) + self.do_arch_makefile(makefile, arch, makeflags, extra) + self.do_arch_packages(packages, makefile, arch, vars, makeflags, extra) + self.do_arch_recurse(packages, makefile, arch, vars, makeflags, extra) + + def do_arch_setup(self, vars, makeflags, arch, extra): + pass + + def do_arch_makefile(self, makefile, arch, makeflags, extra): + makeflags['ARCH'] = arch + + for i in self.makefile_targets: + target1 = i + target2 = '_'.join((target1, arch)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_arch_packages(self, packages, makefile, arch, vars, makeflags, + extra): + pass + + def do_arch_recurse(self, packages, makefile, arch, vars, makeflags, + extra): + for featureset in iter_arch_featuresets(self.config, arch): + self.do_featureset(packages, makefile, arch, featureset, + vars.copy(), makeflags.copy(), extra) + + def do_featureset(self, packages, makefile, arch, featureset, vars, + makeflags, extra): + vars['localversion'] = '' + if featureset != 'none': + vars['localversion'] = '-' + featureset + + self.do_featureset_setup(vars, makeflags, arch, featureset, extra) + self.do_featureset_makefile(makefile, arch, featureset, makeflags, + extra) + self.do_featureset_packages(packages, makefile, arch, featureset, vars, + makeflags, extra) + self.do_featureset_recurse(packages, makefile, arch, featureset, vars, + makeflags, extra) + + def do_featureset_setup(self, vars, makeflags, arch, featureset, extra): + pass + + def do_featureset_makefile(self, makefile, arch, featureset, makeflags, + extra): + makeflags['FEATURESET'] = featureset + + for i in self.makefile_targets: + target1 = '_'.join((i, arch)) + target2 = '_'.join((target1, featureset)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_featureset_packages(self, packages, makefile, arch, featureset, + vars, makeflags, extra): + pass + + def do_featureset_recurse(self, packages, makefile, arch, featureset, vars, + makeflags, extra): + for flavour in iter_flavours(self.config, arch, featureset): + self.do_flavour(packages, makefile, arch, featureset, flavour, + vars.copy(), makeflags.copy(), extra) + + def do_flavour(self, packages, makefile, arch, featureset, flavour, vars, + makeflags, extra): + vars['localversion'] += '-' + flavour + + self.do_flavour_setup(vars, makeflags, arch, featureset, flavour, + extra) + self.do_flavour_makefile(makefile, arch, featureset, flavour, + makeflags, extra) + self.do_flavour_packages(packages, makefile, arch, featureset, flavour, + vars, makeflags, extra) + + def do_flavour_setup(self, vars, makeflags, arch, featureset, flavour, + extra): + for i in ( + ('kernel-arch', 'KERNEL_ARCH'), + ('localversion', 'LOCALVERSION'), + ): + if i[0] in vars: + makeflags[i[1]] = vars[i[0]] + + def do_flavour_makefile(self, makefile, arch, featureset, flavour, + makeflags, extra): + makeflags['FLAVOUR'] = flavour + + for i in self.makefile_targets: + target1 = '_'.join((i, arch, featureset)) + target2 = '_'.join((target1, flavour)) + target3 = '_'.join((target2, 'real')) + makefile.add(target1, [target2]) + makefile.add(target2, [target3]) + + def do_flavour_packages(self, packages, makefile, arch, featureset, + flavour, vars, makeflags, extra): + pass + + def process_relation(self, dep, vars): + import copy + dep = copy.deepcopy(dep) + for groups in dep: + for item in groups: + item.name = self.substitute(item.name, vars) + if item.version: + item.version = self.substitute(item.version, vars) + return dep + + def process_description(self, in_desc, vars): + desc = in_desc.__class__() + desc.short = self.substitute(in_desc.short, vars) + for i in in_desc.long: + desc.append(self.substitute(i, vars)) + return desc + + def process_package(self, in_entry, vars={}): + entry = in_entry.__class__() + for key, value in in_entry.items(): + if isinstance(value, PackageRelation): + value = self.process_relation(value, vars) + elif isinstance(value, PackageDescription): + value = self.process_description(value, vars) + else: + value = self.substitute(value, vars) + entry[key] = value + return entry + + def process_packages(self, entries, vars): + return [self.process_package(i, vars) for i in entries] + + def substitute(self, s, vars): + if isinstance(s, (list, tuple)): + return [self.substitute(i, vars) for i in s] + + def subst(match): + return vars[match.group(1)] + + return re.sub(r'@([-_a-z0-9]+)@', subst, str(s)) + + def merge_build_depends(self, packages): + # Merge Build-Depends pseudo-fields from binary packages into the + # source package + source = packages["source"] + arch_all = PackageArchitecture("all") + for name, package in packages.items(): + if name == "source": + continue + dep = package.get("Build-Depends") + if not dep: + continue + del package["Build-Depends"] + for group in dep: + for item in group: + if package["Architecture"] != arch_all and not item.arches: + item.arches = sorted(package["Architecture"]) + if package.get("Build-Profiles") and not item.restrictions: + profiles = package["Build-Profiles"] + assert profiles[0] == "<" and profiles[-1] == ">" + item.restrictions = re.split(r"\s+", profiles[1:-1]) + if package["Architecture"] == arch_all: + dep_type = "Build-Depends-Indep" + else: + dep_type = "Build-Depends-Arch" + if dep_type not in source: + source[dep_type] = PackageRelation() + source[dep_type].extend(dep) + + def write(self, packages, makefile): + self.write_control(packages.values()) + self.write_makefile(makefile) + + def write_control(self, list, name='debian/control'): + self.write_rfc822(codecs.open(name, 'w', 'utf-8'), list) + + def write_makefile(self, makefile, name='debian/rules.gen'): + f = open(name, 'w') + makefile.write(f) + f.close() + + def write_rfc822(self, f, list): + for entry in list: + for key, value in entry.items(): + f.write(u"%s: %s\n" % (key, value)) + f.write('\n') + + +def merge_packages(packages, new, arch): + for new_package in new: + name = new_package['Package'] + if name in packages: + package = packages.get(name) + package['Architecture'].add(arch) + + for field in ('Depends', 'Provides', 'Suggests', 'Recommends', + 'Conflicts'): + if field in new_package: + if field in package: + v = package[field] + v.extend(new_package[field]) + else: + package[field] = new_package[field] + + else: + new_package['Architecture'] = arch + packages.append(new_package) diff --git a/debian/lib/python/debian_linux/kconfig.py b/debian/lib/python/debian_linux/kconfig.py new file mode 100644 index 000000000..73d491d7f --- /dev/null +++ b/debian/lib/python/debian_linux/kconfig.py @@ -0,0 +1,93 @@ +from collections import OrderedDict + +__all__ = ( + "KconfigFile", +) + + +class KConfigEntry(object): + __slots__ = 'name', 'value', 'comments' + + def __init__(self, name, value, comments=None): + self.name, self.value = name, value + self.comments = comments or [] + + def __eq__(self, other): + return self.name == other.name and self.value == other.value + + def __hash__(self): + return hash(self.name) | hash(self.value) + + def __repr__(self): + return ('<{}({!r}, {!r}, {!r})>' + .format(self.__class__.__name__, self.name, self.value, + self.comments)) + + def __str__(self): + return 'CONFIG_{}={}'.format(self.name, self.value) + + def write(self): + for comment in self.comments: + yield '#. ' + comment + yield str(self) + + +class KConfigEntryTristate(KConfigEntry): + __slots__ = () + + VALUE_NO = False + VALUE_YES = True + VALUE_MOD = object() + + def __init__(self, name, value, comments=None): + if value == 'n' or value is None: + value = self.VALUE_NO + elif value == 'y': + value = self.VALUE_YES + elif value == 'm': + value = self.VALUE_MOD + else: + raise NotImplementedError + super(KConfigEntryTristate, self).__init__(name, value, comments) + + def __str__(self): + if self.value is self.VALUE_MOD: + return 'CONFIG_{}=m'.format(self.name) + if self.value: + return 'CONFIG_{}=y'.format(self.name) + return '# CONFIG_{} is not set'.format(self.name) + + +class KconfigFile(OrderedDict): + def __str__(self): + ret = [] + for i in self.str_iter(): + ret.append(i) + return '\n'.join(ret) + '\n' + + def read(self, f): + for line in iter(f.readlines()): + line = line.strip() + if line.startswith("CONFIG_"): + i = line.find('=') + option = line[7:i] + value = line[i + 1:] + self.set(option, value) + elif line.startswith("# CONFIG_"): + option = line[9:-11] + self.set(option, 'n') + elif line.startswith("#") or not line: + pass + else: + raise RuntimeError("Can't recognize %s" % line) + + def set(self, key, value): + if value in ('y', 'm', 'n'): + entry = KConfigEntryTristate(key, value) + else: + entry = KConfigEntry(key, value) + self[key] = entry + + def str_iter(self): + for key, value in self.items(): + yield str(value) diff --git a/debian/lib/python/debian_linux/utils.py b/debian/lib/python/debian_linux/utils.py new file mode 100644 index 000000000..34c15536d --- /dev/null +++ b/debian/lib/python/debian_linux/utils.py @@ -0,0 +1,97 @@ +import codecs +import os +import re +import textwrap + + +class Templates(object): + def __init__(self, dirs=["debian/templates"]): + self.dirs = dirs + + self._cache = {} + + def __getitem__(self, key): + ret = self.get(key) + if ret is not None: + return ret + raise KeyError(key) + + def _read(self, name): + prefix, id = name.split('.', 1) + + for suffix in ['.in', '']: + for dir in self.dirs: + filename = "%s/%s%s" % (dir, name, suffix) + if os.path.exists(filename): + f = codecs.open(filename, 'r', 'utf-8') + if prefix == 'control': + return read_control(f) + if prefix == 'tests-control': + return read_tests_control(f) + return f.read() + + def get(self, key, default=None): + if key in self._cache: + return self._cache[key] + + value = self._cache.setdefault(key, self._read(key)) + if value is None: + return default + return value + + +def read_control(f): + from .debian import Package + return _read_rfc822(f, Package) + + +def read_tests_control(f): + from .debian import TestsControl + return _read_rfc822(f, TestsControl) + + +def _read_rfc822(f, cls): + entries = [] + eof = False + + while not eof: + e = cls() + last = None + lines = [] + while True: + line = f.readline() + if not line: + eof = True + break + # Strip comments rather than trying to preserve them + if line[0] == '#': + continue + line = line.strip('\n') + if not line: + break + if line[0] in ' \t': + if not last: + raise ValueError( + 'Continuation line seen before first header') + lines.append(line.lstrip()) + continue + if last: + e[last] = '\n'.join(lines) + i = line.find(':') + if i < 0: + raise ValueError(u"Not a header, not a continuation: ``%s''" % + line) + last = line[:i] + lines = [line[i + 1:].lstrip()] + if last: + e[last] = '\n'.join(lines) + if e: + entries.append(e) + + return entries + + +class TextWrapper(textwrap.TextWrapper): + wordsep_re = re.compile( + r'(\s+|' # any whitespace + r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash |