diff options
Diffstat (limited to 'bootstrap/config.py')
-rw-r--r-- | bootstrap/config.py | 688 |
1 files changed, 688 insertions, 0 deletions
diff --git a/bootstrap/config.py b/bootstrap/config.py new file mode 100644 index 0000000..11d8314 --- /dev/null +++ b/bootstrap/config.py @@ -0,0 +1,688 @@ +#!/usr/bin/env python3 + +# Copyright (C) Catalyst.Net Ltd 2019 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Manage dependencies and bootstrap environments for Samba. + +Config file for packages and templates. + +Update the lists in this file to require new packages in the +container images used in GitLab CI + +Author: Joe Guo <joeg@catalyst.net.nz> +""" +import os +from os.path import abspath, dirname, join +HERE = abspath(dirname(__file__)) +# output dir for rendered files +OUT = join(HERE, 'generated-dists') + + +# pkgs with same name in all packaging systems +COMMON = [ + 'acl', + 'attr', + 'autoconf', + 'binutils', + 'bison', + 'ccache', + 'curl', + 'chrpath', + 'codespell', + 'flex', + 'gcc', + 'gdb', + 'git', + 'gzip', + 'hostname', + 'htop', + 'jq', + 'lcov', + 'make', + 'patch', + 'perl', + 'psmisc', # for pstree in test + 'rng-tools', + 'rsync', + 'sed', + 'shfmt', + 'sudo', # docker images has no sudo by default + 'tar', + 'tree', + 'wget', +] + + +# define pkgs for all packaging systems in parallel +# make it easier to find missing ones +# use latest ubuntu and fedora as defaults +# deb, rpm, ... +PKGS = [ + # NAME1-dev, NAME2-devel + ('lmdb-utils', 'lmdb'), + ('mingw-w64', 'mingw64-gcc'), + ('zlib1g-dev', 'zlib-devel'), + ('libbsd-dev', 'libbsd-devel'), + ('liburing-dev', 'liburing-devel'), + ('libarchive-dev', 'libarchive-devel'), + ('libblkid-dev', 'libblkid-devel'), + ('libcap-dev', 'libcap-devel'), + ('libacl1-dev', 'libacl-devel'), + ('libattr1-dev', 'libattr-devel'), + + # libNAME1-dev, NAME2-devel + ('libpopt-dev', 'popt-devel'), + ('libreadline-dev', 'readline-devel'), + ('libjansson-dev', 'jansson-devel'), + ('liblmdb-dev', 'lmdb-devel'), + ('libncurses5-dev', 'ncurses-devel'), + # NOTE: Debian 7+ or Ubuntu 16.04+ + ('libsystemd-dev', 'systemd-devel'), + ('libkrb5-dev', 'krb5-devel'), + ('libldap2-dev', 'openldap-devel'), + ('libcups2-dev', 'cups-devel'), + ('libpam0g-dev', 'pam-devel'), + ('libgpgme11-dev', 'gpgme-devel'), + # NOTE: Debian 8+ and Ubuntu 14.04+ + ('libgnutls28-dev', 'gnutls-devel'), + ('gnutls-bin', 'gnutls-utils'), + ('libtasn1-bin', 'libtasn1-tools'), + ('libtasn1-dev', 'libtasn1-devel'), + ('', 'quota-devel'), + ('uuid-dev', 'libuuid-devel'), + ('libjs-jquery', ''), + ('libavahi-common-dev', 'avahi-devel'), + ('libdbus-1-dev', 'dbus-devel'), + ('libpcap-dev', 'libpcap-devel'), + ('libunwind-dev', 'libunwind-devel'), # for back trace + ('libglib2.0-dev', 'glib2-devel'), + ('libicu-dev', 'libicu-devel'), + ('heimdal-multidev', ''), + + # NAME1, NAME2 + # for debian, locales provide locale support with language packs + # ubuntu split language packs to language-pack-xx + # for centos, glibc-common provide locale support with language packs + # fedora split language packs to glibc-langpack-xx + ('locales', 'glibc-common'), # required for locale + ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8 + ('bind9utils', 'bind-utils'), + ('dnsutils', ''), + ('xsltproc', 'libxslt'), + ('krb5-user', 'krb5-workstation'), + ('krb5-config', ''), + ('krb5-kdc', 'krb5-server'), + ('apt-utils', 'yum-utils'), + ('pkg-config', 'pkgconfig'), + ('procps', 'procps-ng'), # required for the free cmd in tests + ('lsb-release', 'lsb-release'), # we need lsb_release to show info + ('', 'rpcgen'), # required for test + # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval + ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora + ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header + ('mawk', 'gawk'), + ('', 'mold'), + ('shellcheck', 'ShellCheck'), + ('', 'crypto-policies-scripts'), + + ('python3', 'python3'), + ('python3-cryptography', 'python3-cryptography'), # for krb5 tests + ('python3-dev', 'python3-devel'), + ('python3-dbg', ''), + ('python3-iso8601', 'python3-iso8601'), + ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest + ('python3-markdown', 'python3-markdown'), + ('python3-dnspython', 'python3-dns'), + ('python3-pexpect', ''), # for wintest only + ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests + ('python3-setproctitle', 'python3-setproctitle'), + ('python3-requests', 'python3-requests'), # for cert auto enroll + + ('', 'python3-libsemanage'), + ('', 'python3-policycoreutils'), + + # perl + ('libparse-yapp-perl', 'perl-Parse-Yapp'), + ('perl-modules', ''), + ('', 'perl-FindBin'), + ('', 'perl-Archive-Tar'), + ('', 'perl-ExtUtils-MakeMaker'), + ('', 'perl-Test-Base'), + ('', 'perl-generators'), + ('', 'perl-interpreter'), + + # fs + ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support + ('', 'glusterfs-api-devel'), + ('glusterfs-common', 'glusterfs-devel'), + ('libcephfs-dev', 'libcephfs-devel'), + + # spotlight + ('libtracker-sparql-2.0-dev', 'tracker-devel'), + + # misc + # @ means group for rpm, use fedora as rpm default + ('build-essential', '@development-tools'), + ('debhelper', ''), + # rpm has no pkg for docbook-xml + ('docbook-xml', 'docbook-dtds'), + ('docbook-xsl', 'docbook-style-xsl'), + ('libkeyutils-dev', 'keyutils-libs-devel'), + ('', 'which'), + ('xz-utils', 'xz') +] + + +DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg] +RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg] + +GENERATED_MARKER = r""" +# +# This file is generated by 'bootstrap/template.py --render' +# See also bootstrap/config.py +# +""" + + +APT_BOOTSTRAP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +export DEBIAN_FRONTEND=noninteractive +apt-get -y update + +apt-get -y install \ + {pkgs} + +apt-get -y autoremove +apt-get -y autoclean +apt-get -y clean +""" + + +YUM_BOOTSTRAP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +yum update -y +yum install -y epel-release +yum install -y yum-plugin-copr +yum copr enable -y sergiomb/SambaAD +yum update -y + +yum install -y \ + {pkgs} + +yum clean all + +if [ ! -f /usr/bin/python3 ]; then + ln -sf /usr/bin/python3.6 /usr/bin/python3 +fi +""" + +CENTOS8S_YUM_BOOTSTRAP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +yum update -y +yum install -y dnf-plugins-core +yum install -y epel-release + +yum -v repolist all +yum config-manager --set-enabled powertools -y + +yum update -y + +yum install -y \ + --setopt=install_weak_deps=False \ + {pkgs} + +yum clean all +""" + +DNF_BOOTSTRAP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +dnf update -y + +dnf install -y \ + --setopt=install_weak_deps=False \ + {pkgs} + +dnf clean all + +update-crypto-policies --set DEFAULT:AD-SUPPORT +""" + +DNF_BOOTSTRAP_MIT = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +dnf update -y +dnf install -y dnf-plugins-core +dnf copr -y enable abbra/krb5-test +dnf update -y + +dnf install -y \ + --setopt=install_weak_deps=False \ + {pkgs} + +dnf clean all +""" + +ZYPPER_BOOTSTRAP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +zypper --non-interactive refresh +zypper --non-interactive update +zypper --non-interactive install \ + --no-recommends \ + system-user-nobody \ + {pkgs} + +zypper --non-interactive clean + +if [ -f /usr/lib/mit/bin/krb5-config ]; then + ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config +fi +""" + +# A generic shell script to setup locale +LOCALE_SETUP = r""" +#!/bin/bash +{GENERATED_MARKER} +set -xueo pipefail + +# refer to /usr/share/i18n/locales +INPUTFILE=en_US +# refer to /usr/share/i18n/charmaps +CHARMAP=UTF-8 +# locale to generate in /usr/lib/locale +# glibc/localedef will normalize UTF-8 to utf8, follow the naming style +LOCALE=$INPUTFILE.utf8 + +# if locale is already correct, exit +( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0 + +# if locale not available, generate locale into /usr/lib/locale +if ! ( locale --all-locales | grep -i $LOCALE ) +then + # no-archive means create its own dir + localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE +fi + +# update locale conf and global env file +# set both LC_ALL and LANG for safe + +# update conf for Debian family +FILE=/etc/default/locale +if [ -f $FILE ] +then + echo LC_ALL="$LOCALE" > $FILE + echo LANG="$LOCALE" >> $FILE +fi + +# update conf for RedHat family +FILE=/etc/locale.conf +if [ -f $FILE ] +then + # LC_ALL is not valid in this file, set LANG only + echo LANG="$LOCALE" > $FILE +fi + +# update global env file +FILE=/etc/environment +if [ -f $FILE ] +then + # append LC_ALL if not exist + grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE + # append LANG if not exist + grep LANG $FILE || echo LANG="$LOCALE" >> $FILE +fi +""" + + +DOCKERFILE = r""" +{GENERATED_MARKER} +FROM {docker_image} + +# pass in with --build-arg while build +ARG SHA1SUM +RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt + +ADD *.sh /tmp/ +# need root permission, do it before USER samba +RUN /tmp/bootstrap.sh && /tmp/locale.sh + +# if ld.gold exists, force link it to ld +RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }} +# if ld.mold exists, force link it to ld (prefer mold over gold! ;-) +RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }} + +# make test can not work with root, so we have to create a new user +RUN useradd -m -U -s /bin/bash samba && \ + mkdir -p /etc/sudoers.d && \ + echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba + +USER samba +WORKDIR /home/samba +# samba tests rely on this +ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8 LANGUAGE=en_US +""" + +# Vagrantfile snippet for each dist +VAGRANTFILE_SNIPPET = r""" + config.vm.define "{name}" do |v| + v.vm.box = "{vagrant_box}" + v.vm.hostname = "{name}" + v.vm.provision :shell, path: "{name}/bootstrap.sh" + v.vm.provision :shell, path: "{name}/locale.sh" + end +""" + +# global Vagrantfile with snippets for all dists +VAGRANTFILE_GLOBAL = r""" +{GENERATED_MARKER} + +Vagrant.configure("2") do |config| + config.ssh.insert_key = false + +{vagrantfile_snippets} + +end +""" + +DEB_DISTS = { + 'debian11': { + 'docker_image': 'debian:11', + 'vagrant_box': 'debian/bullseye64', + 'replace': { + 'language-pack-en': '', # included in locales + 'shfmt': '', + } + }, + 'debian11-32bit': { + 'docker_image': 'registry-1.docker.io/i386/debian:11', + 'vagrant_box': 'debian/bullseye32', + 'replace': { + 'language-pack-en': '', # included in locales + 'shfmt': '', + } + }, + 'debian12': { + 'docker_image': 'debian:12', + 'vagrant_box': 'debian/bookworm64', + 'replace': { + 'language-pack-en': '', # included in locales + 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available + } + }, + 'debian12-32bit': { + 'docker_image': 'registry-1.docker.io/i386/debian:12', + 'vagrant_box': 'debian/bookworm32', + 'replace': { + 'language-pack-en': '', # included in locales + 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available + } + }, + 'ubuntu1804': { + 'docker_image': 'ubuntu:18.04', + 'vagrant_box': 'ubuntu/bionic64', + 'replace': { + 'liburing-dev': '', # not available + 'shfmt': '', + } + }, + 'ubuntu1804-32bit': { + 'docker_image': 'registry-1.docker.io/i386/ubuntu:18.04', + 'vagrant_box': 'ubuntu/bionic32', + 'replace': { + 'liburing-dev': '', # not available + 'shfmt': '', + } + }, + 'ubuntu2004': { + 'docker_image': 'ubuntu:20.04', + 'vagrant_box': 'ubuntu/focal64', + 'replace': { + 'liburing-dev': '', # not available + 'shfmt': '', + } + }, + 'ubuntu2204': { + 'docker_image': 'ubuntu:22.04', + 'vagrant_box': 'ubuntu/jammy64', + 'replace': { + 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available + }, + } +} + + +RPM_DISTS = { + 'centos7': { + 'docker_image': 'centos:7', + 'vagrant_box': 'centos/7', + 'bootstrap': YUM_BOOTSTRAP, + 'replace': { + 'lsb-release': 'redhat-lsb', + 'python3': 'python36', + 'python3-cryptography': 'python36-cryptography', + 'python3-devel': 'python36-devel', + 'python3-dns': 'python36-dns', + 'python3-pyasn1': 'python36-pyasn1', + 'python3-gpg': 'python36-gpg', + 'python3-iso8601' : 'python36-iso8601', + 'python3-markdown': 'python36-markdown', + 'python3-requests': 'python36-requests', + # although python36-devel is available + # after epel-release installed + # however, all other python3 pkgs are still python36-ish + 'python2-gpg': 'pygpgme', + '@development-tools': '"@Development Tools"', # add quotes + 'glibc-langpack-en': '', # included in glibc-common + 'glibc-locale-source': '', # included in glibc-common + # update perl core modules on centos + # fix: Can't locate Archive/Tar.pm in @INC + 'perl': 'perl-core', + 'perl-FindBin': '', + 'rpcsvc-proto-devel': '', + 'glusterfs-api-devel': '', + 'glusterfs-devel': '', + 'libcephfs-devel': '', + 'gnutls-devel': 'compat-gnutls37-devel', + 'gnutls-utils': 'compat-gnutls37-utils', + 'liburing-devel': '', # not available + 'python3-setproctitle': 'python36-setproctitle', + 'tracker-devel': '', # do not install + 'mold': '', + 'ShellCheck': '', + 'shfmt': '', + 'codespell': '', + } + }, + 'centos8s': { + 'docker_image': 'quay.io/centos/centos:stream8', + 'vagrant_box': 'centos/stream8', + 'bootstrap': CENTOS8S_YUM_BOOTSTRAP, + 'replace': { + 'lsb-release': 'redhat-lsb', + '@development-tools': '"@Development Tools"', # add quotes + 'lcov': '', # does not exist + 'perl-JSON-Parse': '', # does not exist? + 'perl-Test-Base': 'perl-Test-Simple', + 'perl-FindBin': '', + 'liburing-devel': '', # not available yet, Add me back, once available! + 'mold': '', + 'ShellCheck': '', + 'shfmt': '', + 'codespell': '', + } + }, + 'fedora39': { + 'docker_image': 'quay.io/fedora/fedora:39', + 'vagrant_box': 'fedora/39-cloud-base', + 'bootstrap': DNF_BOOTSTRAP, + 'replace': { + 'lsb-release': 'redhat-lsb', + 'perl-FindBin': '', + 'python3-iso8601': 'python3-dateutil', + 'libtracker-sparql-2.0-dev': '', # only tracker 3.x is available + } + }, + 'opensuse155': { + 'docker_image': 'opensuse/leap:15.5', + 'vagrant_box': 'opensuse/openSUSE-15.5-x86_64', + 'bootstrap': ZYPPER_BOOTSTRAP, + 'replace': { + '@development-tools': '', + 'dbus-devel': 'dbus-1-devel', + 'docbook-style-xsl': 'docbook-xsl-stylesheets', + 'glibc-common': 'glibc-locale', + 'glibc-locale-source': 'glibc-i18ndata', + 'glibc-langpack-en': '', + 'jansson-devel': 'libjansson-devel', + 'keyutils-libs-devel': 'keyutils-devel', + 'krb5-workstation': 'krb5-client', + 'python3-libsemanage': 'python3-semanage', + 'openldap-devel': 'openldap2-devel', + 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper', + 'perl-JSON-Parse': 'perl-JSON-XS', + 'perl-generators': '', + 'perl-interpreter': '', + 'perl-FindBin': '', + 'procps-ng': 'procps', + 'python3-iso8601': 'python3-python-dateutil', + 'python3-dns': 'python3-dnspython', + 'python3-markdown': 'python3-Markdown', + 'quota-devel': '', + 'glusterfs-api-devel': '', + 'gnutls-utils': 'gnutls', + 'libtasn1-tools': '', # asn1Parser is part of libtasn1 + 'mold': '', + 'shfmt': '', + 'yum-utils': '', + } + } +} + + +DEB_FAMILY = { + 'name': 'deb', + 'pkgs': DEB_PKGS, + 'bootstrap': APT_BOOTSTRAP, # family default + 'dists': DEB_DISTS, +} + + +RPM_FAMILY = { + 'name': 'rpm', + 'pkgs': RPM_PKGS, + 'bootstrap': YUM_BOOTSTRAP, # family default + 'dists': RPM_DISTS, +} + + +YML_HEADER = r""" +--- +packages: +""" + + +def expand_family_dists(family): + dists = {} + for name, config in family['dists'].items(): + config = config.copy() + config['name'] = name + config['home'] = join(OUT, name) + config['family'] = family['name'] + config['GENERATED_MARKER'] = GENERATED_MARKER + + # replace dist specific pkgs + replace = config.get('replace', {}) + pkgs = [] + for pkg in family['pkgs']: + pkg = replace.get(pkg, pkg) # replace if exists or get self + if pkg: + pkgs.append(pkg) + pkgs.sort() + + lines = [' - {}'.format(pkg) for pkg in pkgs] + config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines) + + sep = ' \\' + os.linesep + ' ' + config['pkgs'] = sep.join(pkgs) + + # get dist bootstrap template or fall back to family default + bootstrap_template = config.get('bootstrap', family['bootstrap']) + config['bootstrap.sh'] = bootstrap_template.format(**config).strip() + config['locale.sh'] = LOCALE_SETUP.format(**config).strip() + + config['Dockerfile'] = DOCKERFILE.format(**config).strip() + # keep the indent, no strip + config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config) + + dists[name] = config + return dists + + +# expanded config for dists +DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY) +RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY) + +# assemble all together +DISTS = {} +DISTS.update(DEB_DISTS_EXP) +DISTS.update(RPM_DISTS_EXP) + + +def render_vagrantfile(dists): + """ + Render all snippets for each dist into global Vagrantfile. + + Vagrant supports multiple vms in one Vagrantfile. + This make it easier to manage the fleet, e.g: + + start all: vagrant up + start one: vagrant up ubuntu1804 + + All other commands apply to above syntax, e.g.: status, destroy, provision + """ + # sort dists by name and put all vagrantfile snippets together + snippets = [ + dists[dist]['vagrantfile_snippet'] + for dist in sorted(dists.keys())] + + return VAGRANTFILE_GLOBAL.format( + vagrantfile_snippets=''.join(snippets), + GENERATED_MARKER=GENERATED_MARKER + ) + + +VAGRANTFILE = render_vagrantfile(DISTS) + + +# data we need to expose +__all__ = ['DISTS', 'VAGRANTFILE', 'OUT'] |