diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /taskcluster/docker/funsize-update-generator | |
parent | Initial commit. (diff) | |
download | firefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/docker/funsize-update-generator')
11 files changed, 1213 insertions, 0 deletions
diff --git a/taskcluster/docker/funsize-update-generator/Dockerfile b/taskcluster/docker/funsize-update-generator/Dockerfile new file mode 100644 index 0000000000..c032e2231b --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:18.04 +MAINTAINER Simon Fraser <sfraser@mozilla.com> + +# Required software +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + bzip2 \ + ca-certificates \ + curl \ + gcc \ + jq \ + libdpkg-perl \ + libgetopt-simple-perl \ + liblzma-dev \ + locales \ + python3.8 \ + libpython3.8-dev \ + python3-dev \ + xz-utils +RUN useradd -d /home/worker -s /bin/bash -m worker +COPY requirements.txt / + +RUN locale-gen en_CA.UTF-8 +ENV LANG en_CA.UTF-8 +ENV LANGUAGE en_CA.UTF-8 +ENV LANG_ALL en_CA.UTF-8 +ENV LC_ALL en_CA.UTF-8 + +# python-pip installs a lot of dependencies increasing the size of an image +# drastically. Install it like this saves us almost 200M. +RUN bash -c "curl -L https://bootstrap.pypa.io/get-pip.py | python3.8" + +RUN ["pip", "install", "-r", "/requirements.txt"] + +# scripts +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ + +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /*.sh +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker + +USER worker + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/funsize-update-generator/Makefile b/taskcluster/docker/funsize-update-generator/Makefile new file mode 100644 index 0000000000..6b67f0ed90 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Makefile @@ -0,0 +1,9 @@ +IMAGE_NAME = funsize-update-generator + +build: + docker build -t $(IMAGE_NAME) --no-cache --rm . + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff --git a/taskcluster/docker/funsize-update-generator/README b/taskcluster/docker/funsize-update-generator/README new file mode 100644 index 0000000000..5e9507be71 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/README @@ -0,0 +1,7 @@ + +To run this locally for testing/development purposes: + +1. Find a funsize generating task ID +2. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com" -e SIGNING_CERT='nightly' -e MAR_CHANNEL_ID='firefox-mozilla-central' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64" funsize-update-generator /runme.sh + +The TASK_ID should be a recent "partials" Task. diff --git a/taskcluster/docker/funsize-update-generator/dep1.pubkey b/taskcluster/docker/funsize-update-generator/dep1.pubkey new file mode 100644 index 0000000000..927b2cc947 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/dep1.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA8Y6AS+xwKoXZl0X5qOKr +0I00xC4UN+IMjA1LIQoZ2GBkiqQF3q8v2nWTFE0+47+3NtP0l8tvsQY+LSYR4Fek +v2Vx4m/CAMKmWzW6Vtlj80y6rQ04V19l41bZXvCIBW5fm9sAvPgc7CngkcLySNqk +8vf57cUEpOmbsjSOCmK0j8hh03I1eWogpbAVEchSm1xN2sUJaVTvz5j8BfE6Vm0i +nN7V0zF+AOxzvntZIpfUqMZbHRiMkGn4l9rjia1Rz0qUc9RNCJkNocyKtQ2N2wnN +FjHpmK9x2V71cS1JQGhgLegrswPCAWY1lTmiLk9LweqGoVL0rqR4LCkb0VCaeSRe +6bUEYcU1ZQedE80zGKB3AfoC5br1shYY0xjmyRSCQ8m8WE60HzXhL8wczKrn5yoJ +iF6BxFwcYsvrWBPgIYVZLcqjODfR/M62o8yIfTC7yBcIdycJ0sWhB47dHAFxv1kc +wv8Ik9ftvDyupE8kwcl58fNOXz93j7IxMry/ey27NyYpESPOUNcjT8TP26FdGebg +4iJx0/LaYmaNUdchfBBlaYqGdH6ZGK0OeVxzHstGuG0gebm/igYcpaFxiQzvWijX +MIAU56s4g+yj7pSzT5/s9r8Gv+YhsNHKm4hnwLZaITV0lLMT5h/OZGseQTPMBnAR +hK3CIfcqG0I23hdwI29ZuUMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/nightly.pubkey b/taskcluster/docker/funsize-update-generator/nightly.pubkey new file mode 100644 index 0000000000..e51049844c --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/nightly.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc +AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc +ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA +nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg +yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C +H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc +NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ +pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j +H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0 +lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d +2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV +Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/release.pubkey b/taskcluster/docker/funsize-update-generator/release.pubkey new file mode 100644 index 0000000000..ec1103d828 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/release.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6 +i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI +ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h +1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0 +GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi +LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6 +fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE +CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8 +m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF +K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U +zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK +r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/requirements.in b/taskcluster/docker/funsize-update-generator/requirements.in new file mode 100644 index 0000000000..3360c71690 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.in @@ -0,0 +1,7 @@ +aiohttp +awscli +mar +redo +requests +scriptworker +sh diff --git a/taskcluster/docker/funsize-update-generator/requirements.txt b/taskcluster/docker/funsize-update-generator/requirements.txt new file mode 100644 index 0000000000..ecdafe40dc --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.txt @@ -0,0 +1,407 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes requirements.in +# +aiohttp==3.7.4.post0 \ + --hash=sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe \ + --hash=sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe \ + --hash=sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5 \ + --hash=sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8 \ + --hash=sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd \ + --hash=sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb \ + --hash=sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c \ + --hash=sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87 \ + --hash=sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0 \ + --hash=sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290 \ + --hash=sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5 \ + --hash=sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287 \ + --hash=sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde \ + --hash=sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf \ + --hash=sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8 \ + --hash=sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16 \ + --hash=sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf \ + --hash=sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809 \ + --hash=sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213 \ + --hash=sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f \ + --hash=sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013 \ + --hash=sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b \ + --hash=sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9 \ + --hash=sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5 \ + --hash=sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb \ + --hash=sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df \ + --hash=sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4 \ + --hash=sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439 \ + --hash=sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f \ + --hash=sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22 \ + --hash=sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f \ + --hash=sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5 \ + --hash=sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970 \ + --hash=sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009 \ + --hash=sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc \ + --hash=sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a \ + --hash=sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95 + # via + # -r requirements.in + # scriptworker + # taskcluster +aiomemoizettl==0.0.3 \ + --hash=sha256:07a6becac60f6cd2604b9f2b73bcd9a50079a0b7b55e2a4e45b1eec5a3ea9659 \ + --hash=sha256:0a80d2dc765e545263f515363b6700ec8cf86fa3968b529f56390b28e34f743d + # via scriptworker +arrow==1.0.3 \ + --hash=sha256:3515630f11a15c61dcb4cdd245883270dd334c83f3e639824e65a4b79cc48543 \ + --hash=sha256:399c9c8ae732270e1aa58ead835a79a40d7be8aa109c579898eb41029b5a231d + # via scriptworker +asn1crypto==1.4.0 \ + --hash=sha256:4bcdf33c861c7d40bdcd74d8e4dd7661aac320fcdf40b9a3f95b4ee12fde2fa8 \ + --hash=sha256:f4f6e119474e58e04a2b1af817eb585b4fd72bdd89b998624712b5c99be7641c + # via mar +async-timeout==3.0.1 \ + --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ + --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 + # via + # aiohttp + # taskcluster +attrs==20.3.0 \ + --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \ + --hash=sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700 + # via + # aiohttp + # jsonschema +awscli==1.19.33 \ + --hash=sha256:473d1a653b05577440747d387778efbea8b5c05b88231dd215d74e36133c73c1 \ + --hash=sha256:d97316d781f3fb52702779a3e5ca8d5780eb253a957ee5c35542607b6014f737 + # via -r requirements.in +backports.lzma==0.0.14 \ + --hash=sha256:16d8b68e4d3cd4e6c9ddb059850452946da3914c8a8e197a7f2b0954559f2df4 + # via mar +botocore==1.20.33 \ + --hash=sha256:a33e862685259fe22d9790d9c9f3567feda8b824d44d3c62a3617af1133543a4 \ + --hash=sha256:e355305309699d3aca1e0050fc21d48595b40db046cb0d2491cd57ff5b26920b + # via + # awscli + # s3transfer +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 + # via requests +cffi==1.14.5 \ + --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ + --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ + --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ + --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ + --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ + --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ + --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ + --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ + --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ + --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ + --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ + --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ + --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ + --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ + --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ + --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ + --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ + --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ + --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ + --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ + --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ + --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ + --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ + --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ + --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ + --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ + --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ + --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ + --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ + --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ + --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ + --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ + --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ + --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ + --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ + --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ + --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c + # via cryptography +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 + # via + # aiohttp + # requests +click==7.1.2 \ + --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ + --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc + # via mar +colorama==0.4.3 \ + --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \ + --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 + # via awscli +construct==2.10.61 \ + --hash=sha256:d75384a04cb36ae5417dd34ce230c392e6085d872ab8a99ebd756c34bac0dff5 + # via mar +cryptography==3.4.6 \ + --hash=sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b \ + --hash=sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336 \ + --hash=sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87 \ + --hash=sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7 \ + --hash=sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799 \ + --hash=sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b \ + --hash=sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df \ + --hash=sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0 \ + --hash=sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3 \ + --hash=sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724 \ + --hash=sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2 \ + --hash=sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 + # via + # jwcrypto + # mar + # scriptworker +dictdiffer==0.8.1 \ + --hash=sha256:1adec0d67cdf6166bda96ae2934ddb5e54433998ceab63c984574d187cc563d2 \ + --hash=sha256:d79d9a39e459fe33497c858470ca0d2e93cb96621751de06d631856adfd9c390 + # via scriptworker +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 + # via awscli +github3.py==2.0.0 \ + --hash=sha256:7e153e5dc559f27e4d15c91571a2d6e1522c9faa4406e188459d9731f0b98875 \ + --hash=sha256:8dd4ac612fd60cb277eaf6e2ce02f68dda54aba06870ca6fa2b28369bf39aa14 + # via scriptworker +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 + # via + # requests + # yarl +immutabledict==1.2.0 \ + --hash=sha256:1e6bb628eb158d32caf961bed5bb283f9e57201b50f48cd5063175612f0f3efd \ + --hash=sha256:9ba015dc189af8aac5921248af18a06d76a1d026e362f917a8e9ee29e8de5133 + # via scriptworker +jmespath==0.10.0 \ + --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ + --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f + # via botocore +json-e==4.4.1 \ + --hash=sha256:7d9f6235f855ce70418b9d6158c043c588c3c3d7d0902972f3fb7c5399997ce9 + # via scriptworker +jsonschema==3.2.0 \ + --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ + --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a + # via scriptworker +jwcrypto==0.8 \ + --hash=sha256:16e17faa4dce36551ade3a3ccb06236a61e5924ea1db163c9be9827acf935a82 \ + --hash=sha256:b7fee2635bbefdf145399392f5be26ad54161c8271c66b5fe107b4b452f06c24 + # via github3.py +mar==3.1.0 \ + --hash=sha256:1939df482f2d3f6221405da00d6286d77d2dd60d372a0fd37532a8f00544f64f \ + --hash=sha256:5d2904a063f0da625e37515fa9eb340082e69fa1c00fdbeb82d28b7ff3c51e28 + # via -r requirements.in +mohawk==1.1.0 \ + --hash=sha256:3ed296a30453d0b724679e0fd41e4e940497f8e461a9a9c3b7f36e43bab0fa09 \ + --hash=sha256:d2a0e3ab10a209cc79e95e28f2dd54bd4a73fd1998ffe27b7ba0f962b6be9723 + # via taskcluster +multidict==5.1.0 \ + --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \ + --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \ + --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \ + --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \ + --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \ + --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \ + --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \ + --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \ + --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \ + --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \ + --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \ + --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \ + --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \ + --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \ + --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \ + --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \ + --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \ + --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \ + --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \ + --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \ + --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \ + --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \ + --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \ + --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \ + --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \ + --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \ + --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \ + --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \ + --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \ + --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \ + --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \ + --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \ + --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \ + --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \ + --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \ + --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \ + --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80 + # via + # aiohttp + # yarl +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via rsa +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 + # via cffi +pyrsistent==0.17.3 \ + --hash=sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e + # via jsonschema +python-dateutil==2.8.1 \ + --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ + --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a + # via + # arrow + # botocore + # github3.py +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via + # awscli + # scriptworker +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 \ + --hash=sha256:c76e4c23ab2f8840261736a851323cd98493710e7a9d36a1058535dca501f293 + # via -r requirements.in +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e + # via + # -r requirements.in + # github3.py + # taskcluster +rsa==4.5 \ + --hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \ + --hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f + # via awscli +s3transfer==0.3.6 \ + --hash=sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2 \ + --hash=sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547 + # via awscli +scriptworker==37.0.2 \ + --hash=sha256:306e5864d95649ab89056274402f5d933ed525e3b98794beb10eb8994edfd947 \ + --hash=sha256:bb264565a97e84e1f09ce4edc1b453ab9e10a385f2fea10add99ba9b51cb92df + # via -r requirements.in +sh==1.14.1 \ + --hash=sha256:39aa9af22f6558a0c5d132881cf43e34828ca03e4ae11114852ca6a55c7c1d8e \ + --hash=sha256:75e86a836f47de095d4531718fe8489e6f7446c75ddfa5596f632727b919ffae + # via -r requirements.in +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced + # via + # jsonschema + # mohawk + # python-dateutil + # taskcluster +slugid==2.0.0 \ + --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \ + --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c + # via taskcluster +taskcluster-urls==13.0.1 \ + --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \ + --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \ + --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b + # via taskcluster +taskcluster==39.2.0 \ + --hash=sha256:07dec0b12c80122498d772a242f92373d65731f133fb0d9aba5c2adf78475b13 \ + --hash=sha256:844d58207f7ccba73cce2413ff79ad83ef39e4095e0269fb82ff14fa9b1c3e2f \ + --hash=sha256:e5fe89c1f85b3caf141106053c1e98bcb930a629889e67acb2bd9aa92bd95182 + # via scriptworker +typing-extensions==3.7.4.3 \ + --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \ + --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \ + --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f + # via aiohttp +uritemplate==3.0.1 \ + --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ + --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae + # via github3.py +urllib3==1.26.4 \ + --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ + --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 + # via + # botocore + # requests +yarl==1.6.3 \ + --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \ + --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \ + --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \ + --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \ + --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \ + --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \ + --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \ + --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \ + --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \ + --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \ + --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \ + --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \ + --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \ + --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \ + --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \ + --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \ + --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \ + --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \ + --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \ + --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \ + --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \ + --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \ + --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \ + --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \ + --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \ + --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \ + --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \ + --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \ + --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \ + --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \ + --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \ + --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \ + --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \ + --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \ + --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \ + --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \ + --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71 + # via aiohttp + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# setuptools diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh new file mode 100644 index 0000000000..99d80c09e2 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +set -xe + +test "$TASK_ID" +test "$SIGNING_CERT" + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# Strip trailing / if present +TASKCLUSTER_ROOT_URL="${TASKCLUSTER_ROOT_URL%/}" +export TASKCLUSTER_ROOT_URL + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="${TASKCLUSTER_ROOT_URL%/}/api/queue/v1" + +curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + +# auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/ +# -> bucket of tc-gp-private-1d-us-east-1, path of releng/mbsdiff-cache/ +# Trailing slash is important, due to prefix permissions in S3. +S3_BUCKET_AND_PATH=$(jq -r '.scopes[] | select(contains ("auth:aws-s3"))' /home/worker/task.json | awk -F: '{print $4}') + +# Will be empty if there's no scope for AWS S3. +if [ -n "${S3_BUCKET_AND_PATH}" ] && getent hosts taskcluster +then + # Does this parse as we expect? + S3_PATH=${S3_BUCKET_AND_PATH#*/} + AWS_BUCKET_NAME=${S3_BUCKET_AND_PATH%/${S3_PATH}*} + test "${S3_PATH}" + test "${AWS_BUCKET_NAME}" + + set +x # Don't echo these. + secret_url="${TASKCLUSTER_PROXY_URL}/api/auth/v1/aws/s3/read-write/${AWS_BUCKET_NAME}/${S3_PATH}" + AUTH=$(curl "${secret_url}") + AWS_ACCESS_KEY_ID=$(echo "${AUTH}" | jq -r '.credentials.accessKeyId') + AWS_SECRET_ACCESS_KEY=$(echo "${AUTH}" | jq -r '.credentials.secretAccessKey') + AWS_SESSION_TOKEN=$(echo "${AUTH}" | jq -r '.credentials.sessionToken') + export AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY + export AWS_SESSION_TOKEN + AUTH= + + if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ]; then + # Pass the full bucket/path prefix, as the script just appends local files. + export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -S ${S3_BUCKET_AND_PATH}" + fi + set -x +else + # disable caching + export MBSDIFF_HOOK= +fi + +# EXTRA_PARAMS is optional +# shellcheck disable=SC2086 +python3.8 /home/worker/bin/funsize.py \ + --artifacts-dir "$ARTIFACTS_DIR" \ + --task-definition /home/worker/task.json \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + $EXTRA_PARAMS diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py new file mode 100644 index 0000000000..84fd2fbd0b --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import asyncio +import configparser +import json +import logging +import os +import shutil +import tempfile +import time +from contextlib import AsyncExitStack +from distutils.util import strtobool +from pathlib import Path + +import aiohttp +from mardor.reader import MarReader +from mardor.signing import get_keysize +from scriptworker.utils import get_hash, retry_async + +log = logging.getLogger(__name__) + + +ROOT_URL = os.environ.get( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" +) +QUEUE_PREFIX = f"{ROOT_URL}/api/queue/" +ALLOWED_URL_PREFIXES = ( + "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", + "http://download.cdn.mozilla.net/pub/firefox/nightly/", + "http://ftp.mozilla.org/", + "http://download.mozilla.org/", + "https://archive.mozilla.org/", + "http://archive.mozilla.org/", + QUEUE_PREFIX, +) +STAGING_URL_PREFIXES = ( + "http://ftp.stage.mozaws.net/", + "https://ftp.stage.mozaws.net/", +) + +BCJ_OPTIONS = { + "x86": ["--x86"], + "x86_64": ["--x86"], + "aarch64": [], + # macOS Universal Builds + "macos-x86_64-aarch64": [], +} + + +def verify_signature(mar, cert): + log.info("Checking %s signature", mar) + with open(mar, "rb") as mar_fh: + m = MarReader(mar_fh) + if not m.verify(verify_key=cert): + raise ValueError( + "MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert + ) + + +def process_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-dir", required=True) + parser.add_argument("--signing-cert", type=argparse.FileType("rb"), required=True) + parser.add_argument("--task-definition", required=True, type=argparse.FileType("r")) + parser.add_argument( + "--allow-staging-prefixes", + action="store_true", + default=strtobool(os.environ.get("FUNSIZE_ALLOW_STAGING_PREFIXES", "false")), + help="Allow files from staging buckets.", + ) + parser.add_argument( + "-q", + "--quiet", + dest="log_level", + action="store_const", + const=logging.INFO, + default=logging.DEBUG, + ) + parser.add_argument( + "--arch", + type=str, + required=True, + choices=BCJ_OPTIONS.keys(), + help="The archtecture you are building.", + ) + return parser.parse_args() + + +def validate_mar_channel_id(mar, channel_ids): + log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids) + # We may get a string with a list representation, or a single entry string. + channel_ids = set(channel_ids.split(",")) + + product_info = MarReader(open(mar, "rb")).productinfo + if not isinstance(product_info, tuple): + raise ValueError( + "Malformed product information in mar: {}".format(product_info) + ) + + found_channel_ids = set(product_info[1].split(",")) + + if not found_channel_ids.issubset(channel_ids): + raise ValueError( + "MAR_CHANNEL_ID mismatch, {} not in {}".format(product_info[1], channel_ids) + ) + + log.info("%s channel %s in %s", mar, product_info[1], channel_ids) + + +async def retry_download(*args, semaphore=None, **kwargs): # noqa: E999 + """Retry download() calls.""" + async with AsyncExitStack() as stack: + if semaphore: + await stack.enter_async_context(semaphore) + await retry_async( + download, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + args=args, + kwargs=kwargs, + ) + + +def verify_allowed_url(mar, allowed_url_prefixes): + if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes): + raise ValueError( + "{mar} is not in allowed URL prefixes: {p}".format( + mar=mar, p=allowed_url_prefixes + ) + ) + + +async def download(url, dest, mode=None): # noqa: E999 + log.info("Downloading %s to %s", url, dest) + chunk_size = 4096 + bytes_downloaded = 0 + async with aiohttp.ClientSession(raise_for_status=True) as session: + start = time.time() + async with session.get(url, timeout=120) as resp: + # Additional early logging for download timeouts. + log.debug("Fetching from url %s", resp.url) + for history in resp.history: + log.debug("Redirection history: %s", history.url) + log.debug("Headers for %s: %s", resp.url, resp.headers) + if "Content-Length" in resp.headers: + log.debug( + "Content-Length expected for %s: %s", + url, + resp.headers["Content-Length"], + ) + log_interval = chunk_size * 1024 + with open(dest, "wb") as fd: + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + fd.write(chunk) + bytes_downloaded += len(chunk) + log_interval -= len(chunk) + if log_interval <= 0: + log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded) + log_interval = chunk_size * 1024 + end = time.time() + log.info( + "Downloaded %s, %s bytes in %s seconds: sha256:%s", + url, + bytes_downloaded, + int(end - start), + get_hash(dest, hash_alg="sha256"), + ) + if mode: + log.info("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +async def download_buildsystem_bits(partials_config, downloads, tools_dir): + """Download external tools needed to make partials.""" + + # We're making the assumption that the "to" mar is the same for all, + # as that's the way this task is currently used. + to_url = extract_download_urls(partials_config, mar_type="to").pop() + + repo = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceRepository", + ) + revision = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceStamp", + ) + + urls = { + "make_incremental_update.sh": f"{repo}/raw-file/{revision}/tools/" + "update-packaging/make_incremental_update.sh", + "common.sh": f"{repo}/raw-file/{revision}/tools/update-packaging/common.sh", + "mar": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mar", + "mbsdiff": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mbsdiff", + } + for filename, url in urls.items(): + filename = tools_dir / filename + await retry_download(url, dest=filename, mode=0o755) + + +def find_file(directory, filename): + log.debug("Searching for %s in %s", filename, directory) + return next(Path(directory).rglob(filename)) + + +def get_option(directory, filename, section, option): + log.info("Extracting [%s]: %s from %s/**/%s", section, option, directory, filename) + f = find_file(directory, filename) + config = configparser.ConfigParser() + config.read(f) + rv = config.get(section, option) + log.info("Found %s", rv) + return rv + + +def extract_download_urls(partials_config, mar_type): + """Extract a set of urls to download from the task configuration. + + mar_type should be one of "from", "to" + """ + return {definition[f"{mar_type}_mar"] for definition in partials_config} + + +async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert): + """Download, check signature, channel ID and unpack MAR files.""" + # Separate these categories so we can opt to perform checks on only 'to' downloads. + from_urls = extract_download_urls(partials_config, mar_type="from") + to_urls = extract_download_urls(partials_config, mar_type="to") + tasks = list() + downloads = dict() + + semaphore = asyncio.Semaphore(2) # Magic 2 to reduce network timeout errors. + for url in from_urls.union(to_urls): + verify_allowed_url(url, allowed_url_prefixes) + downloads[url] = { + "download_path": Path(tempfile.mkdtemp()) / Path(url).name, + } + tasks.append( + retry_download(url, downloads[url]["download_path"], semaphore=semaphore) + ) + + await asyncio.gather(*tasks) + + for url in downloads: + # Verify signature, but not from an artifact as we don't + # depend on the signing task + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION") and not url.startswith( + QUEUE_PREFIX + ): + verify_signature(downloads[url]["download_path"], signing_cert) + + # Only validate the target channel ID, as we update from beta->release + if url in to_urls: + validate_mar_channel_id( + downloads[url]["download_path"], os.environ["MAR_CHANNEL_ID"] + ) + + downloads[url]["extracted_path"] = tempfile.mkdtemp() + with open(downloads[url]["download_path"], "rb") as mar_fh: + log.info( + "Unpacking %s into %s", + downloads[url]["download_path"], + downloads[url]["extracted_path"], + ) + m = MarReader(mar_fh) + m.extract(downloads[url]["extracted_path"]) + + return downloads + + +async def run_command(cmd, cwd="/", env=None, label=None, silent=False): + log.info("Running: %s", cmd) + if not env: + env = dict() + process = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + env=env, + ) + if label: + label = "{}: ".format(label) + else: + label = "" + + async def read_output(stream, label, printcmd): + while True: + line = await stream.readline() + if line == b"": + break + printcmd("%s%s", label, line.decode("utf-8").rstrip()) + + if silent: + await process.wait() + else: + await asyncio.gather( + read_output(process.stdout, label, log.info), + read_output(process.stderr, label, log.warning), + ) + await process.wait() + + +async def generate_partial(from_dir, to_dir, dest_mar, mar_data, tools_dir, arch): + log.info("Generating partial %s", dest_mar) + env = os.environ.copy() + env["LC_ALL"] = "C" + env["MAR"] = tools_dir / "mar" + env["MBSDIFF"] = tools_dir / "mbsdiff" + if arch: + env["BCJ_OPTIONS"] = " ".join(BCJ_OPTIONS[arch]) + env["MOZ_PRODUCT_VERSION"] = mar_data["version"] + env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"] + env["BRANCH"] = mar_data["branch"] + + make_incremental_update = tools_dir / "make_incremental_update.sh" + cmd = f"{make_incremental_update} {dest_mar} {from_dir} {to_dir}" + + await run_command(cmd, cwd=dest_mar.parent, env=env, label=dest_mar.name) + validate_mar_channel_id(dest_mar, mar_data["MAR_CHANNEL_ID"]) + + +async def manage_partial( + partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None +): + from_url = partial_def["from_mar"] + to_url = partial_def["to_mar"] + from_path = downloads[from_url]["extracted_path"] + to_path = downloads[to_url]["extracted_path"] + + mar_data = { + "MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"], + "version": get_option( + to_path, filename="application.ini", section="App", option="Version" + ), + "appName": get_option( + from_path, filename="application.ini", section="App", option="Name" + ), + # Use Gecko repo and rev from platform.ini, not application.ini + "repo": get_option( + to_path, filename="platform.ini", section="Build", option="SourceRepository" + ), + "revision": get_option( + to_path, filename="platform.ini", section="Build", option="SourceStamp" + ), + "locale": partial_def["locale"], + "from_mar": partial_def["from_mar"], + "from_size": os.path.getsize(downloads[from_url]["download_path"]), + "from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"), + "from_buildid": get_option( + from_path, filename="application.ini", section="App", option="BuildID" + ), + "to_mar": partial_def["to_mar"], + "to_size": os.path.getsize(downloads[to_url]["download_path"]), + "to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"), + "to_buildid": get_option( + to_path, filename="application.ini", section="App", option="BuildID" + ), + "mar": partial_def["dest_mar"], + } + # if branch not set explicitly use repo-name + mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name) + + for field in ( + "update_number", + "previousVersion", + "previousBuildNumber", + "toVersion", + "toBuildNumber", + ): + if field in partial_def: + mar_data[field] = partial_def[field] + + dest_mar = Path(artifacts_dir) / mar_data["mar"] + + async with semaphore: + await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch) + + mar_data["size"] = os.path.getsize(dest_mar) + mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512") + return mar_data + + +async def async_main(args, signing_cert): + tasks = [] + + allowed_url_prefixes = list(ALLOWED_URL_PREFIXES) + if args.allow_staging_prefixes: + allowed_url_prefixes += STAGING_URL_PREFIXES + + task = json.load(args.task_definition) + + downloads = await download_and_verify_mars( + task["extra"]["funsize"]["partials"], allowed_url_prefixes, signing_cert + ) + + tools_dir = Path(tempfile.mkdtemp()) + await download_buildsystem_bits( + partials_config=task["extra"]["funsize"]["partials"], + downloads=downloads, + tools_dir=tools_dir, + ) + + # May want to consider os.cpu_count() if we ever run on osx/win. + # sched_getaffinity is the list of cores we can run on, not the total. + semaphore = asyncio.Semaphore(len(os.sched_getaffinity(0))) + for definition in task["extra"]["funsize"]["partials"]: + tasks.append( + asyncio.ensure_future( + retry_async( + manage_partial, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + kwargs=dict( + partial_def=definition, + artifacts_dir=args.artifacts_dir, + tools_dir=tools_dir, + arch=args.arch, + downloads=downloads, + semaphore=semaphore, + ), + ) + ) + ) + manifest = await asyncio.gather(*tasks) + + for url in downloads: + downloads[url]["download_path"].unlink() + shutil.rmtree(downloads[url]["extracted_path"]) + shutil.rmtree(tools_dir) + + return manifest + + +def main(): + args = process_arguments() + + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") + log.setLevel(args.log_level) + + signing_cert = args.signing_cert.read() + assert get_keysize(signing_cert) == 4096 + + artifacts_dir = Path(args.artifacts_dir) + if not artifacts_dir.exists(): + artifacts_dir.mkdir() + + loop = asyncio.get_event_loop() + manifest = loop.run_until_complete(async_main(args, signing_cert)) + loop.close() + + manifest_file = artifacts_dir / "manifest.json" + with open(manifest_file, "w") as fp: + json.dump(manifest, fp, indent=2, sort_keys=True) + + log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True))) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh new file mode 100755 index 0000000000..965d938247 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool contains functions that are to be used to handle/enable funsize +# Author: Mihai Tabara +# + +HOOK= +AWS_BUCKET_NAME= +LOCAL_CACHE_DIR= + +# Don't cache files smaller than this, as it's slower with S3 +# Bug 1437473 +CACHE_THRESHOLD=500000 + +S3_CACHE_HITS=0 +S3_CACHE_MISSES=0 + +getsha512(){ + openssl sha512 "${1}" | awk '{print $2}' +} + +print_usage(){ + echo "$(basename "$0") [-S S3-BUCKET-NAME] [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" + echo "Script that saves/retrieves from cache presumptive patches as args" + echo "" + echo "-A SERVER-URL - host where to send the files" + echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" + echo "-g pre hook - tests whether patch already in cache" + echo "-u post hook - upload patch to cache for future use" + echo "" + echo "PATH-FROM-URL : path on disk for source file" + echo "PATH-TO-URL : path on disk for destination file" + echo "PATH-PATCH : path on disk for patch between source and destination" +} + +upload_patch(){ + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 0 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + patch_path="$3" + patch_filename="$(basename "$3")" + + # save to local cache first + if [ -n "$LOCAL_CACHE_DIR" ]; then + local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" + if $local_cmd >&2; then + cp -avf "${patch_path}" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" + echo "${patch_path} saved on local cache." + fi + fi + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${patch_filename}" + if aws s3 cp "${patch_path}" "${BUCKET_PATH}"; then + echo "${patch_path} saved on s://${AWS_BUCKET_NAME}" + return 0 + fi + echo "${patch_path} failed to be uploaded to s3://${AWS_BUCKET_NAME}" + return 1 + fi + return 0 +} + +get_patch(){ + # $1 and $2 are the /path/to/filename + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 1 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + destination_file="$3" + s3_filename="$(basename "$3")" + + # Try to retrieve from local cache first. + if [ -n "$LOCAL_CACHE_DIR" ]; then + if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then + cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" + echo "Successful retrieved ${destination_file} from local cache." + return 0 + fi + fi + # If not in the local cache, we might find it remotely. + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${s3_filename}" + if aws s3 ls "${BUCKET_PATH}"; then + ((S3_CACHE_HITS++)) + echo "s3 cache hit for ${s3_filename} (${S3_CACHE_HITS} total hits)" + if aws s3 cp "${BUCKET_PATH}" "${destination_file}"; then + echo "Successful retrieved ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 0 + else + echo "Failed to retrieve ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 1 + fi + # Not found, fall through to default error + else + ((S3_CACHE_MISSES++)) + echo "s3 cache miss for ${s3_filename} (${S3_CACHE_MISSES} total misses)" + fi + fi + return 1 +} + +OPTIND=1 + +while getopts ":S:c:gu" option; do + case $option in + S) + # This will probably be bucketname/path/prefix but we can use it either way + AWS_BUCKET_NAME="$OPTARG" + # Ensure trailing slash is there. + if [[ ! $AWS_BUCKET_NAME =~ .*/$ ]]; then + AWS_BUCKET_NAME="${AWS_BUCKET_NAME}/" + fi + ;; + c) + LOCAL_CACHE_DIR="$OPTARG" + ;; + g) + HOOK="PRE" + ;; + u) + HOOK="POST" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_usage + exit 1 + ;; + *) + echo "Unimplemented option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$HOOK" == "PRE" ]; then + get_patch "$1" "$2" "$3" +elif [ "$HOOK" == "POST" ]; then + upload_patch "$1" "$2" "$3" +fi |