Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 349 insertions, 0 deletions

diff --git a/taskcluster/docker/image_builder/build-image/src/config.rs b/taskcluster/docker/image_builder/build-image/src/config.rs
new file mode 100644
index 0000000000..94c1d55a10
--- /dev/null
+++ b/taskcluster/docker/image_builder/build-image/src/config.rs
@@ -0,0 +1,112 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+use anyhow::Result;
+use serde::de::Error;
+use serde::Deserialize;
+use std::collections::HashMap;
+
+fn default_image_name() -> String {
+    "mozilla.org/taskgraph/default-image:latest".into()
+}
+fn default_zstd_level() -> i32 {
+    3
+}
+
+fn from_json<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+    T: serde::de::DeserializeOwned,
+{
+    let value: String = serde::Deserialize::deserialize(deserializer)?;
+    serde_json::from_str(&value).map_err(|err| {
+        D::Error::invalid_value(serde::de::Unexpected::Str(&value), &&*err.to_string())
+    })
+}
+
+#[derive(Deserialize, Debug, PartialEq, Eq)]
+pub struct Config {
+    pub context_task_id: String,
+    pub context_path: String,
+    pub parent_task_id: Option<String>,
+    #[serde(default = "default_image_name")]
+    pub image_name: String,
+    #[serde(default = "default_zstd_level")]
+    pub docker_image_zstd_level: i32,
+    #[serde(default)]
+    pub debug: bool,
+    #[serde(default, deserialize_with = "from_json")]
+    pub docker_build_args: HashMap<String, String>,
+}
+
+impl Config {
+    pub fn from_env() -> Result<Config> {
+        Ok(envy::from_env()?)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use anyhow::Result;
+
+    #[test]
+    fn test() -> Result<()> {
+        let env: Vec<(String, String)> = vec![
+            ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()),
+            (
+                "CONTEXT_PATH".into(),
+                "public/docker-contexts/image.tar.gz".into(),
+            ),
+        ];
+        let config: super::Config = envy::from_iter(env.into_iter())?;
+        assert_eq!(
+            config,
+            super::Config {
+                context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(),
+                context_path: "public/docker-contexts/image.tar.gz".into(),
+                parent_task_id: None,
+                image_name: "mozilla.org/taskgraph/default-image:latest".into(),
+                docker_image_zstd_level: 3,
+                debug: false,
+                docker_build_args: Default::default()
+            }
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_docker_build_args() -> Result<()> {
+        let env: Vec<(String, String)> = vec![
+            ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()),
+            (
+                "CONTEXT_PATH".into(),
+                "public/docker-contexts/image.tar.gz".into(),
+            ),
+            (
+                "DOCKER_BUILD_ARGS".into(),
+                serde_json::json! ({
+                    "test": "Value",
+                })
+                .to_string(),
+            ),
+        ];
+        let config: super::Config = envy::from_iter(env.into_iter())?;
+        assert_eq!(
+            config,
+            super::Config {
+                context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(),
+                context_path: "public/docker-contexts/image.tar.gz".into(),
+                parent_task_id: None,
+                image_name: "mozilla.org/taskgraph/default-image:latest".into(),
+                docker_image_zstd_level: 3,
+                debug: false,
+                docker_build_args: [("test".to_string(), "Value".to_string())]
+                    .iter()
+                    .cloned()
+                    .collect(),
+            }
+        );
+        Ok(())
+    }
+}
diff --git a/taskcluster/docker/image_builder/build-image/src/main.rs b/taskcluster/docker/image_builder/build-image/src/main.rs
new file mode 100644
index 0000000000..997617c84e
--- /dev/null
+++ b/taskcluster/docker/image_builder/build-image/src/main.rs
@@ -0,0 +1,182 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#![forbid(unsafe_code)]
+
+use std::collections::HashMap;
+use std::path::Path;
+use std::process::Command;
+
+use anyhow::{ensure, Context, Result};
+use fs_extra::dir::{move_dir, CopyOptions};
+use serde::Deserialize;
+
+mod config;
+mod taskcluster;
+
+use config::Config;
+
+fn log_step(msg: &str) {
+    println!("[build-image] {}", msg);
+}
+
+fn read_image_digest(path: &str) -> Result<String> {
+    let output = Command::new("/kaniko/skopeo")
+        .arg("inspect")
+        .arg(format!("docker-archive:{}", path))
+        .stdout(std::process::Stdio::piped())
+        .spawn()?
+        .wait_with_output()?;
+    ensure!(output.status.success(), "Could not inspect parent image.");
+
+    #[derive(Deserialize, Debug)]
+    #[serde(rename_all = "PascalCase")]
+    struct ImageInfo {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        name: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        tag: Option<String>,
+        digest: String,
+        // ...
+    }
+
+    let image_info: ImageInfo = serde_json::from_slice(&output.stdout)
+        .with_context(|| format!("Could parse image info from {:?}", path))?;
+    Ok(image_info.digest)
+}
+
+fn download_parent_image(
+    cluster: &taskcluster::TaskCluster,
+    task_id: &str,
+    dest: &str,
+) -> Result<String> {
+    zstd::stream::copy_decode(
+        cluster.stream_artifact(&task_id, "public/image.tar.zst")?,
+        std::fs::File::create(dest)?,
+    )
+    .context("Could not download parent image.")?;
+
+    read_image_digest(dest)
+}
+
+fn build_image(
+    context_path: &str,
+    dest: &str,
+    debug: bool,
+    build_args: HashMap<String, String>,
+) -> Result<()> {
+    let mut command = Command::new("/kaniko/executor");
+    command
+        .stderr(std::process::Stdio::inherit())
+        .args(&["--context", &format!("tar://{}", context_path)])
+        .args(&["--destination", "image"])
+        .args(&["--dockerfile", "Dockerfile"])
+        .arg("--no-push")
+        .args(&["--cache-dir", "/workspace/cache"])
+        .arg("--single-snapshot")
+        // FIXME: Generating reproducible layers currently causes OOM.
+        // .arg("--reproducible")
+        .arg("--whitelist-var-run=false")
+        .args(&["--tarPath", dest]);
+    if debug {
+        command.args(&["-v", "debug"]);
+    }
+    for (key, value) in build_args {
+        command.args(&["--build-arg", &format!("{}={}", key, value)]);
+    }
+    let status = command.status()?;
+    ensure!(status.success(), "Could not build image.");
+    Ok(())
+}
+
+fn repack_image(source: &str, dest: &str, image_name: &str) -> Result<()> {
+    let status = Command::new("/kaniko/skopeo")
+        .arg("copy")
+        .arg(format!("docker-archive:{}", source))
+        .arg(format!("docker-archive:{}:{}", dest, image_name))
+        .stderr(std::process::Stdio::inherit())
+        .status()?;
+    ensure!(status.success(), "Could repack image.");
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    // Kaniko expects everything to be in /kaniko, so if not running from there, move
+    // everything there.
+    if let Some(path) = std::env::current_exe()?.parent() {
+        if path != Path::new("/kaniko") {
+            let mut options = CopyOptions::new();
+            options.copy_inside = true;
+            move_dir(path, "/kaniko", &options)?;
+        }
+    }
+
+    let config = Config::from_env().context("Could not parse environment variables.")?;
+
+    let cluster = taskcluster::TaskCluster::from_env()?;
+
+    let mut build_args = config.docker_build_args;
+
+    build_args.insert("TASKCLUSTER_ROOT_URL".into(), cluster.root_url());
+
+    log_step("Downloading context.");
+
+    std::io::copy(
+        &mut cluster.stream_artifact(&config.context_task_id, &config.context_path)?,
+        &mut std::fs::File::create("/workspace/context.tar.gz")?,
+    )
+    .context("Could not download image context.")?;
+
+    if let Some(parent_task_id) = config.parent_task_id {
+        log_step("Downloading image.");
+        let digest = download_parent_image(&cluster, &parent_task_id, "/workspace/parent.tar")?;
+
+        log_step(&format!("Parent image digest {}", &digest));
+        std::fs::create_dir_all("/workspace/cache")?;
+        std::fs::rename(
+            "/workspace/parent.tar",
+            format!("/workspace/cache/{}", digest),
+        )?;
+
+        build_args.insert(
+            "DOCKER_IMAGE_PARENT".into(),
+            format!("parent:latest@{}", digest),
+        );
+    }
+
+    log_step("Building image.");
+    build_image(
+        "/workspace/context.tar.gz",
+        "/workspace/image-pre.tar",
+        config.debug,
+        build_args,
+    )?;
+    log_step("Repacking image.");
+    repack_image(
+        "/workspace/image-pre.tar",
+        "/workspace/image.tar",
+        &config.image_name,
+    )?;
+
+    log_step("Compressing image.");
+    compress_file(
+        "/workspace/image.tar",
+        "/workspace/image.tar.zst",
+        config.docker_image_zstd_level,
+    )?;
+
+    Ok(())
+}
+
+fn compress_file(
+    source: impl AsRef<std::path::Path>,
+    dest: impl AsRef<std::path::Path>,
+    zstd_level: i32,
+) -> Result<()> {
+    Ok(zstd::stream::copy_encode(
+        std::fs::File::open(source)?,
+        std::fs::File::create(dest)?,
+        zstd_level,
+    )?)
+}
diff --git a/taskcluster/docker/image_builder/build-image/src/taskcluster.rs b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs
new file mode 100644
index 0000000000..3b39d669f0
--- /dev/null
+++ b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs
@@ -0,0 +1,55 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+use anyhow::{Context, Result};
+
+pub struct TaskCluster {
+    root_url: url::Url,
+    client: reqwest::blocking::Client,
+}
+
+impl TaskCluster {
+    pub fn from_env() -> Result<Self> {
+        std::env::var("TASKCLUSTER_ROOT_URL")
+            .context("TASKCLUSTER_ROOT_URL not set.")
+            .and_then(|var| var.parse().context("Couldn't parse TASKCLUSTER_ROOT_URL."))
+            .map(|root_url| TaskCluster {
+                root_url,
+                client: reqwest::blocking::Client::new(),
+            })
+    }
+
+    /// Return the root URL as suitable for passing to other processes.
+    ///
+    /// In particular, any trailing slashes are removed.
+    pub fn root_url(&self) -> String {
+        self.root_url.as_str().trim_end_matches("/").to_string()
+    }
+
+    pub fn task_artifact_url(&self, task_id: &str, path: &str) -> url::Url {
+        let mut url = self.root_url.clone();
+        url.set_path(&format!("api/queue/v1/task/{}/artifacts/{}", task_id, path));
+        url
+    }
+
+    pub fn stream_artifact(&self, task_id: &str, path: &str) -> Result<impl std::io::Read> {
+        let url = self.task_artifact_url(task_id, path);
+        Ok(self.client.get(url).send()?.error_for_status()?)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_url() {
+        let cluster = super::TaskCluster {
+            root_url: url::Url::parse("http://taskcluster.example").unwrap(),
+            client: reqwest::blocking::Client::new(),
+        };
+        assert_eq!(
+            cluster.task_artifact_url("QzDLgP4YRwanIvgPt6ClfA","public/docker-contexts/decision.tar.gz"),
+            url::Url::parse("http://taskcluster.example/api/queue/v1/task/QzDLgP4YRwanIvgPt6ClfA/artifacts/public/docker-contexts/decision.tar.gz").unwrap(),
+        );
+    }
+}