From b65b89d538e8c6adad31b84584fe2c53ba8ebc09 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 16 Apr 2024 19:48:08 +0200 Subject: Adding upstream version 0.14.0+ds1. Signed-off-by: Daniel Baumann --- .codecov.yaml | 2 + .gitattributes | 7 + .github/ISSUE_TEMPLATE/crane_bug_report.md | 25 + .github/ISSUE_TEMPLATE/ggcr_bug_report.md | 25 + .github/ISSUE_TEMPLATE/question.md | 9 + .github/dependabot.yml | 6 + .github/workflows/analyze.yaml | 25 + .github/workflows/boilerplate.yaml | 33 + .github/workflows/build.yaml | 26 + .github/workflows/bump-deps.yaml | 35 + .github/workflows/donotsubmit.yaml | 15 + .github/workflows/e2e.yaml | 94 + .github/workflows/ecr-auth.yaml | 93 + .github/workflows/ghcr-auth.yaml | 47 + .github/workflows/presubmit.yaml | 34 + .github/workflows/release.yml | 78 + .github/workflows/stale.yaml | 30 + .github/workflows/style.yaml | 55 + .github/workflows/test.yaml | 28 + .gitignore | 12 + .golangci.yaml | 40 + .goreleaser.yml | 119 + .ko/debug/.ko.yaml | 1 + .wokeignore | 1 + CONTRIBUTING.md | 36 + LICENSE | 202 + README.md | 150 + SECURITY.md | 4 + cloudbuild.yaml | 61 + cmd/crane/README.md | 122 + cmd/crane/cmd/append.go | 122 + cmd/crane/cmd/auth.go | 205 + cmd/crane/cmd/blob.go | 48 + cmd/crane/cmd/catalog.go | 54 + cmd/crane/cmd/config.go | 39 + cmd/crane/cmd/copy.go | 34 + cmd/crane/cmd/delete.go | 33 + cmd/crane/cmd/digest.go | 91 + cmd/crane/cmd/export.go | 89 + cmd/crane/cmd/flatten.go | 254 + cmd/crane/cmd/index.go | 291 ++ cmd/crane/cmd/list.go | 62 + cmd/crane/cmd/manifest.go | 40 + cmd/crane/cmd/mutate.go | 207 + cmd/crane/cmd/optimize.go | 42 + cmd/crane/cmd/pull.go | 138 + cmd/crane/cmd/push.go | 126 + cmd/crane/cmd/rebase.go | 210 + cmd/crane/cmd/root.go | 148 + cmd/crane/cmd/serve.go | 84 + cmd/crane/cmd/tag.go | 44 + cmd/crane/cmd/util.go | 86 + cmd/crane/cmd/validate.go | 73 + cmd/crane/cmd/version.go | 56 + cmd/crane/depcheck_test.go | 32 + cmd/crane/doc/crane.md | 42 + cmd/crane/doc/crane_append.md | 43 + cmd/crane/doc/crane_auth.md | 29 + cmd/crane/doc/crane_auth_get.md | 38 + cmd/crane/doc/crane_auth_login.md | 37 + cmd/crane/doc/crane_blob.md | 33 + cmd/crane/doc/crane_catalog.md | 34 + cmd/crane/doc/crane_config.md | 27 + cmd/crane/doc/crane_copy.md | 27 + cmd/crane/doc/crane_delete.md | 27 + cmd/crane/doc/crane_digest.md | 29 + cmd/crane/doc/crane_export.md | 40 + cmd/crane/doc/crane_flatten.md | 28 + cmd/crane/doc/crane_index.md | 29 + cmd/crane/doc/crane_index_append.md | 47 + cmd/crane/doc/crane_index_filter.md | 41 + cmd/crane/doc/crane_ls.md | 29 + cmd/crane/doc/crane_manifest.md | 27 + cmd/crane/doc/crane_mutate.md | 37 + cmd/crane/doc/crane_pull.md | 30 + cmd/crane/doc/crane_push.md | 33 + cmd/crane/doc/crane_rebase.md | 32 + cmd/crane/doc/crane_registry.md | 24 + cmd/crane/doc/crane_registry_serve.md | 35 + cmd/crane/doc/crane_tag.md | 46 + cmd/crane/doc/crane_validate.md | 30 + cmd/crane/doc/crane_version.md | 34 + cmd/crane/help/README.md | 5 + cmd/crane/help/main.go | 45 + cmd/crane/main.go | 38 + cmd/crane/rebase.md | 125 + cmd/crane/rebase.png | Bin 0 -> 49992 bytes cmd/crane/rebase_test.sh | 62 + cmd/crane/recipes.md | 105 + cmd/gcrane/README.md | 65 + cmd/gcrane/cmd/copy.go | 47 + cmd/gcrane/cmd/gc.go | 76 + cmd/gcrane/cmd/list.go | 121 + cmd/gcrane/depcheck_test.go | 32 + cmd/gcrane/main.go | 72 + cmd/ko/README.md | 3 + cmd/krane/README.md | 15 + cmd/krane/go.mod | 67 + cmd/krane/go.sum | 199 + cmd/krane/main.go | 67 + cmd/registry/main.go | 44 + cmd/registry/test.sh | 57 + go.mod | 49 + go.sum | 153 + hack/boilerplate/boilerplate.go.txt | 13 + hack/bump-deps.sh | 47 + hack/presubmit.sh | 58 + hack/update-codegen.sh | 52 + hack/update-deps.sh | 31 + hack/update-dots.sh | 30 + images/containerd.dot.svg | 2074 ++++++++ images/containers.dot.svg | 5365 ++++++++++++++++++++ images/crane.png | Bin 0 -> 539880 bytes images/credhelper-basic.svg | 1 + images/credhelper-oauth.svg | 1 + images/docker.dot.svg | 2155 ++++++++ images/dot/containerd.dot | 316 ++ images/dot/containers.dot | 831 +++ images/dot/docker.dot | 327 ++ images/dot/ggcr.dot | 130 + images/dot/image-anatomy.dot | 26 + images/dot/index-anatomy-strange.dot | 24 + images/dot/index-anatomy.dot | 18 + images/dot/mutate.dot | 59 + images/dot/remote.dot | 66 + images/dot/stream.dot | 47 + images/dot/tarball.dot | 43 + images/dot/upload.dot | 67 + images/gcrane.png | Bin 0 -> 561713 bytes images/ggcr.dot.svg | 874 ++++ images/image-anatomy.dot.svg | 99 + images/index-anatomy-strange.dot.svg | 125 + images/index-anatomy.dot.svg | 85 + images/mutate.dot.svg | 250 + images/ociimage.gv | 97 + images/ociimage.jpeg | Bin 0 -> 114782 bytes images/remote.dot.svg | 180 + images/stream.dot.svg | 217 + images/tarball.dot.svg | 126 + images/upload.dot.svg | 359 ++ internal/and/and_closer.go | 48 + internal/and/and_closer_test.go | 85 + internal/cmd/edit.go | 485 ++ internal/cmd/edit_test.go | 174 + internal/compare/doc.go | 16 + internal/compare/image.go | 111 + internal/compare/image_test.go | 66 + internal/compare/index.go | 83 + internal/compare/index_test.go | 51 + internal/compare/layer.go | 80 + internal/compare/layer_test.go | 48 + internal/compression/compression.go | 97 + internal/compression/compression_test.go | 78 + internal/depcheck/depcheck.go | 186 + internal/editor/editor.go | 64 + internal/estargz/estargz.go | 54 + internal/estargz/estargz_test.go | 108 + internal/gzip/zip.go | 118 + internal/gzip/zip_test.go | 98 + internal/httptest/httptest.go | 104 + internal/legacy/copy.go | 57 + internal/legacy/copy_test.go | 97 + internal/redact/redact.go | 89 + internal/retry/retry.go | 94 + internal/retry/retry_test.go | 100 + .../retry/wait/kubernetes_apimachinery_wait.go | 123 + internal/verify/verify.go | 122 + internal/verify/verify_test.go | 147 + internal/windows/windows.go | 114 + internal/windows/windows_test.go | 81 + internal/zstd/zstd.go | 116 + internal/zstd/zstd_test.go | 96 + pkg/authn/README.md | 322 ++ pkg/authn/anon.go | 26 + pkg/authn/anon_test.go | 31 + pkg/authn/auth.go | 30 + pkg/authn/authn.go | 115 + pkg/authn/authn_test.go | 148 + pkg/authn/basic.go | 29 + pkg/authn/basic_test.go | 33 + pkg/authn/bearer.go | 27 + pkg/authn/bearer_test.go | 31 + pkg/authn/doc.go | 17 + pkg/authn/github/keychain.go | 59 + pkg/authn/github/keychain_test.go | 112 + pkg/authn/k8schain/README.md | 49 + pkg/authn/k8schain/doc.go | 18 + pkg/authn/k8schain/go.mod | 96 + pkg/authn/k8schain/go.sum | 364 ++ pkg/authn/k8schain/k8schain.go | 105 + pkg/authn/k8schain/tests/explicit/main.go | 52 + pkg/authn/k8schain/tests/explicit/test.yaml | 59 + pkg/authn/k8schain/tests/implicit/main.go | 52 + pkg/authn/k8schain/tests/implicit/test.yaml | 47 + pkg/authn/k8schain/tests/noauth/main.go | 47 + pkg/authn/k8schain/tests/noauth/test.yaml | 44 + pkg/authn/k8schain/tests/serviceaccount/main.go | 54 + pkg/authn/k8schain/tests/serviceaccount/test.yaml | 67 + pkg/authn/keychain.go | 180 + pkg/authn/keychain_test.go | 392 ++ pkg/authn/kubernetes/go.mod | 59 + pkg/authn/kubernetes/go.sum | 276 + pkg/authn/kubernetes/keychain.go | 331 ++ pkg/authn/kubernetes/keychain_test.go | 586 +++ pkg/authn/multikeychain.go | 41 + pkg/authn/multikeychain_test.go | 98 + pkg/compression/compression.go | 26 + pkg/crane/append.go | 114 + pkg/crane/append_test.go | 73 + pkg/crane/catalog.go | 35 + pkg/crane/config.go | 24 + pkg/crane/copy.go | 88 + pkg/crane/crane_test.go | 574 +++ pkg/crane/delete.go | 33 + pkg/crane/digest.go | 52 + pkg/crane/digest_test.go | 61 + pkg/crane/doc.go | 16 + pkg/crane/example_test.go | 31 + pkg/crane/export.go | 47 + pkg/crane/export_test.go | 41 + pkg/crane/filemap.go | 72 + pkg/crane/filemap_test.go | 187 + pkg/crane/get.go | 56 + pkg/crane/list.go | 33 + pkg/crane/manifest.go | 32 + pkg/crane/optimize.go | 237 + pkg/crane/optimize_test.go | 179 + pkg/crane/options.go | 149 + pkg/crane/options_test.go | 58 + pkg/crane/pull.go | 142 + pkg/crane/push.go | 65 + pkg/crane/tag.go | 39 + pkg/crane/testdata/content.tar | Bin 0 -> 10240 bytes pkg/gcrane/copy.go | 347 ++ pkg/gcrane/copy_test.go | 428 ++ pkg/gcrane/doc.go | 16 + pkg/gcrane/options.go | 122 + pkg/gcrane/options_test.go | 58 + pkg/legacy/config.go | 33 + pkg/legacy/doc.go | 18 + pkg/legacy/tarball/README.md | 6 + pkg/legacy/tarball/doc.go | 18 + pkg/legacy/tarball/write.go | 374 ++ pkg/legacy/tarball/write_test.go | 615 +++ pkg/logs/logs.go | 39 + pkg/name/README.md | 3 + pkg/name/check.go | 43 + pkg/name/digest.go | 94 + pkg/name/digest_test.go | 152 + pkg/name/doc.go | 42 + pkg/name/errors.go | 48 + pkg/name/errors_test.go | 37 + pkg/name/internal/must_test.go | 27 + pkg/name/internal/must_test.sh | 29 + pkg/name/options.go | 83 + pkg/name/ref.go | 75 + pkg/name/ref_test.go | 157 + pkg/name/registry.go | 136 + pkg/name/registry_test.go | 252 + pkg/name/repository.go | 121 + pkg/name/repository_test.go | 145 + pkg/name/tag.go | 108 + pkg/name/tag_test.go | 162 + pkg/registry/README.md | 14 + pkg/registry/blobs.go | 483 ++ pkg/registry/compatibility_test.go | 63 + pkg/registry/depcheck_test.go | 38 + pkg/registry/error.go | 79 + pkg/registry/manifest.go | 430 ++ pkg/registry/registry.go | 117 + pkg/registry/registry_test.go | 609 +++ pkg/registry/tls.go | 29 + pkg/registry/tls_test.go | 49 + pkg/v1/cache/cache.go | 194 + pkg/v1/cache/cache_test.go | 154 + pkg/v1/cache/example_test.go | 46 + pkg/v1/cache/fs.go | 151 + pkg/v1/cache/fs_test.go | 213 + pkg/v1/cache/ro.go | 27 + pkg/v1/cache/ro_test.go | 79 + pkg/v1/config.go | 151 + pkg/v1/config_test.go | 38 + pkg/v1/daemon/README.md | 11 + pkg/v1/daemon/doc.go | 17 + pkg/v1/daemon/image.go | 203 + pkg/v1/daemon/image_test.go | 159 + pkg/v1/daemon/options.go | 103 + pkg/v1/daemon/write.go | 60 + pkg/v1/daemon/write_test.go | 159 + pkg/v1/doc.go | 18 + pkg/v1/empty/README.md | 8 + pkg/v1/empty/doc.go | 16 + pkg/v1/empty/image.go | 52 + pkg/v1/empty/image_test.go | 48 + pkg/v1/empty/index.go | 64 + pkg/v1/empty/index_test.go | 40 + pkg/v1/fake/image.go | 826 +++ pkg/v1/fake/index.go | 546 ++ pkg/v1/google/README.md | 7 + pkg/v1/google/auth.go | 179 + pkg/v1/google/auth_test.go | 270 + pkg/v1/google/doc.go | 16 + pkg/v1/google/keychain.go | 92 + pkg/v1/google/list.go | 331 ++ pkg/v1/google/list_test.go | 339 ++ pkg/v1/google/options.go | 73 + pkg/v1/google/testdata/README.md | 4 + pkg/v1/google/testdata/key.json | 35 + pkg/v1/hash.go | 123 + pkg/v1/hash_test.go | 115 + pkg/v1/image.go | 59 + pkg/v1/index.go | 43 + pkg/v1/layer.go | 42 + pkg/v1/layout/README.md | 5 + pkg/v1/layout/blob.go | 37 + pkg/v1/layout/doc.go | 19 + pkg/v1/layout/image.go | 139 + pkg/v1/layout/image_test.go | 181 + pkg/v1/layout/index.go | 161 + pkg/v1/layout/index_test.go | 81 + pkg/v1/layout/layoutpath.go | 25 + pkg/v1/layout/options.go | 71 + pkg/v1/layout/read.go | 32 + pkg/v1/layout/read_test.go | 42 + pkg/v1/layout/testdata/README.md | 5 + ...183c1e2da98610e91372fa9f510046d4ce5812addad86b5 | 13 + ...a7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb | 13 + ...950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 | Bin 0 -> 165 bytes ...56033bb3334432a0a513bf9d6aceda0f67c42b003850720 | 1 + ...2e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e | 1 + ...c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 | 1 + ...047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b | Bin 0 -> 167 bytes ...59b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 | 1 + pkg/v1/layout/testdata/test_index/index.json | 37 + pkg/v1/layout/testdata/test_index/oci-layout | 3 + ...9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e | 15 + ...27226140568f3bef7eaac187cebd76878e0b63e9e442356 | 1 + ...047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b | Bin 0 -> 167 bytes .../testdata/test_index_media_type/index.json | 10 + .../testdata/test_index_media_type/oci-layout | 3 + ...9574ab5c066e9f6116b5aec3567675aa13bec63331f0810 | 1 + ...6f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 | Bin 0 -> 114 bytes ...9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46 | 1 + .../testdata/test_index_one_image/index.json | 1 + .../testdata/test_index_one_image/oci-layout | 1 + pkg/v1/layout/write.go | 481 ++ pkg/v1/layout/write_test.go | 672 +++ pkg/v1/manifest.go | 71 + pkg/v1/manifest_test.go | 76 + pkg/v1/match/match.go | 92 + pkg/v1/match/match_test.go | 131 + pkg/v1/mutate/README.md | 56 + pkg/v1/mutate/doc.go | 16 + pkg/v1/mutate/image.go | 287 ++ pkg/v1/mutate/index.go | 204 + pkg/v1/mutate/index_test.go | 235 + pkg/v1/mutate/mutate.go | 553 ++ pkg/v1/mutate/mutate_test.go | 770 +++ pkg/v1/mutate/rebase.go | 144 + pkg/v1/mutate/rebase_test.go | 179 + pkg/v1/mutate/testdata/README.md | 10 + pkg/v1/mutate/testdata/bar | 1 + pkg/v1/mutate/testdata/foo | 1 + pkg/v1/mutate/testdata/overwritten_file.tar | Bin 0 -> 51200 bytes pkg/v1/mutate/testdata/source_image.tar | Bin 0 -> 20480 bytes .../source_image_with_empty_layer_history.tar | Bin 0 -> 20480 bytes pkg/v1/mutate/testdata/whiteout/bar.txt | 1 + pkg/v1/mutate/testdata/whiteout/foo.txt | 1 + pkg/v1/mutate/testdata/whiteout_image.tar | Bin 0 -> 51200 bytes pkg/v1/mutate/whiteout_test.go | 43 + pkg/v1/partial/README.md | 82 + pkg/v1/partial/compressed.go | 188 + pkg/v1/partial/compressed_test.go | 193 + pkg/v1/partial/configlayer_test.go | 139 + pkg/v1/partial/doc.go | 17 + pkg/v1/partial/image.go | 28 + pkg/v1/partial/index.go | 85 + pkg/v1/partial/index_test.go | 119 + pkg/v1/partial/uncompressed.go | 223 + pkg/v1/partial/uncompressed_test.go | 233 + pkg/v1/partial/with.go | 436 ++ pkg/v1/partial/with_test.go | 246 + pkg/v1/platform.go | 149 + pkg/v1/platform_test.go | 235 + pkg/v1/progress.go | 25 + pkg/v1/random/doc.go | 16 + pkg/v1/random/image.go | 116 + pkg/v1/random/image_test.go | 129 + pkg/v1/random/index.go | 111 + pkg/v1/random/index_test.go | 64 + pkg/v1/remote/README.md | 117 + pkg/v1/remote/catalog.go | 154 + pkg/v1/remote/catalog_test.go | 183 + pkg/v1/remote/check.go | 72 + pkg/v1/remote/check_e2e_test.go | 46 + pkg/v1/remote/check_test.go | 76 + pkg/v1/remote/delete.go | 61 + pkg/v1/remote/delete_test.go | 89 + pkg/v1/remote/descriptor.go | 511 ++ pkg/v1/remote/descriptor_test.go | 259 + pkg/v1/remote/doc.go | 17 + pkg/v1/remote/error_roundtrip_test.go | 127 + pkg/v1/remote/image.go | 256 + pkg/v1/remote/image_test.go | 743 +++ pkg/v1/remote/index.go | 319 ++ pkg/v1/remote/index_test.go | 504 ++ pkg/v1/remote/layer.go | 94 + pkg/v1/remote/layer_test.go | 148 + pkg/v1/remote/list.go | 141 + pkg/v1/remote/list_test.go | 159 + pkg/v1/remote/mount.go | 108 + pkg/v1/remote/mount_test.go | 55 + pkg/v1/remote/multi_write.go | 302 ++ pkg/v1/remote/multi_write_test.go | 351 ++ pkg/v1/remote/options.go | 317 ++ pkg/v1/remote/progress.go | 69 + pkg/v1/remote/progress_test.go | 463 ++ pkg/v1/remote/referrers.go | 35 + pkg/v1/remote/referrers_test.go | 183 + pkg/v1/remote/transport/README.md | 129 + pkg/v1/remote/transport/basic.go | 62 + pkg/v1/remote/transport/basic_test.go | 138 + pkg/v1/remote/transport/bearer.go | 320 ++ pkg/v1/remote/transport/bearer_test.go | 561 ++ pkg/v1/remote/transport/doc.go | 18 + pkg/v1/remote/transport/error.go | 173 + pkg/v1/remote/transport/error_test.go | 236 + pkg/v1/remote/transport/logger.go | 91 + pkg/v1/remote/transport/logger_test.go | 93 + pkg/v1/remote/transport/ping.go | 227 + pkg/v1/remote/transport/ping_test.go | 260 + pkg/v1/remote/transport/retry.go | 111 + pkg/v1/remote/transport/retry_test.go | 177 + pkg/v1/remote/transport/schemer.go | 44 + pkg/v1/remote/transport/scope.go | 24 + pkg/v1/remote/transport/transport.go | 116 + pkg/v1/remote/transport/transport_test.go | 282 + pkg/v1/remote/transport/useragent.go | 94 + pkg/v1/remote/write.go | 1003 ++++ pkg/v1/remote/write_test.go | 1643 ++++++ pkg/v1/static/layer.go | 68 + pkg/v1/static/static_test.go | 83 + pkg/v1/stream/README.md | 68 + pkg/v1/stream/layer.go | 273 + pkg/v1/stream/layer_test.go | 298 ++ pkg/v1/tarball/README.md | 280 + pkg/v1/tarball/doc.go | 17 + pkg/v1/tarball/image.go | 429 ++ pkg/v1/tarball/image_test.go | 139 + pkg/v1/tarball/layer.go | 349 ++ pkg/v1/tarball/layer_test.go | 381 ++ pkg/v1/tarball/progress_test.go | 57 + pkg/v1/tarball/testdata/bar | 1 + pkg/v1/tarball/testdata/bat/bat | 1 + pkg/v1/tarball/testdata/baz | 1 + pkg/v1/tarball/testdata/content.tar | Bin 0 -> 10240 bytes pkg/v1/tarball/testdata/foo | 1 + pkg/v1/tarball/testdata/no_manifest.tar | Bin 0 -> 20480 bytes pkg/v1/tarball/testdata/null_manifest.tar | Bin 0 -> 2048 bytes pkg/v1/tarball/testdata/test_bundle.tar | Bin 0 -> 40960 bytes pkg/v1/tarball/testdata/test_image_1.tar | Bin 0 -> 20480 bytes pkg/v1/tarball/testdata/test_image_2.tar | Bin 0 -> 20480 bytes pkg/v1/tarball/testdata/test_link.tar | Bin 0 -> 29696 bytes pkg/v1/tarball/testdata/test_load_manifest.tar | Bin 0 -> 20480 bytes pkg/v1/tarball/write.go | 457 ++ pkg/v1/tarball/write_test.go | 502 ++ pkg/v1/types/types.go | 82 + pkg/v1/types/types_test.go | 112 + pkg/v1/validate/doc.go | 16 + pkg/v1/validate/image.go | 288 ++ pkg/v1/validate/index.go | 175 + pkg/v1/validate/layer.go | 191 + pkg/v1/validate/options.go | 37 + pkg/v1/zz_deepcopy_generated.go | 339 ++ 474 files changed, 65397 insertions(+) create mode 100644 .codecov.yaml create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/crane_bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/ggcr_bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/analyze.yaml create mode 100644 .github/workflows/boilerplate.yaml create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/bump-deps.yaml create mode 100644 .github/workflows/donotsubmit.yaml create mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/ecr-auth.yaml create mode 100644 .github/workflows/ghcr-auth.yaml create mode 100644 .github/workflows/presubmit.yaml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/stale.yaml create mode 100644 .github/workflows/style.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .gitignore create mode 100644 .golangci.yaml create mode 100644 .goreleaser.yml create mode 100644 .ko/debug/.ko.yaml create mode 100644 .wokeignore create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 cloudbuild.yaml create mode 100644 cmd/crane/README.md create mode 100644 cmd/crane/cmd/append.go create mode 100644 cmd/crane/cmd/auth.go create mode 100644 cmd/crane/cmd/blob.go create mode 100644 cmd/crane/cmd/catalog.go create mode 100644 cmd/crane/cmd/config.go create mode 100644 cmd/crane/cmd/copy.go create mode 100644 cmd/crane/cmd/delete.go create mode 100644 cmd/crane/cmd/digest.go create mode 100644 cmd/crane/cmd/export.go create mode 100644 cmd/crane/cmd/flatten.go create mode 100644 cmd/crane/cmd/index.go create mode 100644 cmd/crane/cmd/list.go create mode 100644 cmd/crane/cmd/manifest.go create mode 100644 cmd/crane/cmd/mutate.go create mode 100644 cmd/crane/cmd/optimize.go create mode 100644 cmd/crane/cmd/pull.go create mode 100644 cmd/crane/cmd/push.go create mode 100644 cmd/crane/cmd/rebase.go create mode 100644 cmd/crane/cmd/root.go create mode 100644 cmd/crane/cmd/serve.go create mode 100644 cmd/crane/cmd/tag.go create mode 100644 cmd/crane/cmd/util.go create mode 100644 cmd/crane/cmd/validate.go create mode 100644 cmd/crane/cmd/version.go create mode 100644 cmd/crane/depcheck_test.go create mode 100644 cmd/crane/doc/crane.md create mode 100644 cmd/crane/doc/crane_append.md create mode 100644 cmd/crane/doc/crane_auth.md create mode 100644 cmd/crane/doc/crane_auth_get.md create mode 100644 cmd/crane/doc/crane_auth_login.md create mode 100644 cmd/crane/doc/crane_blob.md create mode 100644 cmd/crane/doc/crane_catalog.md create mode 100644 cmd/crane/doc/crane_config.md create mode 100644 cmd/crane/doc/crane_copy.md create mode 100644 cmd/crane/doc/crane_delete.md create mode 100644 cmd/crane/doc/crane_digest.md create mode 100644 cmd/crane/doc/crane_export.md create mode 100644 cmd/crane/doc/crane_flatten.md create mode 100644 cmd/crane/doc/crane_index.md create mode 100644 cmd/crane/doc/crane_index_append.md create mode 100644 cmd/crane/doc/crane_index_filter.md create mode 100644 cmd/crane/doc/crane_ls.md create mode 100644 cmd/crane/doc/crane_manifest.md create mode 100644 cmd/crane/doc/crane_mutate.md create mode 100644 cmd/crane/doc/crane_pull.md create mode 100644 cmd/crane/doc/crane_push.md create mode 100644 cmd/crane/doc/crane_rebase.md create mode 100644 cmd/crane/doc/crane_registry.md create mode 100644 cmd/crane/doc/crane_registry_serve.md create mode 100644 cmd/crane/doc/crane_tag.md create mode 100644 cmd/crane/doc/crane_validate.md create mode 100644 cmd/crane/doc/crane_version.md create mode 100644 cmd/crane/help/README.md create mode 100644 cmd/crane/help/main.go create mode 100644 cmd/crane/main.go create mode 100644 cmd/crane/rebase.md create mode 100644 cmd/crane/rebase.png create mode 100755 cmd/crane/rebase_test.sh create mode 100644 cmd/crane/recipes.md create mode 100644 cmd/gcrane/README.md create mode 100644 cmd/gcrane/cmd/copy.go create mode 100644 cmd/gcrane/cmd/gc.go create mode 100644 cmd/gcrane/cmd/list.go create mode 100644 cmd/gcrane/depcheck_test.go create mode 100644 cmd/gcrane/main.go create mode 100644 cmd/ko/README.md create mode 100644 cmd/krane/README.md create mode 100644 cmd/krane/go.mod create mode 100644 cmd/krane/go.sum create mode 100644 cmd/krane/main.go create mode 100644 cmd/registry/main.go create mode 100755 cmd/registry/test.sh create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate/boilerplate.go.txt create mode 100755 hack/bump-deps.sh create mode 100755 hack/presubmit.sh create mode 100755 hack/update-codegen.sh create mode 100755 hack/update-deps.sh create mode 100755 hack/update-dots.sh create mode 100644 images/containerd.dot.svg create mode 100644 images/containers.dot.svg create mode 100644 images/crane.png create mode 100644 images/credhelper-basic.svg create mode 100644 images/credhelper-oauth.svg create mode 100644 images/docker.dot.svg create mode 100644 images/dot/containerd.dot create mode 100644 images/dot/containers.dot create mode 100644 images/dot/docker.dot create mode 100644 images/dot/ggcr.dot create mode 100644 images/dot/image-anatomy.dot create mode 100644 images/dot/index-anatomy-strange.dot create mode 100644 images/dot/index-anatomy.dot create mode 100644 images/dot/mutate.dot create mode 100644 images/dot/remote.dot create mode 100644 images/dot/stream.dot create mode 100644 images/dot/tarball.dot create mode 100644 images/dot/upload.dot create mode 100644 images/gcrane.png create mode 100644 images/ggcr.dot.svg create mode 100644 images/image-anatomy.dot.svg create mode 100644 images/index-anatomy-strange.dot.svg create mode 100644 images/index-anatomy.dot.svg create mode 100644 images/mutate.dot.svg create mode 100644 images/ociimage.gv create mode 100644 images/ociimage.jpeg create mode 100644 images/remote.dot.svg create mode 100644 images/stream.dot.svg create mode 100644 images/tarball.dot.svg create mode 100644 images/upload.dot.svg create mode 100644 internal/and/and_closer.go create mode 100644 internal/and/and_closer_test.go create mode 100644 internal/cmd/edit.go create mode 100644 internal/cmd/edit_test.go create mode 100644 internal/compare/doc.go create mode 100644 internal/compare/image.go create mode 100644 internal/compare/image_test.go create mode 100644 internal/compare/index.go create mode 100644 internal/compare/index_test.go create mode 100644 internal/compare/layer.go create mode 100644 internal/compare/layer_test.go create mode 100644 internal/compression/compression.go create mode 100644 internal/compression/compression_test.go create mode 100644 internal/depcheck/depcheck.go create mode 100644 internal/editor/editor.go create mode 100644 internal/estargz/estargz.go create mode 100644 internal/estargz/estargz_test.go create mode 100644 internal/gzip/zip.go create mode 100644 internal/gzip/zip_test.go create mode 100644 internal/httptest/httptest.go create mode 100644 internal/legacy/copy.go create mode 100644 internal/legacy/copy_test.go create mode 100644 internal/redact/redact.go create mode 100644 internal/retry/retry.go create mode 100644 internal/retry/retry_test.go create mode 100644 internal/retry/wait/kubernetes_apimachinery_wait.go create mode 100644 internal/verify/verify.go create mode 100644 internal/verify/verify_test.go create mode 100644 internal/windows/windows.go create mode 100644 internal/windows/windows_test.go create mode 100644 internal/zstd/zstd.go create mode 100644 internal/zstd/zstd_test.go create mode 100644 pkg/authn/README.md create mode 100644 pkg/authn/anon.go create mode 100644 pkg/authn/anon_test.go create mode 100644 pkg/authn/auth.go create mode 100644 pkg/authn/authn.go create mode 100644 pkg/authn/authn_test.go create mode 100644 pkg/authn/basic.go create mode 100644 pkg/authn/basic_test.go create mode 100644 pkg/authn/bearer.go create mode 100644 pkg/authn/bearer_test.go create mode 100644 pkg/authn/doc.go create mode 100644 pkg/authn/github/keychain.go create mode 100644 pkg/authn/github/keychain_test.go create mode 100644 pkg/authn/k8schain/README.md create mode 100644 pkg/authn/k8schain/doc.go create mode 100644 pkg/authn/k8schain/go.mod create mode 100644 pkg/authn/k8schain/go.sum create mode 100644 pkg/authn/k8schain/k8schain.go create mode 100644 pkg/authn/k8schain/tests/explicit/main.go create mode 100644 pkg/authn/k8schain/tests/explicit/test.yaml create mode 100644 pkg/authn/k8schain/tests/implicit/main.go create mode 100644 pkg/authn/k8schain/tests/implicit/test.yaml create mode 100644 pkg/authn/k8schain/tests/noauth/main.go create mode 100644 pkg/authn/k8schain/tests/noauth/test.yaml create mode 100644 pkg/authn/k8schain/tests/serviceaccount/main.go create mode 100644 pkg/authn/k8schain/tests/serviceaccount/test.yaml create mode 100644 pkg/authn/keychain.go create mode 100644 pkg/authn/keychain_test.go create mode 100644 pkg/authn/kubernetes/go.mod create mode 100644 pkg/authn/kubernetes/go.sum create mode 100644 pkg/authn/kubernetes/keychain.go create mode 100644 pkg/authn/kubernetes/keychain_test.go create mode 100644 pkg/authn/multikeychain.go create mode 100644 pkg/authn/multikeychain_test.go create mode 100644 pkg/compression/compression.go create mode 100644 pkg/crane/append.go create mode 100644 pkg/crane/append_test.go create mode 100644 pkg/crane/catalog.go create mode 100644 pkg/crane/config.go create mode 100644 pkg/crane/copy.go create mode 100644 pkg/crane/crane_test.go create mode 100644 pkg/crane/delete.go create mode 100644 pkg/crane/digest.go create mode 100644 pkg/crane/digest_test.go create mode 100644 pkg/crane/doc.go create mode 100644 pkg/crane/example_test.go create mode 100644 pkg/crane/export.go create mode 100644 pkg/crane/export_test.go create mode 100644 pkg/crane/filemap.go create mode 100644 pkg/crane/filemap_test.go create mode 100644 pkg/crane/get.go create mode 100644 pkg/crane/list.go create mode 100644 pkg/crane/manifest.go create mode 100644 pkg/crane/optimize.go create mode 100644 pkg/crane/optimize_test.go create mode 100644 pkg/crane/options.go create mode 100644 pkg/crane/options_test.go create mode 100644 pkg/crane/pull.go create mode 100644 pkg/crane/push.go create mode 100644 pkg/crane/tag.go create mode 100755 pkg/crane/testdata/content.tar create mode 100644 pkg/gcrane/copy.go create mode 100644 pkg/gcrane/copy_test.go create mode 100644 pkg/gcrane/doc.go create mode 100644 pkg/gcrane/options.go create mode 100644 pkg/gcrane/options_test.go create mode 100644 pkg/legacy/config.go create mode 100644 pkg/legacy/doc.go create mode 100644 pkg/legacy/tarball/README.md create mode 100644 pkg/legacy/tarball/doc.go create mode 100644 pkg/legacy/tarball/write.go create mode 100644 pkg/legacy/tarball/write_test.go create mode 100644 pkg/logs/logs.go create mode 100644 pkg/name/README.md create mode 100644 pkg/name/check.go create mode 100644 pkg/name/digest.go create mode 100644 pkg/name/digest_test.go create mode 100644 pkg/name/doc.go create mode 100644 pkg/name/errors.go create mode 100644 pkg/name/errors_test.go create mode 100644 pkg/name/internal/must_test.go create mode 100755 pkg/name/internal/must_test.sh create mode 100644 pkg/name/options.go create mode 100644 pkg/name/ref.go create mode 100644 pkg/name/ref_test.go create mode 100644 pkg/name/registry.go create mode 100644 pkg/name/registry_test.go create mode 100644 pkg/name/repository.go create mode 100644 pkg/name/repository_test.go create mode 100644 pkg/name/tag.go create mode 100644 pkg/name/tag_test.go create mode 100644 pkg/registry/README.md create mode 100644 pkg/registry/blobs.go create mode 100644 pkg/registry/compatibility_test.go create mode 100644 pkg/registry/depcheck_test.go create mode 100644 pkg/registry/error.go create mode 100644 pkg/registry/manifest.go create mode 100644 pkg/registry/registry.go create mode 100644 pkg/registry/registry_test.go create mode 100644 pkg/registry/tls.go create mode 100644 pkg/registry/tls_test.go create mode 100644 pkg/v1/cache/cache.go create mode 100644 pkg/v1/cache/cache_test.go create mode 100644 pkg/v1/cache/example_test.go create mode 100644 pkg/v1/cache/fs.go create mode 100644 pkg/v1/cache/fs_test.go create mode 100644 pkg/v1/cache/ro.go create mode 100644 pkg/v1/cache/ro_test.go create mode 100644 pkg/v1/config.go create mode 100644 pkg/v1/config_test.go create mode 100644 pkg/v1/daemon/README.md create mode 100644 pkg/v1/daemon/doc.go create mode 100644 pkg/v1/daemon/image.go create mode 100644 pkg/v1/daemon/image_test.go create mode 100644 pkg/v1/daemon/options.go create mode 100644 pkg/v1/daemon/write.go create mode 100644 pkg/v1/daemon/write_test.go create mode 100644 pkg/v1/doc.go create mode 100644 pkg/v1/empty/README.md create mode 100644 pkg/v1/empty/doc.go create mode 100644 pkg/v1/empty/image.go create mode 100644 pkg/v1/empty/image_test.go create mode 100644 pkg/v1/empty/index.go create mode 100644 pkg/v1/empty/index_test.go create mode 100644 pkg/v1/fake/image.go create mode 100644 pkg/v1/fake/index.go create mode 100644 pkg/v1/google/README.md create mode 100644 pkg/v1/google/auth.go create mode 100644 pkg/v1/google/auth_test.go create mode 100644 pkg/v1/google/doc.go create mode 100644 pkg/v1/google/keychain.go create mode 100644 pkg/v1/google/list.go create mode 100644 pkg/v1/google/list_test.go create mode 100644 pkg/v1/google/options.go create mode 100644 pkg/v1/google/testdata/README.md create mode 100644 pkg/v1/google/testdata/key.json create mode 100644 pkg/v1/hash.go create mode 100644 pkg/v1/hash_test.go create mode 100644 pkg/v1/image.go create mode 100644 pkg/v1/index.go create mode 100644 pkg/v1/layer.go create mode 100644 pkg/v1/layout/README.md create mode 100644 pkg/v1/layout/blob.go create mode 100644 pkg/v1/layout/doc.go create mode 100644 pkg/v1/layout/image.go create mode 100644 pkg/v1/layout/image_test.go create mode 100644 pkg/v1/layout/index.go create mode 100644 pkg/v1/layout/index_test.go create mode 100644 pkg/v1/layout/layoutpath.go create mode 100644 pkg/v1/layout/options.go create mode 100644 pkg/v1/layout/read.go create mode 100644 pkg/v1/layout/read_test.go create mode 100644 pkg/v1/layout/testdata/README.md create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b create mode 100644 pkg/v1/layout/testdata/test_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 create mode 100644 pkg/v1/layout/testdata/test_index/index.json create mode 100644 pkg/v1/layout/testdata/test_index/oci-layout create mode 100644 pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e create mode 100644 pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 create mode 100644 pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b create mode 100644 pkg/v1/layout/testdata/test_index_media_type/index.json create mode 100644 pkg/v1/layout/testdata/test_index_media_type/oci-layout create mode 100644 pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/381d958b555884ba59574ab5c066e9f6116b5aec3567675aa13bec63331f0810 create mode 100644 pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 create mode 100644 pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/98ceaf93e482fe91b9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46 create mode 100644 pkg/v1/layout/testdata/test_index_one_image/index.json create mode 100644 pkg/v1/layout/testdata/test_index_one_image/oci-layout create mode 100644 pkg/v1/layout/write.go create mode 100644 pkg/v1/layout/write_test.go create mode 100644 pkg/v1/manifest.go create mode 100644 pkg/v1/manifest_test.go create mode 100644 pkg/v1/match/match.go create mode 100644 pkg/v1/match/match_test.go create mode 100644 pkg/v1/mutate/README.md create mode 100644 pkg/v1/mutate/doc.go create mode 100644 pkg/v1/mutate/image.go create mode 100644 pkg/v1/mutate/index.go create mode 100644 pkg/v1/mutate/index_test.go create mode 100644 pkg/v1/mutate/mutate.go create mode 100644 pkg/v1/mutate/mutate_test.go create mode 100644 pkg/v1/mutate/rebase.go create mode 100644 pkg/v1/mutate/rebase_test.go create mode 100644 pkg/v1/mutate/testdata/README.md create mode 100644 pkg/v1/mutate/testdata/bar create mode 100644 pkg/v1/mutate/testdata/foo create mode 100755 pkg/v1/mutate/testdata/overwritten_file.tar create mode 100755 pkg/v1/mutate/testdata/source_image.tar create mode 100755 pkg/v1/mutate/testdata/source_image_with_empty_layer_history.tar create mode 100644 pkg/v1/mutate/testdata/whiteout/bar.txt create mode 100644 pkg/v1/mutate/testdata/whiteout/foo.txt create mode 100755 pkg/v1/mutate/testdata/whiteout_image.tar create mode 100644 pkg/v1/mutate/whiteout_test.go create mode 100644 pkg/v1/partial/README.md create mode 100644 pkg/v1/partial/compressed.go create mode 100644 pkg/v1/partial/compressed_test.go create mode 100644 pkg/v1/partial/configlayer_test.go create mode 100644 pkg/v1/partial/doc.go create mode 100644 pkg/v1/partial/image.go create mode 100644 pkg/v1/partial/index.go create mode 100644 pkg/v1/partial/index_test.go create mode 100644 pkg/v1/partial/uncompressed.go create mode 100644 pkg/v1/partial/uncompressed_test.go create mode 100644 pkg/v1/partial/with.go create mode 100644 pkg/v1/partial/with_test.go create mode 100644 pkg/v1/platform.go create mode 100644 pkg/v1/platform_test.go create mode 100644 pkg/v1/progress.go create mode 100644 pkg/v1/random/doc.go create mode 100644 pkg/v1/random/image.go create mode 100644 pkg/v1/random/image_test.go create mode 100644 pkg/v1/random/index.go create mode 100644 pkg/v1/random/index_test.go create mode 100644 pkg/v1/remote/README.md create mode 100644 pkg/v1/remote/catalog.go create mode 100644 pkg/v1/remote/catalog_test.go create mode 100644 pkg/v1/remote/check.go create mode 100644 pkg/v1/remote/check_e2e_test.go create mode 100644 pkg/v1/remote/check_test.go create mode 100644 pkg/v1/remote/delete.go create mode 100644 pkg/v1/remote/delete_test.go create mode 100644 pkg/v1/remote/descriptor.go create mode 100644 pkg/v1/remote/descriptor_test.go create mode 100644 pkg/v1/remote/doc.go create mode 100644 pkg/v1/remote/error_roundtrip_test.go create mode 100644 pkg/v1/remote/image.go create mode 100644 pkg/v1/remote/image_test.go create mode 100644 pkg/v1/remote/index.go create mode 100644 pkg/v1/remote/index_test.go create mode 100644 pkg/v1/remote/layer.go create mode 100644 pkg/v1/remote/layer_test.go create mode 100644 pkg/v1/remote/list.go create mode 100644 pkg/v1/remote/list_test.go create mode 100644 pkg/v1/remote/mount.go create mode 100644 pkg/v1/remote/mount_test.go create mode 100644 pkg/v1/remote/multi_write.go create mode 100644 pkg/v1/remote/multi_write_test.go create mode 100644 pkg/v1/remote/options.go create mode 100644 pkg/v1/remote/progress.go create mode 100644 pkg/v1/remote/progress_test.go create mode 100644 pkg/v1/remote/referrers.go create mode 100644 pkg/v1/remote/referrers_test.go create mode 100644 pkg/v1/remote/transport/README.md create mode 100644 pkg/v1/remote/transport/basic.go create mode 100644 pkg/v1/remote/transport/basic_test.go create mode 100644 pkg/v1/remote/transport/bearer.go create mode 100644 pkg/v1/remote/transport/bearer_test.go create mode 100644 pkg/v1/remote/transport/doc.go create mode 100644 pkg/v1/remote/transport/error.go create mode 100644 pkg/v1/remote/transport/error_test.go create mode 100644 pkg/v1/remote/transport/logger.go create mode 100644 pkg/v1/remote/transport/logger_test.go create mode 100644 pkg/v1/remote/transport/ping.go create mode 100644 pkg/v1/remote/transport/ping_test.go create mode 100644 pkg/v1/remote/transport/retry.go create mode 100644 pkg/v1/remote/transport/retry_test.go create mode 100644 pkg/v1/remote/transport/schemer.go create mode 100644 pkg/v1/remote/transport/scope.go create mode 100644 pkg/v1/remote/transport/transport.go create mode 100644 pkg/v1/remote/transport/transport_test.go create mode 100644 pkg/v1/remote/transport/useragent.go create mode 100644 pkg/v1/remote/write.go create mode 100644 pkg/v1/remote/write_test.go create mode 100644 pkg/v1/static/layer.go create mode 100644 pkg/v1/static/static_test.go create mode 100644 pkg/v1/stream/README.md create mode 100644 pkg/v1/stream/layer.go create mode 100644 pkg/v1/stream/layer_test.go create mode 100644 pkg/v1/tarball/README.md create mode 100644 pkg/v1/tarball/doc.go create mode 100644 pkg/v1/tarball/image.go create mode 100644 pkg/v1/tarball/image_test.go create mode 100644 pkg/v1/tarball/layer.go create mode 100644 pkg/v1/tarball/layer_test.go create mode 100644 pkg/v1/tarball/progress_test.go create mode 100644 pkg/v1/tarball/testdata/bar create mode 100644 pkg/v1/tarball/testdata/bat/bat create mode 100644 pkg/v1/tarball/testdata/baz create mode 100755 pkg/v1/tarball/testdata/content.tar create mode 100644 pkg/v1/tarball/testdata/foo create mode 100644 pkg/v1/tarball/testdata/no_manifest.tar create mode 100644 pkg/v1/tarball/testdata/null_manifest.tar create mode 100755 pkg/v1/tarball/testdata/test_bundle.tar create mode 100755 pkg/v1/tarball/testdata/test_image_1.tar create mode 100755 pkg/v1/tarball/testdata/test_image_2.tar create mode 100644 pkg/v1/tarball/testdata/test_link.tar create mode 100644 pkg/v1/tarball/testdata/test_load_manifest.tar create mode 100644 pkg/v1/tarball/write.go create mode 100644 pkg/v1/tarball/write_test.go create mode 100644 pkg/v1/types/types.go create mode 100644 pkg/v1/types/types_test.go create mode 100644 pkg/v1/validate/doc.go create mode 100644 pkg/v1/validate/image.go create mode 100644 pkg/v1/validate/index.go create mode 100644 pkg/v1/validate/layer.go create mode 100644 pkg/v1/validate/options.go create mode 100644 pkg/v1/zz_deepcopy_generated.go diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 0000000..68c99ae --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,2 @@ +ignore: + - "**/zz_*_generated.go" # Ignore generated files. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e4fb951 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# This file is documented at https://git-scm.com/docs/gitattributes. +# Linguist-specific attributes are documented at +# https://github.com/github/linguist. + +**/zz_deepcopy_generated.go linguist-generated=true +cmd/crane/doc/crane*.md linguist-generated=true +go.sum linguist-generated=true diff --git a/.github/ISSUE_TEMPLATE/crane_bug_report.md b/.github/ISSUE_TEMPLATE/crane_bug_report.md new file mode 100644 index 0000000..fb14c38 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crane_bug_report.md @@ -0,0 +1,25 @@ +--- +name: crane bug report +about: Create a report to help us improve the crane or gcrane CLIs +title: 'crane:' +labels: bug +assignees: '' + +--- + +### Describe the bug + +A clear and concise description of what the bug is. + +### To Reproduce + +### Expected behavior + +A clear and concise description of what you expected to happen. + +### Additional context + +Add any other context about the problem here. + +- Output of `crane version` +- Registry used (e.g., GCR, ECR, Quay) diff --git a/.github/ISSUE_TEMPLATE/ggcr_bug_report.md b/.github/ISSUE_TEMPLATE/ggcr_bug_report.md new file mode 100644 index 0000000..790d97a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/ggcr_bug_report.md @@ -0,0 +1,25 @@ +--- +name: Go library bug report +about: Create a report to help us improve the Go library +title: 'ggcr:' +labels: bug +assignees: '' + +--- + +### Describe the bug + +A clear and concise description of what the bug is. + +### To Reproduce + +### Expected behavior + +A clear and concise description of what you expected to happen. + +### Additional context + +Add any other context about the problem here. + +- Version of the module +- Registry used (e.g., GCR, ECR, Quay) diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..ff4f551 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,9 @@ +--- +name: Question +about: Ask a question about the project +title: 'question:' +labels: question +assignees: '' + +--- + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e2347a8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/analyze.yaml b/.github/workflows/analyze.yaml new file mode 100644 index 0000000..5982e60 --- /dev/null +++ b/.github/workflows/analyze.yaml @@ -0,0 +1,25 @@ +name: Analyze + +on: + workflow_dispatch: + push: + branches: ['main'] + pull_request: + branches: ['main'] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + - uses: github/codeql-action/init@v2 + with: + languages: go + - uses: github/codeql-action/autobuild@v2 + - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/boilerplate.yaml b/.github/workflows/boilerplate.yaml new file mode 100644 index 0000000..3782e51 --- /dev/null +++ b/.github/workflows/boilerplate.yaml @@ -0,0 +1,33 @@ +name: Boilerplate + +on: + pull_request: + branches: ['main'] + +jobs: + + check: + name: Boilerplate Check + runs-on: ubuntu-latest + strategy: + fail-fast: false # Keep running if one leg fails. + matrix: + extension: + - go + - sh + + # Map between extension and human-readable name. + include: + - extension: go + language: Go + - extension: sh + language: Bash + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - uses: chainguard-dev/actions/boilerplate@5e21cb47971231c078a677dfe89a348371cb880c # main + with: + extension: ${{ matrix.extension }} + language: ${{ matrix.language }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..b3ba675 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,26 @@ +name: Build + +on: + pull_request: + branches: ['main'] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + + strategy: + matrix: + go-version: [1.19, '1.20'] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + + - run: | + go build ./... + go test -run=^$ ./... diff --git a/.github/workflows/bump-deps.yaml b/.github/workflows/bump-deps.yaml new file mode 100644 index 0000000..4712489 --- /dev/null +++ b/.github/workflows/bump-deps.yaml @@ -0,0 +1,35 @@ +name: Bump Deps + +on: + schedule: + - cron: '0 6 * * 2' # weekly at 6AM Tuesday + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + bump-deps: + name: Bump Deps + + # Don't bother bumping deps on forks. + if: ${{ github.repository == 'google/go-containerregistry' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.19 + check-latest: true + + - run: ./hack/bump-deps.sh + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + title: "Bump dependencies using hack/bump-deps.sh" + commit-message: "Bump dependencies using hack/bump-deps.sh" + labels: dependencies + assignees: imjasonh + delete-branch: true diff --git a/.github/workflows/donotsubmit.yaml b/.github/workflows/donotsubmit.yaml new file mode 100644 index 0000000..92d454b --- /dev/null +++ b/.github/workflows/donotsubmit.yaml @@ -0,0 +1,15 @@ +name: Do Not Submit + +on: + pull_request: + branches: ['main'] + +jobs: + + donotsubmit: + name: Do Not Submit + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: chainguard-dev/actions/donotsubmit@5e21cb47971231c078a677dfe89a348371cb880c # main diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..0991bf7 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,94 @@ +name: Basic e2e test + +on: + pull_request: + branches: ['main'] + +jobs: + e2e: + strategy: + fail-fast: false + matrix: + platform: + - ubuntu-latest + - windows-latest + name: e2e ${{ matrix.platform }} + runs-on: ${{ matrix.platform }} + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.19 + check-latest: true + + - name: crane append to an image, set the entrypoint, run it locally, roundtrip it + shell: bash + run: | + set -euxo pipefail + + # Setup local registry + go run ./cmd/registry & + + base=alpine + platform=linux/amd64 + if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then + base=mcr.microsoft.com/windows/nanoserver:ltsc2022 + platform=windows/amd64 + fi + + CGO_ENABLED=0 go build -o app/crane ./cmd/crane + tar cvf crane.tar app + + # This prevents Bash for Windows from mangling path names. + # It shouldn't be necessary in general unless you're using Bash for + # Windows. + export MSYS_NO_PATHCONV=1 + + img=$(./app/crane mutate \ + --entrypoint=/app/crane,version \ + $(./app/crane append \ + --platform ${platform} \ + --base ${base} \ + --new_tag localhost:1338/append-test \ + --new_layer crane.tar)) + + # Run the image with and without args. + docker run $img + docker run $img --help + + # Make sure we can roundtrip it through pull/push + layout=$(mktemp -d) + dst=localhost:1338/roundtrip-test + + ./app/crane pull --format=oci $img $layout + ./app/crane push --image-refs=foo.images $layout $dst + diff <(./app/crane manifest $img) <(./app/crane manifest $(cat foo.images)) + + # Make sure we can roundtrip an index (distroless). + distroless=$(mktemp -d) + remote="gcr.io/distroless/static" + local="localhost:1338/distroless:static" + + ./app/crane pull --format=oci $remote $distroless + ./app/crane push $distroless $local + diff <(./app/crane manifest $remote) <(./app/crane manifest $local) + + # And that it works for a single platform (pulling from what we just pushed). + distroless=$(mktemp -d) + remote="$local" + local="localhost:1338/distroless/platform:static" + + ./app/crane pull --platform=linux/arm64 --format=oci $remote $distroless + ./app/crane push $distroless $local + diff <(./app/crane manifest --platform linux/arm64 $remote) <(./app/crane manifest $local) + + - name: crane pull image, and export it from stdin to filesystem tar to stdout + shell: bash + run: | + set -euxo pipefail + + ./app/crane pull ubuntu ubuntu.tar + ./app/crane export - - < ubuntu.tar > filesystem.tar + ls -la *.tar + diff --git a/.github/workflows/ecr-auth.yaml b/.github/workflows/ecr-auth.yaml new file mode 100644 index 0000000..47cfe29 --- /dev/null +++ b/.github/workflows/ecr-auth.yaml @@ -0,0 +1,93 @@ +name: ECR Authentication test + +on: + pull_request_target: + branches: [ 'main' ] + +permissions: + # This lets us clone the repo + contents: read + # This lets us mint identity tokens. + id-token: write + +jobs: + krane: + runs-on: ubuntu-latest + env: + AWS_ACCOUNT: 479305788615 + AWS_REGION: us-east-2 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.19 + check-latest: true + + - name: Install krane + working-directory: ./cmd/krane + run: go install . + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT }}:role/federated-ecr-readonly + aws-region: ${{ env.AWS_REGION }} + + - name: Test krane + ECR + run: | + # List the tags + krane ls ${{ env.AWS_ACCOUNT }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/go-containerregistry-test + + - name: Test krane auth get + ECR + shell: bash + run: | + CRED1=$(krane auth get ${{ env.AWS_ACCOUNT }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com) + CRED2=$(krane auth get ${{ env.AWS_ACCOUNT }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com) + if [[ "$CRED1" == "" ]] ; then + exit 1 + fi + if [[ "$CRED1" == "$CRED2" ]] ; then + echo "credentials are cached by infrastructure" + fi + + crane-ecr-login: + runs-on: ubuntu-latest + env: + AWS_ACCOUNT: 479305788615 + AWS_REGION: us-east-2 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.19 + check-latest: true + + - name: Install crane + working-directory: ./cmd/crane + run: go install . + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT }}:role/federated-ecr-readonly + aws-region: ${{ env.AWS_REGION }} + + - run: | + wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login + chmod +x ./docker-credential-ecr-login + mv docker-credential-ecr-login /usr/local/bin + + cat > $HOME/.docker/config.json < to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Testing + +Ensure the following passes: +``` +./hack/presubmit.sh +``` +and commit any resultant changes to `go.mod` and `go.sum`. To update any docs +after client changes, run: + +``` +./hack/update-codegen.sh +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c77c03e --- /dev/null +++ b/README.md @@ -0,0 +1,150 @@ +# go-containerregistry + +[![GitHub Actions Build Status](https://github.com/google/go-containerregistry/workflows/Build/badge.svg)](https://github.com/google/go-containerregistry/actions?query=workflow%3ABuild) +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry?status.svg)](https://godoc.org/github.com/google/go-containerregistry) +[![Code Coverage](https://codecov.io/gh/google/go-containerregistry/branch/main/graph/badge.svg)](https://codecov.io/gh/google/go-containerregistry) + +## Introduction + +This is a golang library for working with container registries. +It's largely based on the [Python library of the same name](https://github.com/google/containerregistry). + +The following diagram shows the main types that this library handles. +![OCI image representation](images/ociimage.jpeg) + +## Philosophy + +The overarching design philosophy of this library is to define interfaces that present an immutable +view of resources (e.g. [`Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1#Image), +[`Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1#Layer), +[`ImageIndex`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1#ImageIndex)), +which can be backed by a variety of medium (e.g. [registry](./pkg/v1/remote/README.md), +[tarball](./pkg/v1/tarball/README.md), [daemon](./pkg/v1/daemon/README.md), ...). + +To complement these immutable views, we support functional mutations that produce new immutable views +of the resulting resource (e.g. [mutate](./pkg/v1/mutate/README.md)). The end goal is to provide a +set of versatile primitives that can compose to do extraordinarily powerful things efficiently and easily. + +Both the resource views and mutations may be lazy, eager, memoizing, etc, and most are optimized +for common paths based on the tooling we have seen in the wild (e.g. writing new images from disk +to the registry as a compressed tarball). + + +### Experiments + +Over time, we will add new functionality under experimental environment variables listed here. + +| Env Var | Value(s) | What is does | +|---------|----------|--------------| +| `GGCR_EXPERIMENT_ESTARGZ` | `"1"` | When enabled this experiment will direct `tarball.LayerFromOpener` to emit [estargz](https://github.com/opencontainers/image-spec/issues/815) compatible layers, which enable them to be lazily loaded by an appropriately configured containerd. | + + +### `v1.Image` + +#### Sources + +* [`remote.Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Image) +* [`tarball.Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball#Image) +* [`daemon.Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon#Image) +* [`layout.Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout#Path.Image) +* [`random.Image`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/random#Image) + +#### Sinks + +* [`remote.Write`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Write) +* [`tarball.Write`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball#Write) +* [`daemon.Write`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon#Write) +* [`legacy/tarball.Write`](https://godoc.org/github.com/google/go-containerregistry/pkg/legacy/tarball#Write) +* [`layout.AppendImage`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout#Path.AppendImage) + +### `v1.ImageIndex` + +#### Sources + +* [`remote.Index`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Index) +* [`random.Index`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/random#Index) +* [`layout.ImageIndexFromPath`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout#ImageIndexFromPath) + +#### Sinks + +* [`remote.WriteIndex`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#WriteIndex) +* [`layout.Write`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout#Write) + +### `v1.Layer` + +#### Sources + +* [`remote.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Layer) +* [`tarball.LayerFromFile`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball#LayerFromFile) +* [`random.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/random#Layer) +* [`stream.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream#Layer) + +#### Sinks + +* [`remote.WriteLayer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#WriteLayer) + +## Overview + +### `mutate` + +The simplest use for these libraries is to read from one source and write to another. + +For example, + + * `crane pull` is `remote.Image -> tarball.Write`, + * `crane push` is `tarball.Image -> remote.Write`, + * `crane cp` is `remote.Image -> remote.Write`. + +However, often you actually want to _change something_ about an image. +This is the purpose of the [`mutate`](pkg/v1/mutate) package, which exposes +some commonly useful things to change about an image. + +### `partial` + +If you're trying to use this library with a different source or sink than it already supports, +it can be somewhat cumbersome. The `Image` and `Layer` interfaces are pretty wide, with a lot +of redundant information. This is somewhat by design, because we want to expose this information +as efficiently as possible where we can, but again it is a pain to implement yourself. + +The purpose of the [`partial`](pkg/v1/partial) package is to make implementing a `v1.Image` +much easier, by filling in all the derived accessors for you if you implement a minimal +subset of `v1.Image`. + +### `transport` + +You might think our abstractions are bad and you just want to authenticate +and send requests to a registry. + +This is the purpose of the [`transport`](pkg/v1/remote/transport) and [`authn`](pkg/authn) packages. + +## Tools + +This repo hosts some tools built on top of the library. + +### `crane` + +[`crane`](cmd/crane/README.md) is a tool for interacting with remote images +and registries. + +### `gcrane` + +[`gcrane`](cmd/gcrane/README.md) is a GCR-specific variant of `crane` that has +richer output for the `ls` subcommand and some basic garbage collection support. + +### `krane` + +[`krane`](cmd/krane/README.md) is a drop-in replacement for `crane` that supports +common Kubernetes-based workload identity mechanisms using [`k8schain`](#k8schain) +as a fallback to traditional authentication mechanisms. + +### `k8schain` + +[`k8schain`](pkg/authn/k8schain/README.md) implements the authentication +semantics used by kubelets in a way that is easily consumable by this library. + +`k8schain` is not a standalone tool, but it is linked here for visibility. + +### Emeritus: [`ko`](https://github.com/google/ko) + +This tool was originally developed in this repo but has since been moved to its +own repo. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..ce1f393 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,4 @@ +To report a security issue, please use http://g.co/vulnz. We use +http://g.co/vulnz for our intake, and do coordination and disclosure here on +GitHub (including using GitHub Security Advisory). The Google Security Team will +respond within 5 working days of your report on g.co/vulnz. diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000..b9f2ad7 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,61 @@ +timeout: 3600s # 60 minutes + +steps: +- name: golang + entrypoint: sh + args: + - -c + - | + set -eux + + export GOROOT=/usr/local/go + export KO_DOCKER_REPO="gcr.io/$PROJECT_ID" + export GOFLAGS="-ldflags=-X=github.com/google/go-containerregistry/cmd/crane/cmd.Version=$COMMIT_SHA" + + # Put contents of /workspace on GOPATH. + shadow=$$GOPATH/src/github.com/google/go-containerregistry + link_dir=$$(dirname "$$shadow") + mkdir -p $$link_dir + ln -s $$PWD $$shadow || stat $$shadow + + # Install ko from release. + curl -L -o ko.tar.gz https://github.com/google/ko/releases/download/v0.8.2/ko_0.8.2_Linux_i386.tar.gz + tar xvfz ko.tar.gz + chmod +x ko + alias ko=$${PWD}/ko + + # Use the ko binary to build the crane-ish builder images. + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/crane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/gcrane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + # ./cmd/krane is a separate module, so switch directories. + cd ./cmd/krane + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/krane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + cd ../../ + + # Use the ko binary to build the crane-ish builder *debug* images. + export KO_CONFIG_PATH=$(pwd)/.ko/debug/ + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/crane -t "debug" + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/gcrane -t "debug" + # ./cmd/krane is a separate module, so switch directories. + cd ./cmd/krane + ko publish --platform=all -B github.com/google/go-containerregistry/cmd/krane -t "debug" + cd ../../ + + # Tag-specific debug images are pushed to gcr.io/go-containerregistry/TOOL/debug:... + KO_DOCKER_REPO=gcr.io/$PROJECT_ID/crane/debug ko publish --platform=all --bare github.com/google/go-containerregistry/cmd/crane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + KO_DOCKER_REPO=gcr.io/$PROJECT_ID/gcrane/debug ko publish --platform=all --bare github.com/google/go-containerregistry/cmd/gcrane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + # ./cmd/krane is a separate module, so switch directories. + cd ./cmd/krane + KO_DOCKER_REPO=gcr.io/$PROJECT_ID/krane/debug ko publish --platform=all --bare github.com/google/go-containerregistry/cmd/krane -t latest -t "$COMMIT_SHA" -t "$TAG_NAME" + cd ../../ + +# Use the crane builder to get the digest for crane-ish. +- name: gcr.io/$PROJECT_ID/crane + args: ['digest', 'gcr.io/$PROJECT_ID/crane'] + +- name: gcr.io/$PROJECT_ID/crane + args: ['digest', 'gcr.io/$PROJECT_ID/gcrane'] + +- name: gcr.io/$PROJECT_ID/crane + args: ['digest', 'gcr.io/$PROJECT_ID/krane'] + diff --git a/cmd/crane/README.md b/cmd/crane/README.md new file mode 100644 index 0000000..a05dbeb --- /dev/null +++ b/cmd/crane/README.md @@ -0,0 +1,122 @@ +# `crane` + +[`crane`](doc/crane.md) is a tool for interacting with remote images +and registries. + + + +A collection of useful things you can do with `crane` is [here](recipes.md). + +## Installation + +### Install from Releases + +1. Get the [latest release](https://github.com/google/go-containerregistry/releases/latest) version. + + ```sh + $ VERSION=$(curl -s "https://api.github.com/repos/google/go-containerregistry/releases/latest" | jq -r '.tag_name') + ``` + + or set a specific version: + + ```sh + $ VERSION=vX.Y.Z # Version number with a leading v + ``` + +1. Download the release. + + ```sh + $ OS=Linux # or Darwin, Windows + $ ARCH=x86_64 # or arm64, x86_64, armv6, i386, s390x + $ curl -sL "https://github.com/google/go-containerregistry/releases/download/${VERSION}/go-containerregistry_${OS}_${ARCH}.tar.gz" > go-containerregistry.tar.gz + ``` + +1. Verify the signature. We generate [SLSA 3 provenance](https://slsa.dev) using + the OpenSSF's [slsa-framework/slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + To verify our release, install the verification tool from [slsa-framework/slsa-verifier#installation](https://github.com/slsa-framework/slsa-verifier#installation) + and verify as follows: + + ```sh + $ curl -sL https://github.com/google/go-containerregistry/releases/download/${VERSION}/multiple.intoto.jsonl > provenance.intoto.jsonl + $ # NOTE: You may be using a different architecture. + $ slsa-verifier-linux-amd64 verify-artifact go-containerregistry.tar.gz --provenance-path provenance.intoto.jsonl --source-uri github.com/google/go-containerregistry --source-tag "${VERSION}" + PASSED: Verified SLSA provenance + ``` + +1. Unpack it in the PATH. + + ```sh + $ tar -zxvf go-containerregistry.tar.gz -C /usr/local/bin/ crane + ``` + +### Install manually + +Install manually: + +```sh +go install github.com/google/go-containerregistry/cmd/crane@latest +``` + +### Install via brew + +If you're macOS user and using [Homebrew](https://brew.sh/), you can install via brew command: + +```sh +$ brew install crane +``` + +### Install on Arch Linux + +If you're an Arch Linux user you can install via pacman command: + +```sh +$ pacman -S crane +``` + +### Setup on GitHub Actions + +You can use the [`setup-crane`](https://github.com/imjasonh/setup-crane) action +to install `crane` and setup auth to [GitHub Container +Registry](https://github.com/features/packages) in a GitHub Action workflow: + +``` +steps: +- uses: imjasonh/setup-crane@v0.1 +``` + +## Images + +You can also use crane as docker image + +```sh +$ docker run --rm gcr.io/go-containerregistry/crane ls ubuntu +10.04 +12.04.5 +12.04 +12.10 +``` + +And it's also available with a shell, at the `:debug` tag: + +```sh +docker run --rm -it --entrypoint "/busybox/sh" gcr.io/go-containerregistry/crane:debug +``` + +Tagged debug images are available at `gcr.io/go-containerregistry/crane/debug:[tag]`. + +### Using with GitLab + +```yaml +# Tags an existing Docker image which was tagged with the short commit hash with the tag 'latest' +docker-tag-latest: + stage: latest + only: + refs: + - main + image: + name: gcr.io/go-containerregistry/crane:debug + entrypoint: [""] + script: + - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - crane tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA latest +``` diff --git a/cmd/crane/cmd/append.go b/cmd/crane/cmd/append.go new file mode 100644 index 0000000..9eb2acc --- /dev/null +++ b/cmd/crane/cmd/append.go @@ -0,0 +1,122 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" + specsv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" +) + +// NewCmdAppend creates a new cobra.Command for the append subcommand. +func NewCmdAppend(options *[]crane.Option) *cobra.Command { + var baseRef, newTag, outFile string + var newLayers []string + var annotate, ociEmptyBase bool + + appendCmd := &cobra.Command{ + Use: "append", + Short: "Append contents of a tarball to a remote image", + Long: `This sub-command pushes an image based on an (optional) +base image, with appended layers containing the contents of the +provided tarballs. + +If the base image is a Windows base image (i.e., its config.OS is "windows"), +the contents of the tarballs will be modified to be suitable for a Windows +container image.`, + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, args []string) error { + var base v1.Image + var err error + + if baseRef == "" { + logs.Warn.Printf("base unspecified, using empty image") + base = empty.Image + if ociEmptyBase { + base = mutate.MediaType(base, types.OCIManifestSchema1) + base = mutate.ConfigMediaType(base, types.OCIConfigJSON) + } + } else { + base, err = crane.Pull(baseRef, *options...) + if err != nil { + return fmt.Errorf("pulling %s: %w", baseRef, err) + } + } + + img, err := crane.Append(base, newLayers...) + if err != nil { + return fmt.Errorf("appending %v: %w", newLayers, err) + } + + if baseRef != "" && annotate { + ref, err := name.ParseReference(baseRef) + if err != nil { + return fmt.Errorf("parsing ref %q: %w", baseRef, err) + } + + baseDigest, err := base.Digest() + if err != nil { + return err + } + anns := map[string]string{ + specsv1.AnnotationBaseImageDigest: baseDigest.String(), + } + if _, ok := ref.(name.Tag); ok { + anns[specsv1.AnnotationBaseImageName] = ref.Name() + } + img = mutate.Annotations(img, anns).(v1.Image) + } + + if outFile != "" { + if err := crane.Save(img, newTag, outFile); err != nil { + return fmt.Errorf("writing output %q: %w", outFile, err) + } + } else { + if err := crane.Push(img, newTag, *options...); err != nil { + return fmt.Errorf("pushing image %s: %w", newTag, err) + } + ref, err := name.ParseReference(newTag) + if err != nil { + return fmt.Errorf("parsing reference %s: %w", newTag, err) + } + d, err := img.Digest() + if err != nil { + return fmt.Errorf("digest: %w", err) + } + fmt.Println(ref.Context().Digest(d.String())) + } + return nil + }, + } + appendCmd.Flags().StringVarP(&baseRef, "base", "b", "", "Name of base image to append to") + appendCmd.Flags().StringVarP(&newTag, "new_tag", "t", "", "Tag to apply to resulting image") + appendCmd.Flags().StringSliceVarP(&newLayers, "new_layer", "f", []string{}, "Path to tarball to append to image") + appendCmd.Flags().StringVarP(&outFile, "output", "o", "", "Path to new tarball of resulting image") + appendCmd.Flags().BoolVar(&annotate, "set-base-image-annotations", false, "If true, annotate the resulting image as being based on the base image") + appendCmd.Flags().BoolVar(&ociEmptyBase, "oci-empty-base", false, "If true, empty base image will have OCI media types instead of Docker") + + appendCmd.MarkFlagsMutuallyExclusive("oci-empty-base", "base") + appendCmd.MarkFlagRequired("new_tag") + appendCmd.MarkFlagRequired("new_layer") + return appendCmd +} diff --git a/cmd/crane/cmd/auth.go b/cmd/crane/cmd/auth.go new file mode 100644 index 0000000..1964ade --- /dev/null +++ b/cmd/crane/cmd/auth.go @@ -0,0 +1,205 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "strings" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/types" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/spf13/cobra" +) + +// NewCmdAuth creates a new cobra.Command for the auth subcommand. +func NewCmdAuth(options []crane.Option, argv ...string) *cobra.Command { + cmd := &cobra.Command{ + Use: "auth", + Short: "Log in or access credentials", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Usage() }, + } + cmd.AddCommand(NewCmdAuthGet(options, argv...), NewCmdAuthLogin(argv...)) + return cmd +} + +type credentials struct { + Username string + Secret string +} + +// https://github.com/docker/cli/blob/2291f610ae73533e6e0749d4ef1e360149b1e46b/cli/config/credentials/native_store.go#L100-L109 +func toCreds(config *authn.AuthConfig) credentials { + creds := credentials{ + Username: config.Username, + Secret: config.Password, + } + + if config.IdentityToken != "" { + creds.Username = "" + creds.Secret = config.IdentityToken + } + return creds +} + +// NewCmdAuthGet creates a new `crane auth get` command. +func NewCmdAuthGet(options []crane.Option, argv ...string) *cobra.Command { + if len(argv) == 0 { + argv = []string{os.Args[0]} + } + + baseCmd := strings.Join(argv, " ") + eg := fmt.Sprintf(` # Read configured credentials for reg.example.com + $ echo "reg.example.com" | %s get + {"username":"AzureDiamond","password":"hunter2"} + # or + $ %s get reg.example.com + {"username":"AzureDiamond","password":"hunter2"}`, baseCmd, baseCmd) + + return &cobra.Command{ + Use: "get [REGISTRY_ADDR]", + Short: "Implements a credential helper", + Example: eg, + Args: cobra.MaximumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + registryAddr := "" + if len(args) == 1 { + registryAddr = args[0] + } else { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return err + } + registryAddr = strings.TrimSpace(string(b)) + } + + reg, err := name.NewRegistry(registryAddr) + if err != nil { + return err + } + authorizer, err := crane.GetOptions(options...).Keychain.Resolve(reg) + if err != nil { + return err + } + + // If we don't find any credentials, there's a magic error to return: + // + // https://github.com/docker/docker-credential-helpers/blob/f78081d1f7fef6ad74ad6b79368de6348386e591/credentials/error.go#L4-L6 + // https://github.com/docker/docker-credential-helpers/blob/f78081d1f7fef6ad74ad6b79368de6348386e591/credentials/credentials.go#L61-L63 + if authorizer == authn.Anonymous { + fmt.Fprint(os.Stdout, "credentials not found in native keychain\n") + os.Exit(1) + } + + auth, err := authorizer.Authorization() + if err != nil { + return err + } + + // Convert back to a form that credential helpers can parse so that this + // can act as a meta credential helper. + creds := toCreds(auth) + return json.NewEncoder(os.Stdout).Encode(creds) + }, + } +} + +// NewCmdAuthLogin creates a new `crane auth login` command. +func NewCmdAuthLogin(argv ...string) *cobra.Command { + var opts loginOptions + + if len(argv) == 0 { + argv = []string{os.Args[0]} + } + + eg := fmt.Sprintf(` # Log in to reg.example.com + %s login reg.example.com -u AzureDiamond -p hunter2`, strings.Join(argv, " ")) + + cmd := &cobra.Command{ + Use: "login [OPTIONS] [SERVER]", + Short: "Log in to a registry", + Example: eg, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + reg, err := name.NewRegistry(args[0]) + if err != nil { + return err + } + + opts.serverAddress = reg.Name() + + return login(opts) + }, + } + + flags := cmd.Flags() + + flags.StringVarP(&opts.user, "username", "u", "", "Username") + flags.StringVarP(&opts.password, "password", "p", "", "Password") + flags.BoolVarP(&opts.passwordStdin, "password-stdin", "", false, "Take the password from stdin") + + return cmd +} + +type loginOptions struct { + serverAddress string + user string + password string + passwordStdin bool +} + +func login(opts loginOptions) error { + if opts.passwordStdin { + contents, err := io.ReadAll(os.Stdin) + if err != nil { + return err + } + + opts.password = strings.TrimSuffix(string(contents), "\n") + opts.password = strings.TrimSuffix(opts.password, "\r") + } + if opts.user == "" && opts.password == "" { + return errors.New("username and password required") + } + cf, err := config.Load(os.Getenv("DOCKER_CONFIG")) + if err != nil { + return err + } + creds := cf.GetCredentialsStore(opts.serverAddress) + if opts.serverAddress == name.DefaultRegistry { + opts.serverAddress = authn.DefaultAuthKey + } + if err := creds.Store(types.AuthConfig{ + ServerAddress: opts.serverAddress, + Username: opts.user, + Password: opts.password, + }); err != nil { + return err + } + + if err := cf.Save(); err != nil { + return err + } + log.Printf("logged in via %s", cf.Filename) + return nil +} diff --git a/cmd/crane/cmd/blob.go b/cmd/crane/cmd/blob.go new file mode 100644 index 0000000..fef405f --- /dev/null +++ b/cmd/crane/cmd/blob.go @@ -0,0 +1,48 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "io" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdBlob creates a new cobra.Command for the blob subcommand. +func NewCmdBlob(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "blob BLOB", + Short: "Read a blob from the registry", + Example: "crane blob ubuntu@sha256:4c1d20cdee96111c8acf1858b62655a37ce81ae48648993542b7ac363ac5c0e5 > blob.tar.gz", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + src := args[0] + layer, err := crane.PullLayer(src, *options...) + if err != nil { + return fmt.Errorf("pulling layer %s: %w", src, err) + } + blob, err := layer.Compressed() + if err != nil { + return fmt.Errorf("fetching blob %s: %w", src, err) + } + if _, err := io.Copy(cmd.OutOrStdout(), blob); err != nil { + return fmt.Errorf("copying blob %s: %w", src, err) + } + return nil + }, + } +} diff --git a/cmd/crane/cmd/catalog.go b/cmd/crane/cmd/catalog.go new file mode 100644 index 0000000..0bfa0a9 --- /dev/null +++ b/cmd/crane/cmd/catalog.go @@ -0,0 +1,54 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdCatalog creates a new cobra.Command for the repos subcommand. +func NewCmdCatalog(options *[]crane.Option, argv ...string) *cobra.Command { + if len(argv) == 0 { + argv = []string{os.Args[0]} + } + + baseCmd := strings.Join(argv, " ") + eg := fmt.Sprintf(` # list the repos for reg.example.com + $ %s catalog reg.example.com`, baseCmd) + + return &cobra.Command{ + Use: "catalog [REGISTRY]", + Short: "List the repos in a registry", + Example: eg, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + reg := args[0] + repos, err := crane.Catalog(reg, *options...) + if err != nil { + return fmt.Errorf("reading repos for %s: %w", reg, err) + } + + for _, repo := range repos { + fmt.Println(repo) + } + return nil + }, + } +} diff --git a/cmd/crane/cmd/config.go b/cmd/crane/cmd/config.go new file mode 100644 index 0000000..ed2a3fb --- /dev/null +++ b/cmd/crane/cmd/config.go @@ -0,0 +1,39 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdConfig creates a new cobra.Command for the config subcommand. +func NewCmdConfig(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "config IMAGE", + Short: "Get the config of an image", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + cfg, err := crane.Config(args[0], *options...) + if err != nil { + return fmt.Errorf("fetching config: %w", err) + } + fmt.Print(string(cfg)) + return nil + }, + } +} diff --git a/cmd/crane/cmd/copy.go b/cmd/crane/cmd/copy.go new file mode 100644 index 0000000..81f2e70 --- /dev/null +++ b/cmd/crane/cmd/copy.go @@ -0,0 +1,34 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdCopy creates a new cobra.Command for the copy subcommand. +func NewCmdCopy(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "copy SRC DST", + Aliases: []string{"cp"}, + Short: "Efficiently copy a remote image from src to dst while retaining the digest value", + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + src, dst := args[0], args[1] + return crane.Copy(src, dst, *options...) + }, + } +} diff --git a/cmd/crane/cmd/delete.go b/cmd/crane/cmd/delete.go new file mode 100644 index 0000000..18966ea --- /dev/null +++ b/cmd/crane/cmd/delete.go @@ -0,0 +1,33 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdDelete creates a new cobra.Command for the delete subcommand. +func NewCmdDelete(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "delete IMAGE", + Short: "Delete an image reference from its registry", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + ref := args[0] + return crane.Delete(ref, *options...) + }, + } +} diff --git a/cmd/crane/cmd/digest.go b/cmd/crane/cmd/digest.go new file mode 100644 index 0000000..2060bbc --- /dev/null +++ b/cmd/crane/cmd/digest.go @@ -0,0 +1,91 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/spf13/cobra" +) + +// NewCmdDigest creates a new cobra.Command for the digest subcommand. +func NewCmdDigest(options *[]crane.Option) *cobra.Command { + var tarball string + var fullRef bool + cmd := &cobra.Command{ + Use: "digest IMAGE", + Short: "Get the digest of an image", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if tarball == "" && len(args) == 0 { + if err := cmd.Help(); err != nil { + return err + } + return errors.New("image reference required without --tarball") + } + if fullRef && tarball != "" { + return errors.New("cannot specify --full-ref with --tarball") + } + + digest, err := getDigest(tarball, args, options) + if err != nil { + return err + } + if fullRef { + ref, err := name.ParseReference(args[0]) + if err != nil { + return err + } + fmt.Println(ref.Context().Digest(digest)) + } else { + fmt.Println(digest) + } + return nil + }, + } + + cmd.Flags().StringVar(&tarball, "tarball", "", "(Optional) path to tarball containing the image") + cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference by digest") + + return cmd +} + +func getDigest(tarball string, args []string, options *[]crane.Option) (string, error) { + if tarball != "" { + return getTarballDigest(tarball, args, options) + } + + return crane.Digest(args[0], *options...) +} + +func getTarballDigest(tarball string, args []string, options *[]crane.Option) (string, error) { + tag := "" + if len(args) > 0 { + tag = args[0] + } + + img, err := crane.LoadTag(tarball, tag, *options...) + if err != nil { + return "", fmt.Errorf("loading image from %q: %w", tarball, err) + } + digest, err := img.Digest() + if err != nil { + return "", fmt.Errorf("computing digest: %w", err) + } + return digest.String(), nil +} diff --git a/cmd/crane/cmd/export.go b/cmd/crane/cmd/export.go new file mode 100644 index 0000000..70b58c1 --- /dev/null +++ b/cmd/crane/cmd/export.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/spf13/cobra" +) + +// NewCmdExport creates a new cobra.Command for the export subcommand. +func NewCmdExport(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "export IMAGE|- TARBALL|-", + Short: "Export filesystem of a container image as a tarball", + Example: ` # Write tarball to stdout + crane export ubuntu - + + # Write tarball to file + crane export ubuntu ubuntu.tar + + # Read image from stdin + crane export - ubuntu.tar`, + Args: cobra.RangeArgs(1, 2), + RunE: func(_ *cobra.Command, args []string) error { + src, dst := args[0], "-" + if len(args) > 1 { + dst = args[1] + } + + f, err := openFile(dst) + if err != nil { + return fmt.Errorf("failed to open %s: %w", dst, err) + } + defer f.Close() + + var img v1.Image + if src == "-" { + tmpfile, err := os.CreateTemp("", "crane") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := io.Copy(tmpfile, os.Stdin); err != nil { + log.Fatal(err) + } + tmpfile.Close() + + img, err = tarball.ImageFromPath(tmpfile.Name(), nil) + if err != nil { + return fmt.Errorf("reading tarball from stdin: %w", err) + } + } else { + img, err = crane.Pull(src, *options...) + if err != nil { + return fmt.Errorf("pulling %s: %w", src, err) + } + } + + return crane.Export(img, f) + }, + } +} + +func openFile(s string) (*os.File, error) { + if s == "-" { + return os.Stdout, nil + } + return os.Create(s) +} diff --git a/cmd/crane/cmd/flatten.go b/cmd/crane/cmd/flatten.go new file mode 100644 index 0000000..76c13f1 --- /dev/null +++ b/cmd/crane/cmd/flatten.go @@ -0,0 +1,254 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "log" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/spf13/cobra" +) + +// NewCmdFlatten creates a new cobra.Command for the flatten subcommand. +func NewCmdFlatten(options *[]crane.Option) *cobra.Command { + var dst string + + flattenCmd := &cobra.Command{ + Use: "flatten", + Short: "Flatten an image's layers into a single layer", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + // We need direct access to the underlying remote options because crane + // doesn't expose great facilities for working with an index (yet). + o := crane.GetOptions(*options...) + + // Pull image and get config. + src := args[0] + + // If the new ref isn't provided, write over the original image. + // If that ref was provided by digest (e.g., output from + // another crane command), then strip that and push the + // mutated image by digest instead. + if dst == "" { + dst = src + } + + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + log.Fatalf("parsing %s: %v", src, err) + } + newRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + log.Fatalf("parsing %s: %v", dst, err) + } + repo := newRef.Context() + + flat, err := flatten(ref, repo, cmd.Parent().Use, o) + if err != nil { + log.Fatalf("flattening %s: %v", ref, err) + } + + digest, err := flat.Digest() + if err != nil { + log.Fatalf("digesting new image: %v", err) + } + + if _, ok := ref.(name.Digest); ok { + newRef = repo.Digest(digest.String()) + } + + if err := push(flat, newRef, o); err != nil { + log.Fatalf("pushing %s: %v", newRef, err) + } + fmt.Println(repo.Digest(digest.String())) + }, + } + flattenCmd.Flags().StringVarP(&dst, "tag", "t", "", "New tag to apply to flattened image. If not provided, push by digest to the original image repository.") + return flattenCmd +} + +func flatten(ref name.Reference, repo name.Repository, use string, o crane.Options) (partial.Describable, error) { + desc, err := remote.Get(ref, o.Remote...) + if err != nil { + return nil, fmt.Errorf("pulling %s: %w", ref, err) + } + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return nil, err + } + return flattenIndex(idx, repo, use, o) + } else if desc.MediaType.IsImage() { + img, err := desc.Image() + if err != nil { + return nil, err + } + return flattenImage(img, repo, use, o) + } + + return nil, fmt.Errorf("can't flatten %s", desc.MediaType) +} + +func push(flat partial.Describable, ref name.Reference, o crane.Options) error { + if idx, ok := flat.(v1.ImageIndex); ok { + return remote.WriteIndex(ref, idx, o.Remote...) + } else if img, ok := flat.(v1.Image); ok { + return remote.Write(ref, img, o.Remote...) + } + + return fmt.Errorf("can't push %T", flat) +} + +type remoteIndex interface { + Manifests() ([]partial.Describable, error) +} + +func flattenIndex(old v1.ImageIndex, repo name.Repository, use string, o crane.Options) (partial.Describable, error) { + ri, ok := old.(remoteIndex) + if !ok { + return nil, fmt.Errorf("unexpected index") + } + + m, err := old.IndexManifest() + if err != nil { + return nil, err + } + + manifests, err := ri.Manifests() + if err != nil { + return nil, err + } + + adds := []mutate.IndexAddendum{} + + for _, m := range manifests { + // Keep the old descriptor (annotations and whatnot). + desc, err := partial.Descriptor(m) + if err != nil { + return nil, err + } + + flattened, err := flattenChild(m, repo, use, o) + if err != nil { + return nil, err + } + desc.Size, err = flattened.Size() + if err != nil { + return nil, err + } + desc.Digest, err = flattened.Digest() + if err != nil { + return nil, err + } + adds = append(adds, mutate.IndexAddendum{ + Add: flattened, + Descriptor: *desc, + }) + } + + idx := mutate.AppendManifests(empty.Index, adds...) + + // Retain any annotations from the original index. + if len(m.Annotations) != 0 { + idx = mutate.Annotations(idx, m.Annotations).(v1.ImageIndex) + } + + // This is stupid, but some registries get mad if you try to push OCI media types that reference docker media types. + mt, err := old.MediaType() + if err != nil { + return nil, err + } + idx = mutate.IndexMediaType(idx, mt) + + return idx, nil +} + +func flattenChild(old partial.Describable, repo name.Repository, use string, o crane.Options) (partial.Describable, error) { + if idx, ok := old.(v1.ImageIndex); ok { + return flattenIndex(idx, repo, use, o) + } else if img, ok := old.(v1.Image); ok { + return flattenImage(img, repo, use, o) + } + + logs.Warn.Printf("can't flatten %T, skipping", old) + return old, nil +} + +func flattenImage(old v1.Image, repo name.Repository, use string, o crane.Options) (partial.Describable, error) { + digest, err := old.Digest() + if err != nil { + return nil, fmt.Errorf("getting old digest: %w", err) + } + m, err := old.Manifest() + if err != nil { + return nil, fmt.Errorf("reading manifest: %w", err) + } + + cf, err := old.ConfigFile() + if err != nil { + return nil, fmt.Errorf("getting config: %w", err) + } + cf = cf.DeepCopy() + + oldHistory, err := json.Marshal(cf.History) + if err != nil { + return nil, fmt.Errorf("marshal history") + } + + // Clear layer-specific config file information. + cf.RootFS.DiffIDs = []v1.Hash{} + cf.History = []v1.History{} + + img, err := mutate.ConfigFile(empty.Image, cf) + if err != nil { + return nil, fmt.Errorf("mutating config: %w", err) + } + + // TODO: Make compression configurable? + layer := stream.NewLayer(mutate.Extract(old), stream.WithCompressionLevel(gzip.BestCompression)) + if err := remote.WriteLayer(repo, layer, o.Remote...); err != nil { + return nil, fmt.Errorf("uploading layer: %w", err) + } + + img, err = mutate.Append(img, mutate.Addendum{ + Layer: layer, + History: v1.History{ + CreatedBy: fmt.Sprintf("%s flatten %s", use, digest), + Comment: string(oldHistory), + }, + }) + if err != nil { + return nil, fmt.Errorf("appending layers: %w", err) + } + + // Retain any annotations from the original image. + if len(m.Annotations) != 0 { + img = mutate.Annotations(img, m.Annotations).(v1.Image) + } + + return img, nil +} diff --git a/cmd/crane/cmd/index.go b/cmd/crane/cmd/index.go new file mode 100644 index 0000000..8d4b425 --- /dev/null +++ b/cmd/crane/cmd/index.go @@ -0,0 +1,291 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/spf13/cobra" +) + +// NewCmdIndex creates a new cobra.Command for the index subcommand. +func NewCmdIndex(options *[]crane.Option) *cobra.Command { + cmd := &cobra.Command{ + Use: "index", + Short: "Modify an image index.", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, _ []string) { + cmd.Usage() + }, + } + cmd.AddCommand(NewCmdIndexFilter(options), NewCmdIndexAppend(options)) + return cmd +} + +// NewCmdIndexFilter creates a new cobra.Command for the index filter subcommand. +func NewCmdIndexFilter(options *[]crane.Option) *cobra.Command { + var newTag string + platforms := &platformsValue{} + + cmd := &cobra.Command{ + Use: "filter", + Short: "Modifies a remote index by filtering based on platform.", + Example: ` # Filter out weird platforms from ubuntu, copy result to example.com/ubuntu + crane index filter ubuntu --platform linux/amd64 --platform linux/arm64 -t example.com/ubuntu + + # Filter out any non-linux platforms, push to example.com/hello-world + crane index filter hello-world --platform linux -t example.com/hello-world + + # Same as above, but in-place + crane index filter example.com/hello-world:some-tag --platform linux`, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + o := crane.GetOptions(*options...) + baseRef := args[0] + + ref, err := name.ParseReference(baseRef) + if err != nil { + return err + } + base, err := remote.Index(ref, o.Remote...) + if err != nil { + return fmt.Errorf("pulling %s: %w", baseRef, err) + } + + idx := filterIndex(base, platforms.platforms) + + digest, err := idx.Digest() + if err != nil { + return err + } + + if newTag != "" { + ref, err = name.ParseReference(newTag) + if err != nil { + return fmt.Errorf("parsing reference %s: %w", newTag, err) + } + } else { + if _, ok := ref.(name.Digest); ok { + ref = ref.Context().Digest(digest.String()) + } + } + + if err := remote.WriteIndex(ref, idx, o.Remote...); err != nil { + return fmt.Errorf("pushing image %s: %w", newTag, err) + } + fmt.Println(ref.Context().Digest(digest.String())) + return nil + }, + } + cmd.Flags().StringVarP(&newTag, "tag", "t", "", "Tag to apply to resulting image") + + // Consider reusing the persistent flag for this, it's separate so we can have multiple values. + cmd.Flags().Var(platforms, "platform", "Specifies the platform(s) to keep from base in the form os/arch[/variant][:osversion][,] (e.g. linux/amd64).") + + return cmd +} + +// NewCmdIndexAppend creates a new cobra.Command for the index append subcommand. +func NewCmdIndexAppend(options *[]crane.Option) *cobra.Command { + var baseRef, newTag string + var newManifests []string + var dockerEmptyBase, flatten bool + + cmd := &cobra.Command{ + Use: "append", + Short: "Append manifests to a remote index.", + Long: `This sub-command pushes an index based on an (optional) base index, with appended manifests. + +The platform for appended manifests is inferred from the config file or omitted if that is infeasible.`, + Example: ` # Append a windows hello-world image to ubuntu, push to example.com/hello-world:weird + crane index append ubuntu -m hello-world@sha256:87b9ca29151260634b95efb84d43b05335dc3ed36cc132e2b920dd1955342d20 -t example.com/hello-world:weird + + # Create an index from scratch for etcd. + crane index append -m registry.k8s.io/etcd-amd64:3.4.9 -m registry.k8s.io/etcd-arm64:3.4.9 -t example.com/etcd`, + Args: cobra.MaximumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + if len(args) == 1 { + baseRef = args[0] + } + o := crane.GetOptions(*options...) + + var ( + base v1.ImageIndex + err error + ref name.Reference + ) + + if baseRef == "" { + if newTag == "" { + return errors.New("at least one of --base or --tag must be specified") + } + + logs.Warn.Printf("base unspecified, using empty index") + base = empty.Index + if dockerEmptyBase { + base = mutate.IndexMediaType(base, types.DockerManifestList) + } + } else { + ref, err = name.ParseReference(baseRef) + if err != nil { + return err + } + base, err = remote.Index(ref, o.Remote...) + if err != nil { + return fmt.Errorf("pulling %s: %w", baseRef, err) + } + } + + adds := make([]mutate.IndexAddendum, 0, len(newManifests)) + + for _, m := range newManifests { + ref, err := name.ParseReference(m) + if err != nil { + return err + } + desc, err := remote.Get(ref, o.Remote...) + if err != nil { + return err + } + if desc.MediaType.IsImage() { + img, err := desc.Image() + if err != nil { + return err + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + newDesc, err := partial.Descriptor(img) + if err != nil { + return err + } + newDesc.Platform = cf.Platform() + adds = append(adds, mutate.IndexAddendum{ + Add: img, + Descriptor: *newDesc, + }) + } else if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return err + } + if flatten { + im, err := idx.IndexManifest() + if err != nil { + return err + } + for _, child := range im.Manifests { + switch { + case child.MediaType.IsImage(): + img, err := idx.Image(child.Digest) + if err != nil { + return err + } + adds = append(adds, mutate.IndexAddendum{ + Add: img, + Descriptor: child, + }) + case child.MediaType.IsIndex(): + idx, err := idx.ImageIndex(child.Digest) + if err != nil { + return err + } + adds = append(adds, mutate.IndexAddendum{ + Add: idx, + Descriptor: child, + }) + default: + return fmt.Errorf("unexpected child %q with media type %q", child.Digest, child.MediaType) + } + } + } else { + adds = append(adds, mutate.IndexAddendum{ + Add: idx, + }) + } + } else { + return fmt.Errorf("saw unexpected MediaType %q for %q", desc.MediaType, m) + } + } + + idx := mutate.AppendManifests(base, adds...) + digest, err := idx.Digest() + if err != nil { + return err + } + + if newTag != "" { + ref, err = name.ParseReference(newTag) + if err != nil { + return fmt.Errorf("parsing reference %s: %w", newTag, err) + } + } else { + if _, ok := ref.(name.Digest); ok { + ref = ref.Context().Digest(digest.String()) + } + } + + if err := remote.WriteIndex(ref, idx, o.Remote...); err != nil { + return fmt.Errorf("pushing image %s: %w", newTag, err) + } + fmt.Println(ref.Context().Digest(digest.String())) + return nil + }, + } + cmd.Flags().StringVarP(&newTag, "tag", "t", "", "Tag to apply to resulting image") + cmd.Flags().StringSliceVarP(&newManifests, "manifest", "m", []string{}, "References to manifests to append to the base index") + cmd.Flags().BoolVar(&dockerEmptyBase, "docker-empty-base", false, "If true, empty base index will have Docker media types instead of OCI") + cmd.Flags().BoolVar(&flatten, "flatten", true, "If true, appending an index will append each of its children rather than the index itself") + + return cmd +} + +func filterIndex(idx v1.ImageIndex, platforms []v1.Platform) v1.ImageIndex { + matcher := not(satisfiesPlatforms(platforms)) + return mutate.RemoveManifests(idx, matcher) +} + +func satisfiesPlatforms(platforms []v1.Platform) match.Matcher { + return func(desc v1.Descriptor) bool { + if desc.Platform == nil { + return false + } + for _, p := range platforms { + if desc.Platform.Satisfies(p) { + return true + } + } + return false + } +} + +func not(in match.Matcher) match.Matcher { + return func(desc v1.Descriptor) bool { + return !in(desc) + } +} diff --git a/cmd/crane/cmd/list.go b/cmd/crane/cmd/list.go new file mode 100644 index 0000000..3902ccd --- /dev/null +++ b/cmd/crane/cmd/list.go @@ -0,0 +1,62 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/spf13/cobra" +) + +// NewCmdList creates a new cobra.Command for the ls subcommand. +func NewCmdList(options *[]crane.Option) *cobra.Command { + var fullRef, omitDigestTags bool + cmd := &cobra.Command{ + Use: "ls REPO", + Short: "List the tags in a repo", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + repo := args[0] + tags, err := crane.ListTags(repo, *options...) + if err != nil { + return fmt.Errorf("reading tags for %s: %w", repo, err) + } + + r, err := name.NewRepository(repo) + if err != nil { + return err + } + + for _, tag := range tags { + if omitDigestTags && strings.HasPrefix(tag, "sha256-") { + continue + } + + if fullRef { + fmt.Println(r.Tag(tag)) + } else { + fmt.Println(tag) + } + } + return nil + }, + } + cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference") + cmd.Flags().BoolVar(&omitDigestTags, "omit-digest-tags", false, "(Optional), if true, omit digest tags (e.g., ':sha256-...')") + return cmd +} diff --git a/cmd/crane/cmd/manifest.go b/cmd/crane/cmd/manifest.go new file mode 100644 index 0000000..d9ef7fd --- /dev/null +++ b/cmd/crane/cmd/manifest.go @@ -0,0 +1,40 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdManifest creates a new cobra.Command for the manifest subcommand. +func NewCmdManifest(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "manifest IMAGE", + Short: "Get the manifest of an image", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + src := args[0] + manifest, err := crane.Manifest(src, *options...) + if err != nil { + return fmt.Errorf("fetching manifest %s: %w", src, err) + } + fmt.Print(string(manifest)) + return nil + }, + } +} diff --git a/cmd/crane/cmd/mutate.go b/cmd/crane/cmd/mutate.go new file mode 100644 index 0000000..a99def0 --- /dev/null +++ b/cmd/crane/cmd/mutate.go @@ -0,0 +1,207 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/spf13/cobra" +) + +// NewCmdMutate creates a new cobra.Command for the mutate subcommand. +func NewCmdMutate(options *[]crane.Option) *cobra.Command { + var labels map[string]string + var annotations map[string]string + var entrypoint, cmd []string + var envVars map[string]string + var newLayers []string + var outFile string + var newRef string + var newRepo string + var user string + + mutateCmd := &cobra.Command{ + Use: "mutate", + Short: "Modify image labels and annotations. The container must be pushed to a registry, and the manifest is updated there.", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + // Pull image and get config. + ref := args[0] + + if len(annotations) != 0 { + desc, err := crane.Head(ref, *options...) + if err != nil { + return err + } + if desc.MediaType.IsIndex() { + return errors.New("mutating annotations on an index is not yet supported") + } + } + + if newRepo != "" && newRef != "" { + return errors.New("repository can't be set when a tag is specified") + } + + img, err := crane.Pull(ref, *options...) + if err != nil { + return fmt.Errorf("pulling %s: %w", ref, err) + } + if len(newLayers) != 0 { + img, err = crane.Append(img, newLayers...) + if err != nil { + return fmt.Errorf("appending %v: %w", newLayers, err) + } + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + cfg = cfg.DeepCopy() + + // Set labels. + if cfg.Config.Labels == nil { + cfg.Config.Labels = map[string]string{} + } + + if err := validateKeyVals(labels); err != nil { + return err + } + + for k, v := range labels { + cfg.Config.Labels[k] = v + } + + if err := validateKeyVals(annotations); err != nil { + return err + } + + // set envvars if specified + if err := setEnvVars(cfg, envVars); err != nil { + return err + } + + // Set entrypoint. + if len(entrypoint) > 0 { + cfg.Config.Entrypoint = entrypoint + cfg.Config.Cmd = nil // This matches Docker's behavior. + } + + // Set cmd. + if len(cmd) > 0 { + cfg.Config.Cmd = cmd + } + + // Set user. + if len(user) > 0 { + cfg.Config.User = user + } + + // Mutate and write image. + img, err = mutate.Config(img, cfg.Config) + if err != nil { + return fmt.Errorf("mutating config: %w", err) + } + + img = mutate.Annotations(img, annotations).(v1.Image) + + // If the new ref isn't provided, write over the original image. + // If that ref was provided by digest (e.g., output from + // another crane command), then strip that and push the + // mutated image by digest instead. + if newRepo != "" { + newRef = newRepo + } else if newRef == "" { + newRef = ref + } + digest, err := img.Digest() + if err != nil { + return fmt.Errorf("digesting new image: %w", err) + } + if outFile != "" { + if err := crane.Save(img, newRef, outFile); err != nil { + return fmt.Errorf("writing output %q: %w", outFile, err) + } + } else { + r, err := name.ParseReference(newRef) + if err != nil { + return fmt.Errorf("parsing %s: %w", newRef, err) + } + if _, ok := r.(name.Digest); ok || newRepo != "" { + newRef = r.Context().Digest(digest.String()).String() + } + if err := crane.Push(img, newRef, *options...); err != nil { + return fmt.Errorf("pushing %s: %w", newRef, err) + } + fmt.Println(r.Context().Digest(digest.String())) + } + return nil + }, + } + mutateCmd.Flags().StringToStringVarP(&annotations, "annotation", "a", nil, "New annotations to add") + mutateCmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "New labels to add") + mutateCmd.Flags().StringToStringVarP(&envVars, "env", "e", nil, "New envvar to add") + mutateCmd.Flags().StringSliceVar(&entrypoint, "entrypoint", nil, "New entrypoint to set") + mutateCmd.Flags().StringSliceVar(&cmd, "cmd", nil, "New cmd to set") + mutateCmd.Flags().StringVar(&newRepo, "repo", "", "Repository to push the mutated image to. If provided, push by digest to this repository.") + mutateCmd.Flags().StringVarP(&newRef, "tag", "t", "", "New tag reference to apply to mutated image. If not provided, push by digest to the original image repository.") + mutateCmd.Flags().StringVarP(&outFile, "output", "o", "", "Path to new tarball of resulting image") + mutateCmd.Flags().StringSliceVar(&newLayers, "append", []string{}, "Path to tarball to append to image") + mutateCmd.Flags().StringVarP(&user, "user", "u", "", "New user to set") + return mutateCmd +} + +// validateKeyVals ensures no values are empty, returns error if they are +func validateKeyVals(kvPairs map[string]string) error { + for label, value := range kvPairs { + if value == "" { + return fmt.Errorf("parsing label %q, value is empty", label) + } + } + return nil +} + +// setEnvVars override envvars in a config +func setEnvVars(cfg *v1.ConfigFile, envVars map[string]string) error { + newEnv := make([]string, 0, len(cfg.Config.Env)) + for _, old := range cfg.Config.Env { + split := strings.SplitN(old, "=", 2) + if len(split) != 2 { + return fmt.Errorf("invalid key value pair in config: %s", old) + } + // keep order so override if specified again + oldKey := split[0] + if v, ok := envVars[oldKey]; ok { + newEnv = append(newEnv, fmt.Sprintf("%s=%s", oldKey, v)) + delete(envVars, oldKey) + } else { + newEnv = append(newEnv, old) + } + } + isWindows := cfg.OS == "windows" + for k, v := range envVars { + if isWindows { + k = strings.ToUpper(k) + } + newEnv = append(newEnv, fmt.Sprintf("%s=%s", k, v)) + } + cfg.Config.Env = newEnv + return nil +} diff --git a/cmd/crane/cmd/optimize.go b/cmd/crane/cmd/optimize.go new file mode 100644 index 0000000..89c4706 --- /dev/null +++ b/cmd/crane/cmd/optimize.go @@ -0,0 +1,42 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdOptimize creates a new cobra.Command for the optimize subcommand. +func NewCmdOptimize(options *[]crane.Option) *cobra.Command { + var files []string + + cmd := &cobra.Command{ + Use: "optimize SRC DST", + Hidden: true, + Aliases: []string{"opt"}, + Short: "Optimize a remote container image from src to dst", + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + src, dst := args[0], args[1] + return crane.Optimize(src, dst, files, *options...) + }, + } + + cmd.Flags().StringSliceVar(&files, "prioritize", nil, + "The list of files to prioritize in the optimized image.") + + return cmd +} diff --git a/cmd/crane/cmd/pull.go b/cmd/crane/cmd/pull.go new file mode 100644 index 0000000..41c6e95 --- /dev/null +++ b/cmd/crane/cmd/pull.go @@ -0,0 +1,138 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/cache" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" +) + +// NewCmdPull creates a new cobra.Command for the pull subcommand. +func NewCmdPull(options *[]crane.Option) *cobra.Command { + var ( + cachePath, format string + annotateRef bool + ) + + cmd := &cobra.Command{ + Use: "pull IMAGE TARBALL", + Short: "Pull remote images by reference and store their contents locally", + Args: cobra.MinimumNArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + imageMap := map[string]v1.Image{} + indexMap := map[string]v1.ImageIndex{} + srcList, path := args[:len(args)-1], args[len(args)-1] + o := crane.GetOptions(*options...) + for _, src := range srcList { + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", src, err) + } + + rmt, err := remote.Get(ref, o.Remote...) + if err != nil { + return err + } + + // If we're writing an index to a layout and --platform hasn't been set, + // pull the entire index, not just a child image. + if format == "oci" && rmt.MediaType.IsIndex() && o.Platform == nil { + idx, err := rmt.ImageIndex() + if err != nil { + return err + } + indexMap[src] = idx + continue + } + + img, err := rmt.Image() + if err != nil { + return err + } + if cachePath != "" { + img = cache.Image(img, cache.NewFilesystemCache(cachePath)) + } + imageMap[src] = img + } + + switch format { + case "tarball": + if err := crane.MultiSave(imageMap, path); err != nil { + return fmt.Errorf("saving tarball %s: %w", path, err) + } + case "legacy": + if err := crane.MultiSaveLegacy(imageMap, path); err != nil { + return fmt.Errorf("saving legacy tarball %s: %w", path, err) + } + case "oci": + // Don't use crane.MultiSaveOCI so we can control annotations. + p, err := layout.FromPath(path) + if err != nil { + p, err = layout.Write(path, empty.Index) + if err != nil { + return err + } + } + for ref, img := range imageMap { + opts := []layout.Option{} + if annotateRef { + parsed, err := name.ParseReference(ref, o.Name...) + if err != nil { + return err + } + opts = append(opts, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": parsed.Name(), + })) + } + if err = p.AppendImage(img, opts...); err != nil { + return err + } + } + + for ref, idx := range indexMap { + opts := []layout.Option{} + if annotateRef { + parsed, err := name.ParseReference(ref, o.Name...) + if err != nil { + return err + } + opts = append(opts, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": parsed.Name(), + })) + } + if err := p.AppendIndex(idx, opts...); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected --format: %q (valid values are: tarball, legacy, and oci)", format) + } + return nil + }, + } + cmd.Flags().StringVarP(&cachePath, "cache_path", "c", "", "Path to cache image layers") + cmd.Flags().StringVar(&format, "format", "tarball", fmt.Sprintf("Format in which to save images (%q, %q, or %q)", "tarball", "legacy", "oci")) + cmd.Flags().BoolVar(&annotateRef, "annotate-ref", false, "Preserves image reference used to pull as an annotation when used with --format=oci") + + return cmd +} diff --git a/cmd/crane/cmd/push.go b/cmd/crane/cmd/push.go new file mode 100644 index 0000000..887621e --- /dev/null +++ b/cmd/crane/cmd/push.go @@ -0,0 +1,126 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" +) + +// NewCmdPush creates a new cobra.Command for the push subcommand. +func NewCmdPush(options *[]crane.Option) *cobra.Command { + index := false + imageRefs := "" + cmd := &cobra.Command{ + Use: "push PATH IMAGE", + Short: "Push local image contents to a remote registry", + Long: `If the PATH is a directory, it will be read as an OCI image layout. Otherwise, PATH is assumed to be a docker-style tarball.`, + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + path, tag := args[0], args[1] + + img, err := loadImage(path, index) + if err != nil { + return err + } + + o := crane.GetOptions(*options...) + ref, err := name.ParseReference(tag, o.Name...) + if err != nil { + return err + } + var h v1.Hash + switch t := img.(type) { + case v1.Image: + if err := remote.Write(ref, t, o.Remote...); err != nil { + return err + } + if h, err = t.Digest(); err != nil { + return err + } + case v1.ImageIndex: + if err := remote.WriteIndex(ref, t, o.Remote...); err != nil { + return err + } + if h, err = t.Digest(); err != nil { + return err + } + default: + return fmt.Errorf("cannot push type (%T) to registry", img) + } + + digest := ref.Context().Digest(h.String()) + if imageRefs != "" { + return os.WriteFile(imageRefs, []byte(digest.String()), 0600) + } + // TODO(mattmoor): think about printing the digest to standard out + // to facilitate command composition similar to ko build. + + return nil + }, + } + cmd.Flags().BoolVar(&index, "index", false, "push a collection of images as a single index, currently required if PATH contains multiple images") + cmd.Flags().StringVar(&imageRefs, "image-refs", "", "path to file where a list of the published image references will be written") + return cmd +} + +func loadImage(path string, index bool) (partial.WithRawManifest, error) { + stat, err := os.Stat(path) + if err != nil { + return nil, err + } + + if !stat.IsDir() { + img, err := crane.Load(path) + if err != nil { + return nil, fmt.Errorf("loading %s as tarball: %w", path, err) + } + return img, nil + } + + l, err := layout.ImageIndexFromPath(path) + if err != nil { + return nil, fmt.Errorf("loading %s as OCI layout: %w", path, err) + } + + if index { + return l, nil + } + + m, err := l.IndexManifest() + if err != nil { + return nil, err + } + if len(m.Manifests) != 1 { + return nil, fmt.Errorf("layout contains %d entries, consider --index", len(m.Manifests)) + } + + desc := m.Manifests[0] + if desc.MediaType.IsImage() { + return l.Image(desc.Digest) + } else if desc.MediaType.IsIndex() { + return l.ImageIndex(desc.Digest) + } + + return nil, fmt.Errorf("layout contains non-image (mediaType: %q), consider --index", desc.MediaType) +} diff --git a/cmd/crane/cmd/rebase.go b/cmd/crane/cmd/rebase.go new file mode 100644 index 0000000..43f21b5 --- /dev/null +++ b/cmd/crane/cmd/rebase.go @@ -0,0 +1,210 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "fmt" + "log" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + specsv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" +) + +// NewCmdRebase creates a new cobra.Command for the rebase subcommand. +func NewCmdRebase(options *[]crane.Option) *cobra.Command { + var orig, oldBase, newBase, rebased string + + rebaseCmd := &cobra.Command{ + Use: "rebase", + Short: "Rebase an image onto a new base image", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if orig == "" { + orig = args[0] + } else if len(args) != 0 || args[0] != "" { + return fmt.Errorf("cannot use --original with positional argument") + } + + // If the new ref isn't provided, write over the original image. + // If that ref was provided by digest (e.g., output from + // another crane command), then strip that and push the + // mutated image by digest instead. + if rebased == "" { + rebased = orig + } + + // Stupid hack to support insecure flag. + nameOpt := []name.Option{} + if ok, err := cmd.Parent().PersistentFlags().GetBool("insecure"); err != nil { + log.Fatalf("flag problems: %v", err) + } else if ok { + nameOpt = append(nameOpt, name.Insecure) + } + r, err := name.ParseReference(rebased, nameOpt...) + if err != nil { + log.Fatalf("parsing %s: %v", rebased, err) + } + + desc, err := crane.Head(orig, *options...) + if err != nil { + log.Fatalf("checking %s: %v", orig, err) + } + if !cmd.Parent().PersistentFlags().Changed("platform") && desc.MediaType.IsIndex() { + log.Fatalf("rebasing an index is not yet supported") + } + + origImg, err := crane.Pull(orig, *options...) + if err != nil { + return err + } + origMf, err := origImg.Manifest() + if err != nil { + return err + } + anns := origMf.Annotations + if newBase == "" && anns != nil { + newBase = anns[specsv1.AnnotationBaseImageName] + } + if newBase == "" { + return errors.New("could not determine new base image from annotations") + } + newBaseRef, err := name.ParseReference(newBase) + if err != nil { + return err + } + if oldBase == "" && anns != nil { + oldBaseDigest := anns[specsv1.AnnotationBaseImageDigest] + oldBase = newBaseRef.Context().Digest(oldBaseDigest).String() + } + if oldBase == "" { + return errors.New("could not determine old base image by digest from annotations") + } + + rebasedImg, err := rebaseImage(origImg, oldBase, newBase, *options...) + if err != nil { + return fmt.Errorf("rebasing image: %w", err) + } + + rebasedDigest, err := rebasedImg.Digest() + if err != nil { + return fmt.Errorf("digesting new image: %w", err) + } + origDigest, err := origImg.Digest() + if err != nil { + return err + } + if rebasedDigest == origDigest { + logs.Warn.Println("rebasing was no-op") + } + + if _, ok := r.(name.Digest); ok { + rebased = r.Context().Digest(rebasedDigest.String()).String() + } + logs.Progress.Println("pushing rebased image as", rebased) + if err := crane.Push(rebasedImg, rebased, *options...); err != nil { + log.Fatalf("pushing %s: %v", rebased, err) + } + + fmt.Println(r.Context().Digest(rebasedDigest.String())) + return nil + }, + } + rebaseCmd.Flags().StringVar(&orig, "original", "", "Original image to rebase (DEPRECATED: use positional arg instead)") + rebaseCmd.Flags().StringVar(&oldBase, "old_base", "", "Old base image to remove") + rebaseCmd.Flags().StringVar(&newBase, "new_base", "", "New base image to insert") + rebaseCmd.Flags().StringVar(&rebased, "rebased", "", "Tag to apply to rebased image (DEPRECATED: use --tag)") + rebaseCmd.Flags().StringVarP(&rebased, "tag", "t", "", "Tag to apply to rebased image") + return rebaseCmd +} + +// rebaseImage parses the references and uses them to perform a rebase on the +// original image. +// +// If oldBase or newBase are "", rebaseImage attempts to derive them using +// annotations in the original image. If those annotations are not found, +// rebaseImage returns an error. +// +// If rebasing is successful, base image annotations are set on the resulting +// image to facilitate implicit rebasing next time. +func rebaseImage(orig v1.Image, oldBase, newBase string, opt ...crane.Option) (v1.Image, error) { + m, err := orig.Manifest() + if err != nil { + return nil, err + } + if newBase == "" && m.Annotations != nil { + newBase = m.Annotations[specsv1.AnnotationBaseImageName] + if newBase != "" { + logs.Debug.Printf("Detected new base from %q annotation: %s", specsv1.AnnotationBaseImageName, newBase) + } + } + if newBase == "" { + return nil, fmt.Errorf("either new base or %q annotation is required", specsv1.AnnotationBaseImageName) + } + newBaseImg, err := crane.Pull(newBase, opt...) + if err != nil { + return nil, err + } + + if oldBase == "" && m.Annotations != nil { + oldBase = m.Annotations[specsv1.AnnotationBaseImageDigest] + if oldBase != "" { + newBaseRef, err := name.ParseReference(newBase) + if err != nil { + return nil, err + } + + oldBase = newBaseRef.Context().Digest(oldBase).String() + logs.Debug.Printf("Detected old base from %q annotation: %s", specsv1.AnnotationBaseImageDigest, oldBase) + } + } + if oldBase == "" { + return nil, fmt.Errorf("either old base or %q annotation is required", specsv1.AnnotationBaseImageDigest) + } + + oldBaseImg, err := crane.Pull(oldBase, opt...) + if err != nil { + return nil, err + } + + // NB: if newBase is an index, we need to grab the index's digest to + // annotate the resulting image, even though we pull the + // platform-specific image to rebase. + // crane.Digest will pull a platform-specific image, so use crane.Head + // here instead. + newBaseDesc, err := crane.Head(newBase, opt...) + if err != nil { + return nil, err + } + newBaseDigest := newBaseDesc.Digest.String() + + rebased, err := mutate.Rebase(orig, oldBaseImg, newBaseImg) + if err != nil { + return nil, err + } + + // Update base image annotations for the new image manifest. + logs.Debug.Printf("Setting annotation %q: %q", specsv1.AnnotationBaseImageDigest, newBaseDigest) + logs.Debug.Printf("Setting annotation %q: %q", specsv1.AnnotationBaseImageName, newBase) + return mutate.Annotations(rebased, map[string]string{ + specsv1.AnnotationBaseImageDigest: newBaseDigest, + specsv1.AnnotationBaseImageName: newBase, + }).(v1.Image), nil +} diff --git a/cmd/crane/cmd/root.go b/cmd/crane/cmd/root.go new file mode 100644 index 0000000..95412cc --- /dev/null +++ b/cmd/crane/cmd/root.go @@ -0,0 +1,148 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "crypto/tls" + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/docker/cli/cli/config" + "github.com/google/go-containerregistry/internal/cmd" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" +) + +const ( + use = "crane" + short = "Crane is a tool for managing container images" +) + +var Root = New(use, short, []crane.Option{}) + +// New returns a top-level command for crane. This is mostly exposed +// to share code with gcrane. +func New(use, short string, options []crane.Option) *cobra.Command { + verbose := false + insecure := false + ndlayers := false + platform := &platformValue{} + + root := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Usage() }, + DisableAutoGenTag: true, + SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + options = append(options, crane.WithContext(cmd.Context())) + // TODO(jonjohnsonjr): crane.Verbose option? + if verbose { + logs.Debug.SetOutput(os.Stderr) + } + if insecure { + options = append(options, crane.Insecure) + } + if ndlayers { + options = append(options, crane.WithNondistributable()) + } + if Version != "" { + binary := "crane" + if len(os.Args[0]) != 0 { + binary = filepath.Base(os.Args[0]) + } + options = append(options, crane.WithUserAgent(fmt.Sprintf("%s/%s", binary, Version))) + } + + options = append(options, crane.WithPlatform(platform.platform)) + + transport := remote.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: insecure, //nolint: gosec + } + + var rt http.RoundTripper = transport + + // Add any http headers if they are set in the config file. + cf, err := config.Load(os.Getenv("DOCKER_CONFIG")) + if err != nil { + logs.Debug.Printf("failed to read config file: %v", err) + } else if len(cf.HTTPHeaders) != 0 { + rt = &headerTransport{ + inner: rt, + httpHeaders: cf.HTTPHeaders, + } + } + + options = append(options, crane.WithTransport(rt)) + }, + } + + root.AddCommand( + NewCmdAppend(&options), + NewCmdAuth(options, "crane", "auth"), + NewCmdBlob(&options), + NewCmdCatalog(&options, "crane"), + NewCmdConfig(&options), + NewCmdCopy(&options), + NewCmdDelete(&options), + NewCmdDigest(&options), + cmd.NewCmdEdit(&options), + NewCmdExport(&options), + NewCmdFlatten(&options), + NewCmdIndex(&options), + NewCmdList(&options), + NewCmdManifest(&options), + NewCmdMutate(&options), + NewCmdOptimize(&options), + NewCmdPull(&options), + NewCmdPush(&options), + NewCmdRebase(&options), + NewCmdTag(&options), + NewCmdValidate(&options), + NewCmdVersion(), + newCmdRegistry(), + ) + + root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable debug logs") + root.PersistentFlags().BoolVar(&insecure, "insecure", false, "Allow image references to be fetched without TLS") + root.PersistentFlags().BoolVar(&ndlayers, "allow-nondistributable-artifacts", false, "Allow pushing non-distributable (foreign) layers") + root.PersistentFlags().Var(platform, "platform", "Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64).") + + return root +} + +// headerTransport sets headers on outgoing requests. +type headerTransport struct { + httpHeaders map[string]string + inner http.RoundTripper +} + +// RoundTrip implements http.RoundTripper. +func (ht *headerTransport) RoundTrip(in *http.Request) (*http.Response, error) { + for k, v := range ht.httpHeaders { + if http.CanonicalHeaderKey(k) == "User-Agent" { + // Docker sets this, which is annoying, since we're not docker. + // We might want to revisit completely ignoring this. + continue + } + in.Header.Set(k, v) + } + return ht.inner.RoundTrip(in) +} diff --git a/cmd/crane/cmd/serve.go b/cmd/crane/cmd/serve.go new file mode 100644 index 0000000..5b11153 --- /dev/null +++ b/cmd/crane/cmd/serve.go @@ -0,0 +1,84 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/google/go-containerregistry/pkg/registry" +) + +func newCmdRegistry() *cobra.Command { + cmd := &cobra.Command{ + Use: "registry", + } + cmd.AddCommand(newCmdServe()) + return cmd +} + +func newCmdServe() *cobra.Command { + return &cobra.Command{ + Use: "serve", + Short: "Serve an in-memory registry implementation", + Long: `This sub-command serves an in-memory registry implementation on port :8080 (or $PORT) + +The command blocks while the server accepts pushes and pulls. + +Contents are only stored in memory, and when the process exits, pushed data is lost.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + + port := os.Getenv("PORT") + if port == "" { + port = "0" + } + listener, err := net.Listen("tcp", "localhost:"+port) + if err != nil { + log.Fatalln(err) + } + porti := listener.Addr().(*net.TCPAddr).Port + port = fmt.Sprintf("%d", porti) + + s := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, // prevent slowloris, quiet linter + Handler: registry.New(), + } + log.Printf("serving on port %s", port) + + errCh := make(chan error) + go func() { errCh <- s.Serve(listener) }() + + <-ctx.Done() + log.Println("shutting down...") + if err := s.Shutdown(ctx); err != nil { + return err + } + + if err := <-errCh; !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }, + } +} diff --git a/cmd/crane/cmd/tag.go b/cmd/crane/cmd/tag.go new file mode 100644 index 0000000..9af803a --- /dev/null +++ b/cmd/crane/cmd/tag.go @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/google/go-containerregistry/pkg/crane" + "github.com/spf13/cobra" +) + +// NewCmdTag creates a new cobra.Command for the tag subcommand. +func NewCmdTag(options *[]crane.Option) *cobra.Command { + return &cobra.Command{ + Use: "tag IMG TAG", + Short: "Efficiently tag a remote image", + Long: `This differs slightly from the "copy" command in a couple subtle ways: + +1. You don't have to specify the entire repository for the tag you're adding. For example, these two commands are functionally equivalent: +` + "```" + ` +crane cp registry.example.com/library/ubuntu:v0 registry.example.com/library/ubuntu:v1 +crane tag registry.example.com/library/ubuntu:v0 v1 +` + "```" + ` + +2. We can skip layer existence checks because we know the manifest already exists. This makes "tag" slightly faster than "copy".`, + Example: `# Add a v1 tag to ubuntu +crane tag ubuntu v1`, + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + img, tag := args[0], args[1] + return crane.Tag(img, tag, *options...) + }, + } +} diff --git a/cmd/crane/cmd/util.go b/cmd/crane/cmd/util.go new file mode 100644 index 0000000..f4ff651 --- /dev/null +++ b/cmd/crane/cmd/util.go @@ -0,0 +1,86 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type platformsValue struct { + platforms []v1.Platform +} + +func (ps *platformsValue) Set(platform string) error { + if ps.platforms == nil { + ps.platforms = []v1.Platform{} + } + p, err := parsePlatform(platform) + if err != nil { + return err + } + pv := platformValue{p} + ps.platforms = append(ps.platforms, *pv.platform) + return nil +} + +func (ps *platformsValue) String() string { + ss := make([]string, 0, len(ps.platforms)) + for _, p := range ps.platforms { + ss = append(ss, p.String()) + } + return strings.Join(ss, ",") +} + +func (ps *platformsValue) Type() string { + return "platform(s)" +} + +type platformValue struct { + platform *v1.Platform +} + +func (pv *platformValue) Set(platform string) error { + p, err := parsePlatform(platform) + if err != nil { + return err + } + pv.platform = p + return nil +} + +func (pv *platformValue) String() string { + return platformToString(pv.platform) +} + +func (pv *platformValue) Type() string { + return "platform" +} + +func platformToString(p *v1.Platform) string { + if p == nil { + return "all" + } + return p.String() +} + +func parsePlatform(platform string) (*v1.Platform, error) { + if platform == "all" { + return nil, nil + } + + return v1.ParsePlatform(platform) +} diff --git a/cmd/crane/cmd/validate.go b/cmd/crane/cmd/validate.go new file mode 100644 index 0000000..4a4acbd --- /dev/null +++ b/cmd/crane/cmd/validate.go @@ -0,0 +1,73 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/validate" + "github.com/spf13/cobra" +) + +// NewCmdValidate creates a new cobra.Command for the validate subcommand. +func NewCmdValidate(options *[]crane.Option) *cobra.Command { + var ( + tarballPath, remoteRef string + fast bool + ) + + validateCmd := &cobra.Command{ + Use: "validate", + Short: "Validate that an image is well-formed", + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + for flag, maker := range map[string]func(string, ...crane.Option) (v1.Image, error){ + tarballPath: makeTarball, + remoteRef: crane.Pull, + } { + if flag == "" { + continue + } + img, err := maker(flag, *options...) + if err != nil { + return fmt.Errorf("failed to read image %s: %w", flag, err) + } + + opt := []validate.Option{} + if fast { + opt = append(opt, validate.Fast) + } + if err := validate.Image(img, opt...); err != nil { + fmt.Printf("FAIL: %s: %v\n", flag, err) + return err + } + fmt.Printf("PASS: %s\n", flag) + } + return nil + }, + } + validateCmd.Flags().StringVar(&tarballPath, "tarball", "", "Path to tarball to validate") + validateCmd.Flags().StringVar(&remoteRef, "remote", "", "Name of remote image to validate") + validateCmd.Flags().BoolVar(&fast, "fast", false, "Skip downloading/digesting layers") + + return validateCmd +} + +func makeTarball(path string, _ ...crane.Option) (v1.Image, error) { + return tarball.ImageFromPath(path, nil) +} diff --git a/cmd/crane/cmd/version.go b/cmd/crane/cmd/version.go new file mode 100644 index 0000000..b906a5d --- /dev/null +++ b/cmd/crane/cmd/version.go @@ -0,0 +1,56 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "runtime/debug" + + "github.com/spf13/cobra" +) + +// Version can be set via: +// -ldflags="-X 'github.com/google/go-containerregistry/cmd/crane/cmd.Version=$TAG'" +var Version string + +func init() { + if Version == "" { + i, ok := debug.ReadBuildInfo() + if !ok { + return + } + Version = i.Main.Version + } +} + +// NewCmdVersion creates a new cobra.Command for the version subcommand. +func NewCmdVersion() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the version", + Long: `The version string is completely dependent on how the binary was built, so you should not depend on the version format. It may change without notice. + +This could be an arbitrary string, if specified via -ldflags. +This could also be the go module version, if built with go modules (often "(devel)").`, + Args: cobra.NoArgs, + Run: func(_ *cobra.Command, _ []string) { + if Version == "" { + fmt.Println("could not determine build information") + } else { + fmt.Println(Version) + } + }, + } +} diff --git a/cmd/crane/depcheck_test.go b/cmd/crane/depcheck_test.go new file mode 100644 index 0000000..f36056e --- /dev/null +++ b/cmd/crane/depcheck_test.go @@ -0,0 +1,32 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/go-containerregistry/internal/depcheck" +) + +func TestDeps(t *testing.T) { + if testing.Short() { + t.Skip("skipping slow depcheck") + } + depcheck.AssertNoDependency(t, map[string][]string{ + "github.com/google/go-containerregistry/cmd/crane": { + "github.com/google/go-containerregistry/pkg/v1/daemon", + }, + }) +} diff --git a/cmd/crane/doc/crane.md b/cmd/crane/doc/crane.md new file mode 100644 index 0000000..afd1b24 --- /dev/null +++ b/cmd/crane/doc/crane.md @@ -0,0 +1,42 @@ +## crane + +Crane is a tool for managing container images + +``` +crane [flags] +``` + +### Options + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + -h, --help help for crane + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane append](crane_append.md) - Append contents of a tarball to a remote image +* [crane auth](crane_auth.md) - Log in or access credentials +* [crane blob](crane_blob.md) - Read a blob from the registry +* [crane catalog](crane_catalog.md) - List the repos in a registry +* [crane config](crane_config.md) - Get the config of an image +* [crane copy](crane_copy.md) - Efficiently copy a remote image from src to dst while retaining the digest value +* [crane delete](crane_delete.md) - Delete an image reference from its registry +* [crane digest](crane_digest.md) - Get the digest of an image +* [crane export](crane_export.md) - Export filesystem of a container image as a tarball +* [crane flatten](crane_flatten.md) - Flatten an image's layers into a single layer +* [crane index](crane_index.md) - Modify an image index. +* [crane ls](crane_ls.md) - List the tags in a repo +* [crane manifest](crane_manifest.md) - Get the manifest of an image +* [crane mutate](crane_mutate.md) - Modify image labels and annotations. The container must be pushed to a registry, and the manifest is updated there. +* [crane pull](crane_pull.md) - Pull remote images by reference and store their contents locally +* [crane push](crane_push.md) - Push local image contents to a remote registry +* [crane rebase](crane_rebase.md) - Rebase an image onto a new base image +* [crane registry](crane_registry.md) - +* [crane tag](crane_tag.md) - Efficiently tag a remote image +* [crane validate](crane_validate.md) - Validate that an image is well-formed +* [crane version](crane_version.md) - Print the version + diff --git a/cmd/crane/doc/crane_append.md b/cmd/crane/doc/crane_append.md new file mode 100644 index 0000000..d637dd1 --- /dev/null +++ b/cmd/crane/doc/crane_append.md @@ -0,0 +1,43 @@ +## crane append + +Append contents of a tarball to a remote image + +### Synopsis + +This sub-command pushes an image based on an (optional) +base image, with appended layers containing the contents of the +provided tarballs. + +If the base image is a Windows base image (i.e., its config.OS is "windows"), +the contents of the tarballs will be modified to be suitable for a Windows +container image. + +``` +crane append [flags] +``` + +### Options + +``` + -b, --base string Name of base image to append to + -h, --help help for append + -f, --new_layer strings Path to tarball to append to image + -t, --new_tag string Tag to apply to resulting image + --oci-empty-base If true, empty base image will have OCI media types instead of Docker + -o, --output string Path to new tarball of resulting image + --set-base-image-annotations If true, annotate the resulting image as being based on the base image +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_auth.md b/cmd/crane/doc/crane_auth.md new file mode 100644 index 0000000..6eb8fc8 --- /dev/null +++ b/cmd/crane/doc/crane_auth.md @@ -0,0 +1,29 @@ +## crane auth + +Log in or access credentials + +``` +crane auth [flags] +``` + +### Options + +``` + -h, --help help for auth +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images +* [crane auth get](crane_auth_get.md) - Implements a credential helper +* [crane auth login](crane_auth_login.md) - Log in to a registry + diff --git a/cmd/crane/doc/crane_auth_get.md b/cmd/crane/doc/crane_auth_get.md new file mode 100644 index 0000000..6ff89c1 --- /dev/null +++ b/cmd/crane/doc/crane_auth_get.md @@ -0,0 +1,38 @@ +## crane auth get + +Implements a credential helper + +``` +crane auth get [REGISTRY_ADDR] [flags] +``` + +### Examples + +``` + # Read configured credentials for reg.example.com + $ echo "reg.example.com" | crane auth get + {"username":"AzureDiamond","password":"hunter2"} + # or + $ crane auth get reg.example.com + {"username":"AzureDiamond","password":"hunter2"} +``` + +### Options + +``` + -h, --help help for get +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane auth](crane_auth.md) - Log in or access credentials + diff --git a/cmd/crane/doc/crane_auth_login.md b/cmd/crane/doc/crane_auth_login.md new file mode 100644 index 0000000..1fec423 --- /dev/null +++ b/cmd/crane/doc/crane_auth_login.md @@ -0,0 +1,37 @@ +## crane auth login + +Log in to a registry + +``` +crane auth login [OPTIONS] [SERVER] [flags] +``` + +### Examples + +``` + # Log in to reg.example.com + crane auth login reg.example.com -u AzureDiamond -p hunter2 +``` + +### Options + +``` + -h, --help help for login + -p, --password string Password + --password-stdin Take the password from stdin + -u, --username string Username +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane auth](crane_auth.md) - Log in or access credentials + diff --git a/cmd/crane/doc/crane_blob.md b/cmd/crane/doc/crane_blob.md new file mode 100644 index 0000000..36f615a --- /dev/null +++ b/cmd/crane/doc/crane_blob.md @@ -0,0 +1,33 @@ +## crane blob + +Read a blob from the registry + +``` +crane blob BLOB [flags] +``` + +### Examples + +``` +crane blob ubuntu@sha256:4c1d20cdee96111c8acf1858b62655a37ce81ae48648993542b7ac363ac5c0e5 > blob.tar.gz +``` + +### Options + +``` + -h, --help help for blob +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_catalog.md b/cmd/crane/doc/crane_catalog.md new file mode 100644 index 0000000..99b81ad --- /dev/null +++ b/cmd/crane/doc/crane_catalog.md @@ -0,0 +1,34 @@ +## crane catalog + +List the repos in a registry + +``` +crane catalog [REGISTRY] [flags] +``` + +### Examples + +``` + # list the repos for reg.example.com + $ crane catalog reg.example.com +``` + +### Options + +``` + -h, --help help for catalog +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_config.md b/cmd/crane/doc/crane_config.md new file mode 100644 index 0000000..5d7fa5a --- /dev/null +++ b/cmd/crane/doc/crane_config.md @@ -0,0 +1,27 @@ +## crane config + +Get the config of an image + +``` +crane config IMAGE [flags] +``` + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_copy.md b/cmd/crane/doc/crane_copy.md new file mode 100644 index 0000000..8e7e1a8 --- /dev/null +++ b/cmd/crane/doc/crane_copy.md @@ -0,0 +1,27 @@ +## crane copy + +Efficiently copy a remote image from src to dst while retaining the digest value + +``` +crane copy SRC DST [flags] +``` + +### Options + +``` + -h, --help help for copy +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_delete.md b/cmd/crane/doc/crane_delete.md new file mode 100644 index 0000000..7932ea2 --- /dev/null +++ b/cmd/crane/doc/crane_delete.md @@ -0,0 +1,27 @@ +## crane delete + +Delete an image reference from its registry + +``` +crane delete IMAGE [flags] +``` + +### Options + +``` + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_digest.md b/cmd/crane/doc/crane_digest.md new file mode 100644 index 0000000..f141b36 --- /dev/null +++ b/cmd/crane/doc/crane_digest.md @@ -0,0 +1,29 @@ +## crane digest + +Get the digest of an image + +``` +crane digest IMAGE [flags] +``` + +### Options + +``` + --full-ref (Optional) if true, print the full image reference by digest + -h, --help help for digest + --tarball string (Optional) path to tarball containing the image +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_export.md b/cmd/crane/doc/crane_export.md new file mode 100644 index 0000000..ca10c56 --- /dev/null +++ b/cmd/crane/doc/crane_export.md @@ -0,0 +1,40 @@ +## crane export + +Export filesystem of a container image as a tarball + +``` +crane export IMAGE|- TARBALL|- [flags] +``` + +### Examples + +``` + # Write tarball to stdout + crane export ubuntu - + + # Write tarball to file + crane export ubuntu ubuntu.tar + + # Read image from stdin + crane export - ubuntu.tar +``` + +### Options + +``` + -h, --help help for export +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_flatten.md b/cmd/crane/doc/crane_flatten.md new file mode 100644 index 0000000..68e6bc6 --- /dev/null +++ b/cmd/crane/doc/crane_flatten.md @@ -0,0 +1,28 @@ +## crane flatten + +Flatten an image's layers into a single layer + +``` +crane flatten [flags] +``` + +### Options + +``` + -h, --help help for flatten + -t, --tag string New tag to apply to flattened image. If not provided, push by digest to the original image repository. +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_index.md b/cmd/crane/doc/crane_index.md new file mode 100644 index 0000000..2adea48 --- /dev/null +++ b/cmd/crane/doc/crane_index.md @@ -0,0 +1,29 @@ +## crane index + +Modify an image index. + +``` +crane index [flags] +``` + +### Options + +``` + -h, --help help for index +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images +* [crane index append](crane_index_append.md) - Append manifests to a remote index. +* [crane index filter](crane_index_filter.md) - Modifies a remote index by filtering based on platform. + diff --git a/cmd/crane/doc/crane_index_append.md b/cmd/crane/doc/crane_index_append.md new file mode 100644 index 0000000..a6c1541 --- /dev/null +++ b/cmd/crane/doc/crane_index_append.md @@ -0,0 +1,47 @@ +## crane index append + +Append manifests to a remote index. + +### Synopsis + +This sub-command pushes an index based on an (optional) base index, with appended manifests. + +The platform for appended manifests is inferred from the config file or omitted if that is infeasible. + +``` +crane index append [flags] +``` + +### Examples + +``` + # Append a windows hello-world image to ubuntu, push to example.com/hello-world:weird + crane index append ubuntu -m hello-world@sha256:87b9ca29151260634b95efb84d43b05335dc3ed36cc132e2b920dd1955342d20 -t example.com/hello-world:weird + + # Create an index from scratch for etcd. + crane index append -m registry.k8s.io/etcd-amd64:3.4.9 -m registry.k8s.io/etcd-arm64:3.4.9 -t example.com/etcd +``` + +### Options + +``` + --docker-empty-base If true, empty base index will have Docker media types instead of OCI + --flatten If true, appending an index will append each of its children rather than the index itself (default true) + -h, --help help for append + -m, --manifest strings References to manifests to append to the base index + -t, --tag string Tag to apply to resulting image +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane index](crane_index.md) - Modify an image index. + diff --git a/cmd/crane/doc/crane_index_filter.md b/cmd/crane/doc/crane_index_filter.md new file mode 100644 index 0000000..bda1f8d --- /dev/null +++ b/cmd/crane/doc/crane_index_filter.md @@ -0,0 +1,41 @@ +## crane index filter + +Modifies a remote index by filtering based on platform. + +``` +crane index filter [flags] +``` + +### Examples + +``` + # Filter out weird platforms from ubuntu, copy result to example.com/ubuntu + crane index filter ubuntu --platform linux/amd64 --platform linux/arm64 -t example.com/ubuntu + + # Filter out any non-linux platforms, push to example.com/hello-world + crane index filter hello-world --platform linux -t example.com/hello-world + + # Same as above, but in-place + crane index filter example.com/hello-world:some-tag --platform linux +``` + +### Options + +``` + -h, --help help for filter + --platform platform(s) Specifies the platform(s) to keep from base in the form os/arch[/variant][:osversion][,] (e.g. linux/amd64). + -t, --tag string Tag to apply to resulting image +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane index](crane_index.md) - Modify an image index. + diff --git a/cmd/crane/doc/crane_ls.md b/cmd/crane/doc/crane_ls.md new file mode 100644 index 0000000..6616820 --- /dev/null +++ b/cmd/crane/doc/crane_ls.md @@ -0,0 +1,29 @@ +## crane ls + +List the tags in a repo + +``` +crane ls REPO [flags] +``` + +### Options + +``` + --full-ref (Optional) if true, print the full image reference + -h, --help help for ls + --omit-digest-tags (Optional), if true, omit digest tags (e.g., ':sha256-...') +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_manifest.md b/cmd/crane/doc/crane_manifest.md new file mode 100644 index 0000000..3d61b4e --- /dev/null +++ b/cmd/crane/doc/crane_manifest.md @@ -0,0 +1,27 @@ +## crane manifest + +Get the manifest of an image + +``` +crane manifest IMAGE [flags] +``` + +### Options + +``` + -h, --help help for manifest +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_mutate.md b/cmd/crane/doc/crane_mutate.md new file mode 100644 index 0000000..0174b09 --- /dev/null +++ b/cmd/crane/doc/crane_mutate.md @@ -0,0 +1,37 @@ +## crane mutate + +Modify image labels and annotations. The container must be pushed to a registry, and the manifest is updated there. + +``` +crane mutate [flags] +``` + +### Options + +``` + -a, --annotation stringToString New annotations to add (default []) + --append strings Path to tarball to append to image + --cmd strings New cmd to set + --entrypoint strings New entrypoint to set + -e, --env stringToString New envvar to add (default []) + -h, --help help for mutate + -l, --label stringToString New labels to add (default []) + -o, --output string Path to new tarball of resulting image + --repo string Repository to push the mutated image to. If provided, push by digest to this repository. + -t, --tag string New tag reference to apply to mutated image. If not provided, push by digest to the original image repository. + -u, --user string New user to set +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_pull.md b/cmd/crane/doc/crane_pull.md new file mode 100644 index 0000000..790a1cb --- /dev/null +++ b/cmd/crane/doc/crane_pull.md @@ -0,0 +1,30 @@ +## crane pull + +Pull remote images by reference and store their contents locally + +``` +crane pull IMAGE TARBALL [flags] +``` + +### Options + +``` + --annotate-ref Preserves image reference used to pull as an annotation when used with --format=oci + -c, --cache_path string Path to cache image layers + --format string Format in which to save images ("tarball", "legacy", or "oci") (default "tarball") + -h, --help help for pull +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_push.md b/cmd/crane/doc/crane_push.md new file mode 100644 index 0000000..64bacf6 --- /dev/null +++ b/cmd/crane/doc/crane_push.md @@ -0,0 +1,33 @@ +## crane push + +Push local image contents to a remote registry + +### Synopsis + +If the PATH is a directory, it will be read as an OCI image layout. Otherwise, PATH is assumed to be a docker-style tarball. + +``` +crane push PATH IMAGE [flags] +``` + +### Options + +``` + -h, --help help for push + --image-refs string path to file where a list of the published image references will be written + --index push a collection of images as a single index, currently required if PATH contains multiple images +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_rebase.md b/cmd/crane/doc/crane_rebase.md new file mode 100644 index 0000000..e30f078 --- /dev/null +++ b/cmd/crane/doc/crane_rebase.md @@ -0,0 +1,32 @@ +## crane rebase + +Rebase an image onto a new base image + +``` +crane rebase [flags] +``` + +### Options + +``` + -h, --help help for rebase + --new_base string New base image to insert + --old_base string Old base image to remove + --original string Original image to rebase (DEPRECATED: use positional arg instead) + --rebased string Tag to apply to rebased image (DEPRECATED: use --tag) + -t, --tag string Tag to apply to rebased image +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_registry.md b/cmd/crane/doc/crane_registry.md new file mode 100644 index 0000000..d2bf920 --- /dev/null +++ b/cmd/crane/doc/crane_registry.md @@ -0,0 +1,24 @@ +## crane registry + + + +### Options + +``` + -h, --help help for registry +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images +* [crane registry serve](crane_registry_serve.md) - Serve an in-memory registry implementation + diff --git a/cmd/crane/doc/crane_registry_serve.md b/cmd/crane/doc/crane_registry_serve.md new file mode 100644 index 0000000..6c46861 --- /dev/null +++ b/cmd/crane/doc/crane_registry_serve.md @@ -0,0 +1,35 @@ +## crane registry serve + +Serve an in-memory registry implementation + +### Synopsis + +This sub-command serves an in-memory registry implementation on port :8080 (or $PORT) + +The command blocks while the server accepts pushes and pulls. + +Contents are only stored in memory, and when the process exits, pushed data is lost. + +``` +crane registry serve [flags] +``` + +### Options + +``` + -h, --help help for serve +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane registry](crane_registry.md) - + diff --git a/cmd/crane/doc/crane_tag.md b/cmd/crane/doc/crane_tag.md new file mode 100644 index 0000000..5433467 --- /dev/null +++ b/cmd/crane/doc/crane_tag.md @@ -0,0 +1,46 @@ +## crane tag + +Efficiently tag a remote image + +### Synopsis + +This differs slightly from the "copy" command in a couple subtle ways: + +1. You don't have to specify the entire repository for the tag you're adding. For example, these two commands are functionally equivalent: +``` +crane cp registry.example.com/library/ubuntu:v0 registry.example.com/library/ubuntu:v1 +crane tag registry.example.com/library/ubuntu:v0 v1 +``` + +2. We can skip layer existence checks because we know the manifest already exists. This makes "tag" slightly faster than "copy". + +``` +crane tag IMG TAG [flags] +``` + +### Examples + +``` +# Add a v1 tag to ubuntu +crane tag ubuntu v1 +``` + +### Options + +``` + -h, --help help for tag +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_validate.md b/cmd/crane/doc/crane_validate.md new file mode 100644 index 0000000..cff22f8 --- /dev/null +++ b/cmd/crane/doc/crane_validate.md @@ -0,0 +1,30 @@ +## crane validate + +Validate that an image is well-formed + +``` +crane validate [flags] +``` + +### Options + +``` + --fast Skip downloading/digesting layers + -h, --help help for validate + --remote string Name of remote image to validate + --tarball string Path to tarball to validate +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/doc/crane_version.md b/cmd/crane/doc/crane_version.md new file mode 100644 index 0000000..0972792 --- /dev/null +++ b/cmd/crane/doc/crane_version.md @@ -0,0 +1,34 @@ +## crane version + +Print the version + +### Synopsis + +The version string is completely dependent on how the binary was built, so you should not depend on the version format. It may change without notice. + +This could be an arbitrary string, if specified via -ldflags. +This could also be the go module version, if built with go modules (often "(devel)"). + +``` +crane version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --allow-nondistributable-artifacts Allow pushing non-distributable (foreign) layers + --insecure Allow image references to be fetched without TLS + --platform platform Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default all) + -v, --verbose Enable debug logs +``` + +### SEE ALSO + +* [crane](crane.md) - Crane is a tool for managing container images + diff --git a/cmd/crane/help/README.md b/cmd/crane/help/README.md new file mode 100644 index 0000000..c97606c --- /dev/null +++ b/cmd/crane/help/README.md @@ -0,0 +1,5 @@ +## Generate docs for `crane` + +```go +go run cmd/crane/help/main.go --dir=cmd/crane/doc/ +``` diff --git a/cmd/crane/help/main.go b/cmd/crane/help/main.go new file mode 100644 index 0000000..e086a55 --- /dev/null +++ b/cmd/crane/help/main.go @@ -0,0 +1,45 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/cmd/crane/cmd" + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" +) + +var dir string +var root = &cobra.Command{ + Use: "gendoc", + Short: "Generate crane's help docs", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + return doc.GenMarkdownTree(cmd.Root, dir) + }, +} + +func init() { + root.Flags().StringVarP(&dir, "dir", "d", ".", "Path to directory in which to generate docs") +} + +func main() { + if err := root.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/cmd/crane/main.go b/cmd/crane/main.go new file mode 100644 index 0000000..6dbdf22 --- /dev/null +++ b/cmd/crane/main.go @@ -0,0 +1,38 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "os/signal" + + "github.com/google/go-containerregistry/cmd/crane/cmd" + "github.com/google/go-containerregistry/pkg/logs" +) + +func init() { + logs.Warn.SetOutput(os.Stderr) + logs.Progress.SetOutput(os.Stderr) +} + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if err := cmd.Root.ExecuteContext(ctx); err != nil { + cancel() + os.Exit(1) + } +} diff --git a/cmd/crane/rebase.md b/cmd/crane/rebase.md new file mode 100644 index 0000000..da4b255 --- /dev/null +++ b/cmd/crane/rebase.md @@ -0,0 +1,125 @@ +### This code is experimental and might break you if not used correctly. + +The `rebase` command efficiently rewrites an image to replace the base image it +is `FROM` with a new base image. + +![rebase visualization](./rebase.png) + +([link](https://docs.google.com/drawings/d/1w8UxTZDRbDWVoqnbr17SJuU73pRxpOmOk_vzmC9WB2k/edit)) + +**This is not safe in general**, but it can be extremely useful for platform +providers, e.g. when a vulnerability is discovered in a base layer and many +thousands or millions of applications need to be patched in a short period of +time. + +A commonly accepted guideline for rebase-safety is ABI-compatibility, but this +is still imperfect in a handful of ways, and the exact contract varies between +platform providers. + +Rebasing is best suited for when rebuilding is either impossible (source is not +available) or impractical (too much work, too little time). + +## Using `crane rebase` + +For purposes of illustration, imagine you've built a container image +`my-app:latest`, which is `FROM ubuntu`: + +``` +FROM ubuntu + +RUN ./very-expensive-build-process.sh + +ENTRYPOINT ["/bin/myapp"] +``` + +A serious vulnerability has been found in the `ubuntu` base image, and a new +patched version has been released, tagged as `ubuntu:latest`. + +You could build your app image again, and the Dockerfile's `FROM ubuntu` +directive would pick up the new base image release, but that requires a full +rebuild of your entire app from source, which might take a long time, and might +pull in other unrelated changes in dependencies. + +You may have thousands of images containing the vulnerability. You just want to +release this critical bug fix across all your apps, as quickly as possible. + +Instead, you could use `crane rebase` to replace the vulnerable base image +layers in your image with the patched base image layers, without requiring a +full rebuild from source. + +``` +$ crane rebase my-app:latest \ + --old_base=ubuntu@sha256:deadbeef... \ + --new_base=ubuntu:latest \ + --tag=my-app:rebased +``` + +This command: + +1. fetches the manifest for the original image `my-app:latest`, and the + `old_base` and `new_base` images +1. checks that the original image is indeed based on `old_base` +1. removes `old_base`'s layers from the original image +1. replaces them with `new_base`'s layers +1. computes and uploads a new manifest for the image, tagged as `--tag`. + +If `--tag` is not specified, its value will be assumed to be the original +image's name. If the original image was specified by digest, the resulting +image will be pushed by digest only. + +`crane rebase` will print the rebased image name by digest to `stdout`. + +### Base Image Annotation Hints + +The OCI image spec includes some [standard image +annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md) +that can provide hints for the `--old_base` and `--new_base` flag values, so +these don't need to be specified: + +- **`org.opencontainers.image.base.digest`** specifies the original digest of + the base image +- **`org.opencontainers.image.base.name`** specifies the original base image's + reference + +If the original image has these annotations, you can omit the `--old_base` and +`--new_base` flags, and their values will be assumed to be: + +- `--old_base`: the `base.name` annotation value, plus the `base.digest` + annotation value +- `--new_base`: the `base.name` annotation value + +If these annotation values are invalid, and the flags aren't set, the operation +will fail. + +Whether or not the annotation values were set on the original image, they +_will_ be set on the resulting rebased image, to ease future rebase operations +on that image. + +`crane append` also supports the `--set-base-image-annotations` flag, which, if +true, will set these annotations on the resulting image. + +## Caveats + +The tool has no visibility into what the specific contents of the resulting +image, and has no idea what constitutes a "valid" image. As a result, it's +perfectly capable of producing an image that's entirely invalid garbage. +Rebasing arbitrary layers in an image is not a good idea. + +To help prevent garbage images, rebasing should only be done at a point in the +layer stack between "base" layers and "app" layers. These should adhere to some +contract about what "base" layers can be expected to produce, and what "app" +layers should expect from base layers. + +In the example above, for instance, we assume that the Ubuntu base image is +adhering to some contract with downstream app layers, that it won't remove or +drastically change what it provides to the app layer. If the `new_base` layers +removed some installed package, or made a breaking change to the version of +some compiler expected by the uppermost app layers, the resulting rebased image +might be invalid. + +In general, it's a good practice to tag rebased images to some other tag than +the `original` tag, perform some sanity checks, then tag the image to the +`original` tag once it's determined the image is valid. + +There is ongoing work to standardize and advertise base image contract +adherence to make rebasing safer. diff --git a/cmd/crane/rebase.png b/cmd/crane/rebase.png new file mode 100644 index 0000000..449bdfe Binary files /dev/null and b/cmd/crane/rebase.png differ diff --git a/cmd/crane/rebase_test.sh b/cmd/crane/rebase_test.sh new file mode 100755 index 0000000..727062b --- /dev/null +++ b/cmd/crane/rebase_test.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -ex + +tmp=$(mktemp -d) + +go install ./cmd/registry +go build -o ./crane ./cmd/crane + +# Start a local registry. +registry & +PID=$! +function cleanup { + kill $PID + rm -r ${tmp} + rm ./crane +} +trap cleanup EXIT + +sleep 1 # Wait for registry to be up. + +# Create an image localhost:1338/base containing a.txt +echo a > ${tmp}/a.txt +old_base=$(./crane append -f <(tar -f - -c ${tmp}) -t localhost:1338/base) +rm ${tmp}/a.txt + +# Append to that image localhost:1338/rebaseme +echo top > ${tmp}/top.txt +orig=$(./crane append -f <(tar -f - -c ${tmp}) -b ${old_base} -t localhost:1338/rebaseme) +rm ${tmp}/top.txt + +# Annotate that image as the base image (by ref and digest) +# TODO: do this with a flag to --append +orig=$(./crane mutate ${orig} \ + --annotation org.opencontainers.image.base.name=localhost:1338/base \ + --annotation org.opencontainers.image.base.digest=$(./crane digest localhost:1338/base)) + +# Update localhost:1338/base containing b.txt +echo b > ${tmp}/b.txt +new_base=$(./crane append -f <(tar -f - -c ${tmp}) -t localhost:1338/base) +rm ${tmp}/b.txt + +# Rebase using annotations +rebased=$(./crane rebase ${orig}) + +# List files in the rebased image. +./crane export ${rebased} - | tar -tvf - + +# Extract b.txt out of the rebased image. +./crane export ${rebased} - | tar -Oxf - ${tmp:1}/b.txt + +# Extract top.txt out of the rebased image. +./crane export ${rebased} - | tar -Oxf - ${tmp:1}/top.txt + +# a.txt is _not_ in the rebased image. +set +e +./crane export ${rebased} - | tar -Oxf - ${tmp:1}/a.txt # this should fail +code=$? +echo "finding a.txt exited ${code}" +if [[ $code -eq 0 ]]; then + echo "a.txt was found in rebased image" + exit 1 +fi diff --git a/cmd/crane/recipes.md b/cmd/crane/recipes.md new file mode 100644 index 0000000..1c0121d --- /dev/null +++ b/cmd/crane/recipes.md @@ -0,0 +1,105 @@ +# `crane` Recipes + +Useful tips and things you can do with `crane` and other standard tools. + +### List files in an image + +``` +crane export ubuntu - | tar -tvf - | less +``` + +### Extract a single file from an image + +``` +crane export ubuntu - | tar -Oxf - etc/passwd +``` + +Note: Be sure to remove the leading `/` from the path (not `/etc/passwd`). This behavior will not follow symlinks. + +### Bundle directory contents into an image + +``` +crane append -f <(tar -f - -c some-dir/) -t ${IMAGE} +``` + +By default, this produces an image with one layer containing the directory contents. Add `-b ${BASE_IMAGE}` to append the layer to a base image instead. + +You can extend this even further with `crane mutate`, to make an executable in the appended layer the image's entrypoint. + +``` +crane mutate ${IMAGE} --entrypoint=some-dir/entrypoint.sh +``` + +Because `crane append` emits the full image reference, these calls can even be chained together: + +``` +crane mutate $( + crane append -f <(tar -f - -c some-dir/) -t ${IMAGE} +) --entrypoint=some-dir/entrypoint.sh +``` + +This will bundle `some-dir/` into an image, push it, mutate its entrypoint to `some-dir/entrypoint.sh`, and push that new image by digest. + +### Diff two configs + +``` +diff <(crane config busybox:1.32 | jq) <(crane config busybox:1.33 | jq) +``` + +### Diff two manifests + +``` +diff <(crane manifest busybox:1.32 | jq) <(crane manifest busybox:1.33 | jq) +``` + +### Diff filesystem contents + +``` +diff \ + <(crane export gcr.io/kaniko-project/executor:v1.6.0-debug - | tar -tvf - | sort) \ + <(crane export gcr.io/kaniko-project/executor:v1.7.0-debug - | tar -tvf - | sort) +``` + +This will show file size diffs and (unfortunately) modified time diffs. + +With some work, you can use `cut` and other built-in Unix tools to ignore these diffs. + +### Get total image size + +Given an image manifest, you can calculate the total size of all layer blobs and the image's config blob using `jq`: + +``` +crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)' +``` + +This will produce a number of bytes, which you can make human-readable by passing to [`numfmt`](https://www.gnu.org/software/coreutils/manual/html_node/numfmt-invocation.html) + +``` +crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)' | numfmt --to=iec +``` + +For image indexes, you can pass the `--platform` flag to `crane` to get a platform-specific image. + +### Filter irrelevant platforms from a multi-platform image + +Perhaps you use a base image that supports a wide variety of exotic platforms, but you only care about linux/amd64 and linux/arm64. +If you want to copy that base image into a different registry, you will end up with a bunch of images you don't use. +You can filter the base to include only platforms that are relevant to you. + +``` +crane index filter ubuntu --platform linux/amd64 --platform linux/arm64 -t ${IMAGE} +``` + +Note that this will obviously modify the digest of the multi-platform image you're using, so this may invalidate other artifacts that reference it, e.g. signatures. + +### Create a multi-platform image from scratch + +If you have a bunch of platform-specific images that you want to turn into a multi-platform image, `crane index append` can do that: + +``` +crane index append -t ${IMAGE} \ + -m ubuntu@sha256:c985bc3f77946b8e92c9a3648c6f31751a7dd972e06604785e47303f4ad47c4c \ + -m ubuntu@sha256:61bd0b97000996232eb07b8d0e9375d14197f78aa850c2506417ef995a7199a7 +``` + +Note that this is less flexible than [`manifest-tool`](https://github.com/estesp/manifest-tool) because it derives the platform from each image's config file, but it should work in most cases. diff --git a/cmd/gcrane/README.md b/cmd/gcrane/README.md new file mode 100644 index 0000000..c8c9ba2 --- /dev/null +++ b/cmd/gcrane/README.md @@ -0,0 +1,65 @@ +# `gcrane` + + + +This tool implements a superset of the [`crane`](../crane/README.md) commands, with +additional commands that are specific to [gcr.io](https://gcr.io). + +Note that this relies on some implementation details of GCR that are not +consistent with the [registry spec](https://docs.docker.com/registry/spec/api/), +so this may break in the future. + +## Installation + +Download [latest release](https://github.com/google/go-containerregistry/releases/latest). + +Install manually: + +``` +go install github.com/google/go-containerregistry/cmd/gcrane@latest +``` + +## Commands + +### ls + +`gcrane ls` exposes a more complex form of `ls` than `crane`, which allows for +listing tags, manifests, and sub-repositories. + +### cp + +`gcrane cp` supports a `-r` flag that copies images recursively, which is useful +for backing up images, georeplicating images, or renaming images en masse. + +### gc + +`gcrane gc` will calculate images that can be garbage-collected. +By default, it will print any images that do not have tags pointing to them. + +This can be composed with `gcrane delete` to actually garbage collect them: +```shell +gcrane gc gcr.io/${PROJECT_ID}/repo | xargs -n1 gcrane delete +``` + +## Images + +You can also use gcrane as docker image + +```sh +$ docker run --rm gcr.io/go-containerregistry/gcrane ls gcr.io/google-containers/busybox +gcr.io/google-containers/busybox@sha256:4bdd623e848417d96127e16037743f0cd8b528c026e9175e22a84f639eca58ff +gcr.io/google-containers/busybox:1.24 +gcr.io/google-containers/busybox@sha256:545e6a6310a27636260920bc07b994a299b6708a1b26910cfefd335fdfb60d2b +gcr.io/google-containers/busybox:1.27 +gcr.io/google-containers/busybox:1.27.2 +gcr.io/google-containers/busybox@sha256:d8d3bc2c183ed2f9f10e7258f84971202325ee6011ba137112e01e30f206de67 +gcr.io/google-containers/busybox:latest +``` + +And it's also available with a shell, at the `:debug` tag: + +```sh +docker run --rm -it --entrypoint "/busybox/sh" gcr.io/go-containerregistry/gcrane:debug +``` + +Tagged debug images are available at `gcr.io/go-containerregistry/gcrane/debug:[tag]`. diff --git a/cmd/gcrane/cmd/copy.go b/cmd/gcrane/cmd/copy.go new file mode 100644 index 0000000..5ec4224 --- /dev/null +++ b/cmd/gcrane/cmd/copy.go @@ -0,0 +1,47 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "runtime" + + "github.com/google/go-containerregistry/pkg/gcrane" + "github.com/spf13/cobra" +) + +// NewCmdCopy creates a new cobra.Command for the copy subcommand. +func NewCmdCopy() *cobra.Command { + recursive := false + jobs := 1 + cmd := &cobra.Command{ + Use: "copy SRC DST", + Aliases: []string{"cp"}, + Short: "Efficiently copy a remote image from src to dst", + Args: cobra.ExactArgs(2), + RunE: func(cc *cobra.Command, args []string) error { + src, dst := args[0], args[1] + ctx := cc.Context() + if recursive { + return gcrane.CopyRepository(ctx, src, dst, gcrane.WithJobs(jobs), gcrane.WithUserAgent(userAgent()), gcrane.WithContext(ctx)) + } + return gcrane.Copy(src, dst, gcrane.WithUserAgent(userAgent()), gcrane.WithContext(ctx)) + }, + } + + cmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Whether to recurse through repos") + cmd.Flags().IntVarP(&jobs, "jobs", "j", runtime.GOMAXPROCS(0), "The maximum number of concurrent copies") + + return cmd +} diff --git a/cmd/gcrane/cmd/gc.go b/cmd/gcrane/cmd/gc.go new file mode 100644 index 0000000..b377be4 --- /dev/null +++ b/cmd/gcrane/cmd/gc.go @@ -0,0 +1,76 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + + "github.com/google/go-containerregistry/pkg/gcrane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/spf13/cobra" +) + +// NewCmdGc creates a new cobra.Command for the gc subcommand. +func NewCmdGc() *cobra.Command { + recursive := false + cmd := &cobra.Command{ + Use: "gc", + Short: "List images that are not tagged", + Args: cobra.ExactArgs(1), + RunE: func(cc *cobra.Command, args []string) error { + return gc(cc.Context(), args[0], recursive) + }, + } + + cmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Whether to recurse through repos") + + return cmd +} + +func gc(ctx context.Context, root string, recursive bool) error { + repo, err := name.NewRepository(root) + if err != nil { + return err + } + + opts := []google.Option{ + google.WithAuthFromKeychain(gcrane.Keychain), + google.WithUserAgent(userAgent()), + google.WithContext(ctx), + } + + if recursive { + return google.Walk(repo, printUntaggedImages, opts...) + } + + tags, err := google.List(repo, opts...) + return printUntaggedImages(repo, tags, err) +} + +func printUntaggedImages(repo name.Repository, tags *google.Tags, err error) error { + if err != nil { + return err + } + + for digest, manifest := range tags.Manifests { + if len(manifest.Tags) == 0 { + fmt.Printf("%s@%s\n", repo, digest) + } + } + + return nil +} diff --git a/cmd/gcrane/cmd/list.go b/cmd/gcrane/cmd/list.go new file mode 100644 index 0000000..892f2af --- /dev/null +++ b/cmd/gcrane/cmd/list.go @@ -0,0 +1,121 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "path" + + "github.com/google/go-containerregistry/cmd/crane/cmd" + "github.com/google/go-containerregistry/pkg/gcrane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/spf13/cobra" +) + +func userAgent() string { + if cmd.Version != "" { + return path.Join("gcrane", cmd.Version) + } + + return "gcrane" +} + +// NewCmdList creates a new cobra.Command for the ls subcommand. +func NewCmdList() *cobra.Command { + recursive := false + json := false + cmd := &cobra.Command{ + Use: "ls REPO", + Short: "List the contents of a repo", + Args: cobra.ExactArgs(1), + RunE: func(cc *cobra.Command, args []string) error { + return ls(cc.Context(), args[0], recursive, json) + }, + } + + cmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Whether to recurse through repos") + cmd.Flags().BoolVar(&json, "json", false, "Format the response from the registry as JSON, one line per repo") + + return cmd +} + +func ls(ctx context.Context, root string, recursive, j bool) error { + repo, err := name.NewRepository(root) + if err != nil { + return err + } + + opts := []google.Option{ + google.WithAuthFromKeychain(gcrane.Keychain), + google.WithUserAgent(userAgent()), + google.WithContext(ctx), + } + + if recursive { + return google.Walk(repo, printImages(j), opts...) + } + + tags, err := google.List(repo, opts...) + if err != nil { + return err + } + + if !j { + if len(tags.Manifests) == 0 && len(tags.Children) == 0 { + // If we didn't see any GCR extensions, just list the tags like normal. + for _, tag := range tags.Tags { + fmt.Printf("%s:%s\n", repo, tag) + } + return nil + } + + // Since we're not recursing, print the subdirectories too. + for _, child := range tags.Children { + fmt.Printf("%s/%s\n", repo, child) + } + } + + return printImages(j)(repo, tags, err) +} + +func printImages(j bool) google.WalkFunc { + return func(repo name.Repository, tags *google.Tags, err error) error { + if err != nil { + return err + } + + if j { + b, err := json.Marshal(tags) + if err != nil { + return err + } + fmt.Printf("%s\n", b) + return nil + } + + for digest, manifest := range tags.Manifests { + fmt.Printf("%s@%s\n", repo, digest) + + for _, tag := range manifest.Tags { + fmt.Printf("%s:%s\n", repo, tag) + } + } + + return nil + } +} diff --git a/cmd/gcrane/depcheck_test.go b/cmd/gcrane/depcheck_test.go new file mode 100644 index 0000000..83749b3 --- /dev/null +++ b/cmd/gcrane/depcheck_test.go @@ -0,0 +1,32 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/go-containerregistry/internal/depcheck" +) + +func TestDeps(t *testing.T) { + if testing.Short() { + t.Skip("skipping slow depcheck") + } + depcheck.AssertNoDependency(t, map[string][]string{ + "github.com/google/go-containerregistry/cmd/gcrane": { + "github.com/google/go-containerregistry/pkg/v1/daemon", + }, + }) +} diff --git a/cmd/gcrane/main.go b/cmd/gcrane/main.go new file mode 100644 index 0000000..98871e7 --- /dev/null +++ b/cmd/gcrane/main.go @@ -0,0 +1,72 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "os/signal" + + "github.com/google/go-containerregistry/cmd/crane/cmd" + gcmd "github.com/google/go-containerregistry/cmd/gcrane/cmd" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/gcrane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/spf13/cobra" +) + +func init() { + logs.Warn.SetOutput(os.Stderr) + logs.Progress.SetOutput(os.Stderr) +} + +const ( + use = "gcrane" + short = "gcrane is a tool for managing container images on gcr.io and pkg.dev" +) + +func main() { + options := []crane.Option{crane.WithAuthFromKeychain(gcrane.Keychain)} + // Same as crane, but override usage and keychain. + root := cmd.New(use, short, options) + + // Add or override commands. + gcraneCmds := []*cobra.Command{gcmd.NewCmdList(), gcmd.NewCmdGc(), gcmd.NewCmdCopy(), cmd.NewCmdAuth(options, "gcrane", "auth")} + + // Maintain a map of google-specific commands that we "override". + used := make(map[string]bool) + for _, cmd := range gcraneCmds { + used[cmd.Use] = true + } + + // Remove those from crane's set of commands. + for _, cmd := range root.Commands() { + if _, ok := used[cmd.Use]; ok { + root.RemoveCommand(cmd) + } + } + + // Add our own. + for _, cmd := range gcraneCmds { + root.AddCommand(cmd) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if err := root.ExecuteContext(ctx); err != nil { + cancel() + os.Exit(1) + } +} diff --git a/cmd/ko/README.md b/cmd/ko/README.md new file mode 100644 index 0000000..7f3627e --- /dev/null +++ b/cmd/ko/README.md @@ -0,0 +1,3 @@ +# `ko` has moved + +Please find `ko` at its new home, https://github.com/google/ko diff --git a/cmd/krane/README.md b/cmd/krane/README.md new file mode 100644 index 0000000..2f21e00 --- /dev/null +++ b/cmd/krane/README.md @@ -0,0 +1,15 @@ +# `krane` + + + +This tool is a variant of the [`crane`](../crane/README.md) command, but builds in +support for authenticating against registries using common credential helpers +that find credentials from the environment. + +In particular this tool supports authenticating with common "workload identity" +mechanisms on platforms such as GKE and EKS. + +This additional keychain logic only kicks in if alternative authentication +mechanisms have NOT been configured and `crane` would otherwise perform the +command without credentials, so **it is a drop-in replacement for `crane` that +adds support for authenticating with cloud workload identity mechanisms**. diff --git a/cmd/krane/go.mod b/cmd/krane/go.mod new file mode 100644 index 0000000..3ca44a8 --- /dev/null +++ b/cmd/krane/go.mod @@ -0,0 +1,67 @@ +module github.com/google/go-containerregistry/cmd/krane + +go 1.18 + +replace github.com/google/go-containerregistry => ../../ + +require ( + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1 + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 + github.com/google/go-containerregistry v0.13.0 +) + +require ( + cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/docker/cli v23.0.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v23.0.1+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vbatts/tar-split v0.11.2 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.29.0 // indirect + gotest.tools/v3 v3.1.0 // indirect +) diff --git a/cmd/krane/go.sum b/cmd/krane/go.sum new file mode 100644 index 0000000..23db7e4 --- /dev/null +++ b/cmd/krane/go.sum @@ -0,0 +1,199 @@ +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= +github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15 h1:0rZQIi6deJFjOEgHI9HI2eZcLPPEGQPictX66oRFLL8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 h1:Kbiv9PGnQfG/imNI4L/heyUXvzKmcWSBeDvkrQz5pFc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5 h1:tGA4ZoAsrYhGBypKAo2jwoX/Z5ponBZOTEUMNN/rHP4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5/go.mod h1:cDZh+PHP8Adt9E0zfZT9cK4qadbtIuU/czLpEJtm4wc= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4 h1:6OBVD6KE4gLReaNfG7CSXFvNIVqKIqrywRcG1kUKr4M= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4/go.mod h1:gUxgbzXs+gHsj/6al9dzzoByeSrEl03Oj4iJBu/m/Rk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 h1:QoOybhwRfciWUBbZ0gp9S7XaDnCuSTeK/fySB99V1ls= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 h1:qJdM48OOLl1FBSzI7ZrA1ZfLwOyCYqkXV5lko1hYDBw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 h1:YRkWXQveFb0tFC0TLktmmhGsOcCgLwvq88MC2al47AA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 h1:L1600eLr0YvTT7gNh3Ni24yGI7NSHkq9Gp62vijPRCs= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1 h1:uQhxQriOPUu/knXSPM7D/VyS3GMz+4wsE43eB8f9ojg= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1/go.mod h1:/JmJjW2NJpzRSI3pOxQPC6eOD/tR8SfOA9X1FurmzXI= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= +gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= diff --git a/cmd/krane/main.go b/cmd/krane/main.go new file mode 100644 index 0000000..6912463 --- /dev/null +++ b/cmd/krane/main.go @@ -0,0 +1,67 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "io" + "os" + "os/signal" + + ecr "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" + "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper" + "github.com/google/go-containerregistry/cmd/crane/cmd" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/authn/github" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/v1/google" +) + +var ( + amazonKeychain authn.Keychain = authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard))) + azureKeychain authn.Keychain = authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper()) +) + +func init() { + logs.Warn.SetOutput(os.Stderr) + logs.Progress.SetOutput(os.Stderr) +} + +const ( + use = "krane" + short = "krane is a tool for managing container images" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + keychain := authn.NewMultiKeychain( + authn.DefaultKeychain, + google.Keychain, + github.Keychain, + amazonKeychain, + azureKeychain, + ) + + // Same as crane, but override usage and keychain. + root := cmd.New(use, short, []crane.Option{crane.WithAuthFromKeychain(keychain)}) + + if err := root.ExecuteContext(ctx); err != nil { + cancel() + os.Exit(1) + } +} diff --git a/cmd/registry/main.go b/cmd/registry/main.go new file mode 100644 index 0000000..58a0e4e --- /dev/null +++ b/cmd/registry/main.go @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/http" + "time" + + "github.com/google/go-containerregistry/pkg/registry" +) + +var port = flag.Int("port", 1338, "port to run registry on") + +func main() { + flag.Parse() + + listener, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatal(err) + } + porti := listener.Addr().(*net.TCPAddr).Port + log.Printf("serving on port %d", porti) + s := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, // prevent slowloris, quiet linter + Handler: registry.New(), + } + log.Fatal(s.Serve(listener)) +} diff --git a/cmd/registry/test.sh b/cmd/registry/test.sh new file mode 100755 index 0000000..d0c9507 --- /dev/null +++ b/cmd/registry/test.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -ex + +CONTAINER_OS=$(docker info -f '{{ .OSType }}') + +# crane can run on a Windows system, but doesn't currently support pulling Windows +# containers, so this test can only run if Docker is in Linux container mode. +if [[ ${CONTAINER_OS} = "windows" ]]; then + set +x + echo [TEST SKIPPED] Windows containers are not yet supported by crane + exit +fi + +function cleanup { + [[ -n $PID ]] && kill $PID + [[ -n $CTR ]] && docker stop $CTR + rm -f ubuntu.tar debiand.tar debianc.tar + docker rmi -f \ + localhost:1338/debianc:latest \ + localhost:1338/debiand:latest \ + localhost:1338/ubuntuc:foo \ + localhost:1338/ubuntud:latest \ + || true +} +trap cleanup EXIT + +case "$OSTYPE" in + # On Windows, Docker runs in a VM, so a registry running on the Windows + # host is not accessible via localhost for `docker pull|push`. + win*|msys*|cygwin*) + docker run -d --rm -p 1338:5000 --name test-reg registry:2 + CTR=test-reg + ;; + + *) + registry & + PID=$! + ;; +esac + +go install ./cmd/registry +go install ./cmd/crane + + +crane pull debian:latest debianc.tar +crane push debianc.tar localhost:1338/debianc:latest +docker pull localhost:1338/debianc:latest +docker tag localhost:1338/debianc:latest localhost:1338/debiand:latest +docker push localhost:1338/debiand:latest +crane pull localhost:1338/debiand:latest debiand.tar + +docker pull ubuntu:latest +docker tag ubuntu:latest localhost:1338/ubuntud:latest +docker push localhost:1338/ubuntud:latest +crane pull localhost:1338/ubuntud:latest ubuntu.tar +crane push ubuntu.tar localhost:1338/ubuntuc:foo +docker pull localhost:1338/ubuntuc:foo diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0c905fd --- /dev/null +++ b/go.mod @@ -0,0 +1,49 @@ +module github.com/google/go-containerregistry + +go 1.18 + +require ( + github.com/containerd/stargz-snapshotter/estargz v0.14.3 + github.com/docker/cli v23.0.1+incompatible + github.com/docker/distribution v2.8.1+incompatible + github.com/docker/docker v23.0.1+incompatible + github.com/google/go-cmp v0.5.9 + github.com/klauspost/compress v1.16.0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/spf13/cobra v1.6.1 + golang.org/x/oauth2 v0.6.0 + golang.org/x/sync v0.1.0 + golang.org/x/tools v0.7.0 +) + +require ( + cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vbatts/tar-split v0.11.2 // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.29.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.0.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9cc1ef9 --- /dev/null +++ b/go.sum @@ -0,0 +1,153 @@ +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/hack/boilerplate/boilerplate.go.txt b/hack/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000..a237f5e --- /dev/null +++ b/hack/boilerplate/boilerplate.go.txt @@ -0,0 +1,13 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/hack/bump-deps.sh b/hack/bump-deps.sh new file mode 100755 index 0000000..0e7325d --- /dev/null +++ b/hack/bump-deps.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +pushd ${PROJECT_ROOT} +trap popd EXIT + +go get -u ./... +go mod tidy -compat=1.18 +go mod vendor + +cd ${PROJECT_ROOT}/pkg/authn/k8schain +go get -u ./... +go mod tidy -compat=1.18 +go mod download + +cd ${PROJECT_ROOT}/pkg/authn/kubernetes +go get -u ./... +go mod tidy -compat=1.18 +go mod download + +cd ${PROJECT_ROOT}/cmd/krane +go get -u ./... +go mod tidy -compat=1.18 +go mod download + +cd ${PROJECT_ROOT} + +./hack/update-deps.sh diff --git a/hack/presubmit.sh b/hack/presubmit.sh new file mode 100755 index 0000000..25afe44 --- /dev/null +++ b/hack/presubmit.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# We can't install in the current directory without changing the current module. +TMP_DIR="$(mktemp -d)" +export PATH="${PATH}:${TMP_DIR}/bin" +export GOPATH="${TMP_DIR}" +pushd ${TMP_DIR} +trap popd EXIT +go install honnef.co/go/tools/cmd/staticcheck@v0.3.3 +popd + +pushd ${PROJECT_ROOT} +trap popd EXIT + +staticcheck ./pkg/... + +# Verify that all source files are correctly formatted. +find . -name "*.go" | grep -v vendor/ | xargs gofmt -d -e -l + +# Verify that generated crane docs are up-to-date. +mkdir -p /tmp/gendoc && go run cmd/crane/help/main.go --dir /tmp/gendoc && diff -Naur /tmp/gendoc/ cmd/crane/doc/ + +go test ./... +./pkg/name/internal/must_test.sh + +./cmd/crane/rebase_test.sh + +pushd ${PROJECT_ROOT}/cmd/krane +trap popd EXIT +go build ./... + +pushd ${PROJECT_ROOT}/pkg/authn/k8schain +trap popd EXIT +go build ./... + +pushd ${PROJECT_ROOT}/pkg/authn/kubernetes +trap popd EXIT +go test ./... diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh new file mode 100755 index 0000000..e237a50 --- /dev/null +++ b/hack/update-codegen.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BOILER_PLATE_FILE="${PROJECT_ROOT}/hack/boilerplate/boilerplate.go.txt" +MODULE_NAME=github.com/google/go-containerregistry + +pushd ${PROJECT_ROOT} +trap popd EXIT + +export GOPATH=$(go env GOPATH) +export PATH="${GOPATH}/bin:${PATH}" + +go mod tidy +go mod vendor + +export GOBIN=$(mktemp -d) +export PATH="$GOBIN:$PATH" + +go install github.com/maxbrunsfeld/counterfeiter/v6@latest +go install k8s.io/code-generator/cmd/deepcopy-gen@v0.20.7 + +counterfeiter -o pkg/v1/fake/index.go ${PROJECT_ROOT}/pkg/v1 ImageIndex +counterfeiter -o pkg/v1/fake/image.go ${PROJECT_ROOT}/pkg/v1 Image + +DEEPCOPY_OUTPUT=$(mktemp -d) + +deepcopy-gen -O zz_deepcopy_generated --go-header-file $BOILER_PLATE_FILE \ + --input-dirs "$MODULE_NAME/pkg/v1" \ + --output-base "$DEEPCOPY_OUTPUT" + +# TODO - Generalize this for all directories when we need it +cp $DEEPCOPY_OUTPUT/$MODULE_NAME/pkg/v1/*.go $PROJECT_ROOT/pkg/v1 + +go run $PROJECT_ROOT/cmd/crane/help/main.go --dir=$PROJECT_ROOT/cmd/crane/doc/ diff --git a/hack/update-deps.sh b/hack/update-deps.sh new file mode 100755 index 0000000..25be810 --- /dev/null +++ b/hack/update-deps.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +pushd ${PROJECT_ROOT} +trap popd EXIT + +go mod tidy +go mod vendor + +# Delete all vendored broken symlinks. +# From https://stackoverflow.com/questions/22097130/delete-all-broken-symbolic-links-with-a-line +find vendor/ -type l -exec sh -c 'for x; do [ -e "$x" ] || rm "$x"; done' _ {} + diff --git a/hack/update-dots.sh b/hack/update-dots.sh new file mode 100755 index 0000000..570c794 --- /dev/null +++ b/hack/update-dots.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2019 The original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +pushd ${PROJECT_ROOT} +trap popd EXIT + +dot -Tjpeg images/ociimage.gv > images/ociimage.jpeg + +for f in $(ls images/dot/ | grep -e '.dot$'); do + dot -Tsvg images/dot/$f > images/$f.svg +done diff --git a/images/containerd.dot.svg b/images/containerd.dot.svg new file mode 100644 index 0000000..cb87da6 --- /dev/null +++ b/images/containerd.dot.svg @@ -0,0 +1,2074 @@ + + + + + + +godep + + + +bufio + + +bufio + + + + + +bytes + + +bytes + + + + + +compress/gzip + + +compress/gzip + + + + + +container/list + + +container/list + + + + + +context + + +context + + + + + +crypto + + +crypto + + + + + +encoding + + +encoding + + + + + +encoding/base64 + + +encoding/base64 + + + + + +encoding/json + + +encoding/json + + + + + +errors + + +errors + + + + + +fmt + + +fmt + + + + + +github.com/containerd/containerd/archive/compression + + +github.com/containerd/containerd/archive/compression + + + + + +github.com/containerd/containerd/archive/compression->bufio + + + + + +github.com/containerd/containerd/archive/compression->bytes + + + + + +github.com/containerd/containerd/archive/compression->compress/gzip + + + + + +github.com/containerd/containerd/archive/compression->context + + + + + +github.com/containerd/containerd/archive/compression->fmt + + + + + +github.com/containerd/containerd/log + + +github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/archive/compression->github.com/containerd/containerd/log + + + + + +io + + +io + + + + + +github.com/containerd/containerd/archive/compression->io + + + + + +os + + +os + + + + + +github.com/containerd/containerd/archive/compression->os + + + + + +os/exec + + +os/exec + + + + + +github.com/containerd/containerd/archive/compression->os/exec + + + + + +strconv + + +strconv + + + + + +github.com/containerd/containerd/archive/compression->strconv + + + + + +sync + + +sync + + + + + +github.com/containerd/containerd/archive/compression->sync + + + + + +github.com/containerd/containerd/log->context + + + + + +github.com/sirupsen/logrus + + +github.com/sirupsen/logrus + + + + + +github.com/containerd/containerd/log->github.com/sirupsen/logrus + + + + + +sync/atomic + + +sync/atomic + + + + + +github.com/containerd/containerd/log->sync/atomic + + + + + +github.com/containerd/containerd/content + + +github.com/containerd/containerd/content + + + + + +github.com/containerd/containerd/content->context + + + + + +github.com/containerd/containerd/content->io + + + + + +github.com/containerd/containerd/content->sync + + + + + +github.com/containerd/containerd/errdefs + + +github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/content->github.com/containerd/containerd/errdefs + + + + + +github.com/opencontainers/go-digest + + +github.com/opencontainers/go-digest + + + + + +github.com/containerd/containerd/content->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1 + + +github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/content->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/pkg/errors + + +github.com/pkg/errors + + + + + +github.com/containerd/containerd/content->github.com/pkg/errors + + + + + +io/ioutil + + +io/ioutil + + + + + +github.com/containerd/containerd/content->io/ioutil + + + + + +math/rand + + +math/rand + + + + + +github.com/containerd/containerd/content->math/rand + + + + + +time + + +time + + + + + +github.com/containerd/containerd/content->time + + + + + +github.com/containerd/containerd/errdefs->context + + + + + +github.com/containerd/containerd/errdefs->github.com/pkg/errors + + + + + +google.golang.org/grpc/codes + + +google.golang.org/grpc/codes + + + + + +github.com/containerd/containerd/errdefs->google.golang.org/grpc/codes + + + + + +google.golang.org/grpc/status + + +google.golang.org/grpc/status + + + + + +github.com/containerd/containerd/errdefs->google.golang.org/grpc/status + + + + + +strings + + +strings + + + + + +github.com/containerd/containerd/errdefs->strings + + + + + +github.com/opencontainers/go-digest->crypto + + + + + +github.com/opencontainers/go-digest->fmt + + + + + +github.com/opencontainers/go-digest->io + + + + + +github.com/opencontainers/go-digest->strings + + + + + +regexp + + +regexp + + + + + +github.com/opencontainers/go-digest->regexp + + + + + +hash + + +hash + + + + + +github.com/opencontainers/go-digest->hash + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1->time + + + + + +github.com/opencontainers/image-spec/specs-go + + +github.com/opencontainers/image-spec/specs-go + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/image-spec/specs-go + + + + + +github.com/pkg/errors->fmt + + + + + +github.com/pkg/errors->io + + + + + +github.com/pkg/errors->strings + + + + + +runtime + + +runtime + + + + + +github.com/pkg/errors->runtime + + + + + +path + + +path + + + + + +github.com/pkg/errors->path + + + + + +google.golang.org/grpc/codes->fmt + + + + + +google.golang.org/grpc/codes->strconv + + + + + +google.golang.org/grpc/status->context + + + + + +google.golang.org/grpc/status->errors + + + + + +google.golang.org/grpc/status->fmt + + + + + +google.golang.org/grpc/status->google.golang.org/grpc/codes + + + + + +github.com/golang/protobuf/proto + + +github.com/golang/protobuf/proto + + + + + +google.golang.org/grpc/status->github.com/golang/protobuf/proto + + + + + +github.com/golang/protobuf/ptypes + + +github.com/golang/protobuf/ptypes + + + + + +google.golang.org/grpc/status->github.com/golang/protobuf/ptypes + + + + + +google.golang.org/genproto/googleapis/rpc/status + + +google.golang.org/genproto/googleapis/rpc/status + + + + + +google.golang.org/grpc/status->google.golang.org/genproto/googleapis/rpc/status + + + + + +google.golang.org/grpc/internal + + +google.golang.org/grpc/internal + + + + + +google.golang.org/grpc/status->google.golang.org/grpc/internal + + + + + +github.com/containerd/containerd/images + + +github.com/containerd/containerd/images + + + + + +github.com/containerd/containerd/images->context + + + + + +github.com/containerd/containerd/images->encoding/json + + + + + +github.com/containerd/containerd/images->fmt + + + + + +github.com/containerd/containerd/images->github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/images->io + + + + + +github.com/containerd/containerd/images->github.com/containerd/containerd/content + + + + + +github.com/containerd/containerd/images->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/images->github.com/opencontainers/go-digest + + + + + +github.com/containerd/containerd/images->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/images->github.com/pkg/errors + + + + + +github.com/containerd/containerd/images->time + + + + + +github.com/containerd/containerd/images->strings + + + + + +github.com/containerd/containerd/platforms + + +github.com/containerd/containerd/platforms + + + + + +github.com/containerd/containerd/images->github.com/containerd/containerd/platforms + + + + + +golang.org/x/sync/errgroup + + +golang.org/x/sync/errgroup + + + + + +github.com/containerd/containerd/images->golang.org/x/sync/errgroup + + + + + +golang.org/x/sync/semaphore + + +golang.org/x/sync/semaphore + + + + + +github.com/containerd/containerd/images->golang.org/x/sync/semaphore + + + + + +sort + + +sort + + + + + +github.com/containerd/containerd/images->sort + + + + + +github.com/containerd/containerd/platforms->bufio + + + + + +github.com/containerd/containerd/platforms->github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/platforms->os + + + + + +github.com/containerd/containerd/platforms->strconv + + + + + +github.com/containerd/containerd/platforms->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/platforms->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/platforms->github.com/pkg/errors + + + + + +github.com/containerd/containerd/platforms->strings + + + + + +github.com/containerd/containerd/platforms->regexp + + + + + +github.com/containerd/containerd/platforms->runtime + + + + + +golang.org/x/sync/errgroup->context + + + + + +golang.org/x/sync/errgroup->sync + + + + + +golang.org/x/sync/semaphore->container/list + + + + + +golang.org/x/sync/semaphore->context + + + + + +golang.org/x/sync/semaphore->sync + + + + + +github.com/containerd/containerd/labels + + +github.com/containerd/containerd/labels + + + + + +github.com/containerd/containerd/labels->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/labels->github.com/pkg/errors + + + + + +github.com/sirupsen/logrus->bufio + + + + + +github.com/sirupsen/logrus->bytes + + + + + +github.com/sirupsen/logrus->context + + + + + +github.com/sirupsen/logrus->encoding/json + + + + + +github.com/sirupsen/logrus->fmt + + + + + +github.com/sirupsen/logrus->io + + + + + +github.com/sirupsen/logrus->os + + + + + +github.com/sirupsen/logrus->sync + + + + + +github.com/sirupsen/logrus->time + + + + + +github.com/sirupsen/logrus->strings + + + + + +github.com/sirupsen/logrus->sort + + + + + +github.com/sirupsen/logrus->sync/atomic + + + + + +github.com/sirupsen/logrus->runtime + + + + + +log + + +log + + + + + +github.com/sirupsen/logrus->log + + + + + +reflect + + +reflect + + + + + +github.com/sirupsen/logrus->reflect + + + + + +golang.org/x/sys/unix + + +golang.org/x/sys/unix + + + + + +github.com/sirupsen/logrus->golang.org/x/sys/unix + + + + + +github.com/containerd/containerd/reference + + +github.com/containerd/containerd/reference + + + + + +github.com/containerd/containerd/reference->errors + + + + + +github.com/containerd/containerd/reference->fmt + + + + + +github.com/containerd/containerd/reference->github.com/opencontainers/go-digest + + + + + +github.com/containerd/containerd/reference->strings + + + + + +github.com/containerd/containerd/reference->regexp + + + + + +net/url + + +net/url + + + + + +github.com/containerd/containerd/reference->net/url + + + + + +github.com/containerd/containerd/reference->path + + + + + +github.com/containerd/containerd/remotes + + +github.com/containerd/containerd/remotes + + + + + +github.com/containerd/containerd/remotes->context + + + + + +github.com/containerd/containerd/remotes->fmt + + + + + +github.com/containerd/containerd/remotes->github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/remotes->io + + + + + +github.com/containerd/containerd/remotes->sync + + + + + +github.com/containerd/containerd/remotes->github.com/containerd/containerd/content + + + + + +github.com/containerd/containerd/remotes->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/remotes->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/remotes->github.com/pkg/errors + + + + + +github.com/containerd/containerd/remotes->strings + + + + + +github.com/containerd/containerd/remotes->github.com/containerd/containerd/images + + + + + +github.com/containerd/containerd/remotes->github.com/containerd/containerd/platforms + + + + + +github.com/containerd/containerd/remotes->github.com/sirupsen/logrus + + + + + +github.com/containerd/containerd/remotes/docker + + +github.com/containerd/containerd/remotes/docker + + + + + +github.com/containerd/containerd/remotes/docker->bytes + + + + + +github.com/containerd/containerd/remotes/docker->context + + + + + +github.com/containerd/containerd/remotes/docker->encoding/base64 + + + + + +github.com/containerd/containerd/remotes/docker->encoding/json + + + + + +github.com/containerd/containerd/remotes/docker->fmt + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/remotes/docker->io + + + + + +github.com/containerd/containerd/remotes/docker->sync + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/content + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/remotes/docker->github.com/opencontainers/go-digest + + + + + +github.com/containerd/containerd/remotes/docker->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/remotes/docker->github.com/pkg/errors + + + + + +github.com/containerd/containerd/remotes/docker->io/ioutil + + + + + +github.com/containerd/containerd/remotes/docker->time + + + + + +github.com/containerd/containerd/remotes/docker->strings + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/images + + + + + +github.com/containerd/containerd/remotes/docker->sort + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/labels + + + + + +github.com/containerd/containerd/remotes/docker->github.com/sirupsen/logrus + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/reference + + + + + +github.com/containerd/containerd/remotes/docker->net/url + + + + + +github.com/containerd/containerd/remotes/docker->path + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/remotes + + + + + +github.com/containerd/containerd/remotes/docker/schema1 + + +github.com/containerd/containerd/remotes/docker/schema1 + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/remotes/docker/schema1 + + + + + +github.com/containerd/containerd/version + + +github.com/containerd/containerd/version + + + + + +github.com/containerd/containerd/remotes/docker->github.com/containerd/containerd/version + + + + + +github.com/docker/distribution/registry/api/errcode + + +github.com/docker/distribution/registry/api/errcode + + + + + +github.com/containerd/containerd/remotes/docker->github.com/docker/distribution/registry/api/errcode + + + + + +golang.org/x/net/context/ctxhttp + + +golang.org/x/net/context/ctxhttp + + + + + +github.com/containerd/containerd/remotes/docker->golang.org/x/net/context/ctxhttp + + + + + +net/http + + +net/http + + + + + +github.com/containerd/containerd/remotes/docker->net/http + + + + + +github.com/containerd/containerd/remotes/docker/schema1->bytes + + + + + +github.com/containerd/containerd/remotes/docker/schema1->context + + + + + +github.com/containerd/containerd/remotes/docker/schema1->encoding/base64 + + + + + +github.com/containerd/containerd/remotes/docker/schema1->encoding/json + + + + + +github.com/containerd/containerd/remotes/docker/schema1->fmt + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/archive/compression + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/log + + + + + +github.com/containerd/containerd/remotes/docker/schema1->io + + + + + +github.com/containerd/containerd/remotes/docker/schema1->strconv + + + + + +github.com/containerd/containerd/remotes/docker/schema1->sync + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/content + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/errdefs + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/opencontainers/go-digest + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/pkg/errors + + + + + +github.com/containerd/containerd/remotes/docker/schema1->io/ioutil + + + + + +github.com/containerd/containerd/remotes/docker/schema1->time + + + + + +github.com/containerd/containerd/remotes/docker/schema1->strings + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/images + + + + + +github.com/containerd/containerd/remotes/docker/schema1->golang.org/x/sync/errgroup + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/containerd/containerd/remotes + + + + + +github.com/containerd/containerd/remotes/docker/schema1->github.com/opencontainers/image-spec/specs-go + + + + + +github.com/docker/distribution/registry/api/errcode->encoding/json + + + + + +github.com/docker/distribution/registry/api/errcode->fmt + + + + + +github.com/docker/distribution/registry/api/errcode->sync + + + + + +github.com/docker/distribution/registry/api/errcode->strings + + + + + +github.com/docker/distribution/registry/api/errcode->sort + + + + + +github.com/docker/distribution/registry/api/errcode->net/http + + + + + +golang.org/x/net/context/ctxhttp->context + + + + + +golang.org/x/net/context/ctxhttp->io + + + + + +golang.org/x/net/context/ctxhttp->strings + + + + + +golang.org/x/net/context/ctxhttp->net/url + + + + + +golang.org/x/net/context/ctxhttp->net/http + + + + + +github.com/opencontainers/image-spec/specs-go->fmt + + + + + +github.com/golang/protobuf/proto->bufio + + + + + +github.com/golang/protobuf/proto->bytes + + + + + +github.com/golang/protobuf/proto->encoding + + + + + +github.com/golang/protobuf/proto->encoding/json + + + + + +github.com/golang/protobuf/proto->errors + + + + + +github.com/golang/protobuf/proto->fmt + + + + + +github.com/golang/protobuf/proto->io + + + + + +github.com/golang/protobuf/proto->os + + + + + +github.com/golang/protobuf/proto->strconv + + + + + +github.com/golang/protobuf/proto->sync + + + + + +github.com/golang/protobuf/proto->strings + + + + + +github.com/golang/protobuf/proto->sort + + + + + +github.com/golang/protobuf/proto->sync/atomic + + + + + +github.com/golang/protobuf/proto->log + + + + + +math + + +math + + + + + +github.com/golang/protobuf/proto->math + + + + + +github.com/golang/protobuf/proto->reflect + + + + + +unicode/utf8 + + +unicode/utf8 + + + + + +github.com/golang/protobuf/proto->unicode/utf8 + + + + + +unsafe + + +unsafe + + + + + +github.com/golang/protobuf/proto->unsafe + + + + + +github.com/golang/protobuf/ptypes->errors + + + + + +github.com/golang/protobuf/ptypes->fmt + + + + + +github.com/golang/protobuf/ptypes->time + + + + + +github.com/golang/protobuf/ptypes->strings + + + + + +github.com/golang/protobuf/ptypes->github.com/golang/protobuf/proto + + + + + +github.com/golang/protobuf/ptypes->reflect + + + + + +github.com/golang/protobuf/ptypes/any + + +github.com/golang/protobuf/ptypes/any + + + + + +github.com/golang/protobuf/ptypes->github.com/golang/protobuf/ptypes/any + + + + + +github.com/golang/protobuf/ptypes/duration + + +github.com/golang/protobuf/ptypes/duration + + + + + +github.com/golang/protobuf/ptypes->github.com/golang/protobuf/ptypes/duration + + + + + +github.com/golang/protobuf/ptypes/timestamp + + +github.com/golang/protobuf/ptypes/timestamp + + + + + +github.com/golang/protobuf/ptypes->github.com/golang/protobuf/ptypes/timestamp + + + + + +github.com/golang/protobuf/ptypes/any->fmt + + + + + +github.com/golang/protobuf/ptypes/any->github.com/golang/protobuf/proto + + + + + +github.com/golang/protobuf/ptypes/any->math + + + + + +github.com/golang/protobuf/ptypes/duration->fmt + + + + + +github.com/golang/protobuf/ptypes/duration->github.com/golang/protobuf/proto + + + + + +github.com/golang/protobuf/ptypes/duration->math + + + + + +github.com/golang/protobuf/ptypes/timestamp->fmt + + + + + +github.com/golang/protobuf/ptypes/timestamp->github.com/golang/protobuf/proto + + + + + +github.com/golang/protobuf/ptypes/timestamp->math + + + + + +golang.org/x/sys/unix->bytes + + + + + +golang.org/x/sys/unix->sync + + + + + +golang.org/x/sys/unix->time + + + + + +golang.org/x/sys/unix->strings + + + + + +golang.org/x/sys/unix->sort + + + + + +golang.org/x/sys/unix->runtime + + + + + +golang.org/x/sys/unix->unsafe + + + + + +syscall + + +syscall + + + + + +golang.org/x/sys/unix->syscall + + + + + +google.golang.org/genproto/googleapis/rpc/status->fmt + + + + + +google.golang.org/genproto/googleapis/rpc/status->github.com/golang/protobuf/proto + + + + + +google.golang.org/genproto/googleapis/rpc/status->math + + + + + +google.golang.org/genproto/googleapis/rpc/status->github.com/golang/protobuf/ptypes/any + + + + + +google.golang.org/grpc/connectivity + + +google.golang.org/grpc/connectivity + + + + + +google.golang.org/grpc/connectivity->context + + + + + +google.golang.org/grpc/grpclog + + +google.golang.org/grpc/grpclog + + + + + +google.golang.org/grpc/connectivity->google.golang.org/grpc/grpclog + + + + + +google.golang.org/grpc/grpclog->io + + + + + +google.golang.org/grpc/grpclog->os + + + + + +google.golang.org/grpc/grpclog->strconv + + + + + +google.golang.org/grpc/grpclog->io/ioutil + + + + + +google.golang.org/grpc/grpclog->log + + + + + +google.golang.org/grpc/internal->context + + + + + +google.golang.org/grpc/internal->time + + + + + +google.golang.org/grpc/internal->google.golang.org/grpc/connectivity + + + + + diff --git a/images/containers.dot.svg b/images/containers.dot.svg new file mode 100644 index 0000000..38135cf --- /dev/null +++ b/images/containers.dot.svg @@ -0,0 +1,5365 @@ + + + + + + +godep + + + +bufio + + +bufio + + + + + +bytes + + +bytes + + + + + +compress/bzip2 + + +compress/bzip2 + + + + + +compress/gzip + + +compress/gzip + + + + + +context + + +context + + + + + +crypto + + +crypto + + + + + +crypto/ecdsa + + +crypto/ecdsa + + + + + +crypto/elliptic + + +crypto/elliptic + + + + + +crypto/rand + + +crypto/rand + + + + + +crypto/rsa + + +crypto/rsa + + + + + +crypto/sha256 + + +crypto/sha256 + + + + + +crypto/sha512 + + +crypto/sha512 + + + + + +crypto/tls + + +crypto/tls + + + + + +crypto/x509 + + +crypto/x509 + + + + + +crypto/x509/pkix + + +crypto/x509/pkix + + + + + +encoding + + +encoding + + + + + +encoding/base32 + + +encoding/base32 + + + + + +encoding/base64 + + +encoding/base64 + + + + + +encoding/binary + + +encoding/binary + + + + + +encoding/hex + + +encoding/hex + + + + + +encoding/json + + +encoding/json + + + + + +encoding/pem + + +encoding/pem + + + + + +errors + + +errors + + + + + +expvar + + +expvar + + + + + +fmt + + +fmt + + + + + +github.com/BurntSushi/toml + + +github.com/BurntSushi/toml + + + + + +github.com/BurntSushi/toml->bufio + + + + + +github.com/BurntSushi/toml->encoding + + + + + +github.com/BurntSushi/toml->errors + + + + + +github.com/BurntSushi/toml->fmt + + + + + +io + + +io + + + + + +github.com/BurntSushi/toml->io + + + + + +io/ioutil + + +io/ioutil + + + + + +github.com/BurntSushi/toml->io/ioutil + + + + + +math + + +math + + + + + +github.com/BurntSushi/toml->math + + + + + +reflect + + +reflect + + + + + +github.com/BurntSushi/toml->reflect + + + + + +sort + + +sort + + + + + +github.com/BurntSushi/toml->sort + + + + + +strconv + + +strconv + + + + + +github.com/BurntSushi/toml->strconv + + + + + +strings + + +strings + + + + + +github.com/BurntSushi/toml->strings + + + + + +sync + + +sync + + + + + +github.com/BurntSushi/toml->sync + + + + + +time + + +time + + + + + +github.com/BurntSushi/toml->time + + + + + +unicode + + +unicode + + + + + +github.com/BurntSushi/toml->unicode + + + + + +unicode/utf8 + + +unicode/utf8 + + + + + +github.com/BurntSushi/toml->unicode/utf8 + + + + + +github.com/beorn7/perks/quantile + + +github.com/beorn7/perks/quantile + + + + + +github.com/beorn7/perks/quantile->math + + + + + +github.com/beorn7/perks/quantile->sort + + + + + +github.com/cespare/xxhash/v2 + + +github.com/cespare/xxhash/v2 + + + + + +github.com/cespare/xxhash/v2->encoding/binary + + + + + +github.com/cespare/xxhash/v2->errors + + + + + +github.com/cespare/xxhash/v2->reflect + + + + + +math/bits + + +math/bits + + + + + +github.com/cespare/xxhash/v2->math/bits + + + + + +unsafe + + +unsafe + + + + + +github.com/cespare/xxhash/v2->unsafe + + + + + +github.com/containers/image/docker + + +github.com/containers/image/docker + + + + + +github.com/containers/image/docker->bytes + + + + + +github.com/containers/image/docker->context + + + + + +github.com/containers/image/docker->crypto/rand + + + + + +github.com/containers/image/docker->crypto/tls + + + + + +github.com/containers/image/docker->encoding/json + + + + + +github.com/containers/image/docker->errors + + + + + +github.com/containers/image/docker->fmt + + + + + +github.com/containers/image/docker->io + + + + + +github.com/containers/image/docker->io/ioutil + + + + + +github.com/containers/image/docker->strconv + + + + + +github.com/containers/image/docker->strings + + + + + +github.com/containers/image/docker->sync + + + + + +github.com/containers/image/docker->time + + + + + +github.com/containers/image/v5/docker/policyconfiguration + + +github.com/containers/image/v5/docker/policyconfiguration + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/docker/policyconfiguration + + + + + +github.com/containers/image/v5/docker/reference + + +github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/image + + +github.com/containers/image/v5/image + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/image + + + + + +github.com/containers/image/v5/manifest + + +github.com/containers/image/v5/manifest + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/manifest + + + + + +github.com/containers/image/v5/pkg/blobinfocache/none + + +github.com/containers/image/v5/pkg/blobinfocache/none + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/pkg/blobinfocache/none + + + + + +github.com/containers/image/v5/pkg/docker/config + + +github.com/containers/image/v5/pkg/docker/config + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/pkg/docker/config + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2 + + +github.com/containers/image/v5/pkg/sysregistriesv2 + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/pkg/sysregistriesv2 + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig + + +github.com/containers/image/v5/pkg/tlsclientconfig + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/pkg/tlsclientconfig + + + + + +github.com/containers/image/v5/transports + + +github.com/containers/image/v5/transports + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/transports + + + + + +github.com/containers/image/v5/types + + +github.com/containers/image/v5/types + + + + + +github.com/containers/image/docker->github.com/containers/image/v5/types + + + + + +github.com/docker/distribution/registry/api/errcode + + +github.com/docker/distribution/registry/api/errcode + + + + + +github.com/containers/image/docker->github.com/docker/distribution/registry/api/errcode + + + + + +github.com/docker/distribution/registry/api/v2 + + +github.com/docker/distribution/registry/api/v2 + + + + + +github.com/containers/image/docker->github.com/docker/distribution/registry/api/v2 + + + + + +github.com/docker/distribution/registry/client + + +github.com/docker/distribution/registry/client + + + + + +github.com/containers/image/docker->github.com/docker/distribution/registry/client + + + + + +github.com/docker/go-connections/tlsconfig + + +github.com/docker/go-connections/tlsconfig + + + + + +github.com/containers/image/docker->github.com/docker/go-connections/tlsconfig + + + + + +github.com/ghodss/yaml + + +github.com/ghodss/yaml + + + + + +github.com/containers/image/docker->github.com/ghodss/yaml + + + + + +github.com/opencontainers/go-digest + + +github.com/opencontainers/go-digest + + + + + +github.com/containers/image/docker->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1 + + +github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containers/image/docker->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/pkg/errors + + +github.com/pkg/errors + + + + + +github.com/containers/image/docker->github.com/pkg/errors + + + + + +github.com/sirupsen/logrus + + +github.com/sirupsen/logrus + + + + + +github.com/containers/image/docker->github.com/sirupsen/logrus + + + + + +mime + + +mime + + + + + +github.com/containers/image/docker->mime + + + + + +net/http + + +net/http + + + + + +github.com/containers/image/docker->net/http + + + + + +net/url + + +net/url + + + + + +github.com/containers/image/docker->net/url + + + + + +os + + +os + + + + + +github.com/containers/image/docker->os + + + + + +path + + +path + + + + + +github.com/containers/image/docker->path + + + + + +path/filepath + + +path/filepath + + + + + +github.com/containers/image/docker->path/filepath + + + + + +github.com/containers/image/v5/docker/policyconfiguration->strings + + + + + +github.com/containers/image/v5/docker/policyconfiguration->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/docker/policyconfiguration->github.com/pkg/errors + + + + + +github.com/containers/image/v5/docker/reference->errors + + + + + +github.com/containers/image/v5/docker/reference->fmt + + + + + +github.com/containers/image/v5/docker/reference->strings + + + + + +github.com/containers/image/v5/docker/reference->github.com/opencontainers/go-digest + + + + + +github.com/containers/image/v5/docker/reference->path + + + + + +regexp + + +regexp + + + + + +github.com/containers/image/v5/docker/reference->regexp + + + + + +github.com/containers/image/v5/image->bytes + + + + + +github.com/containers/image/v5/image->context + + + + + +github.com/containers/image/v5/image->crypto/sha256 + + + + + +github.com/containers/image/v5/image->encoding/hex + + + + + +github.com/containers/image/v5/image->encoding/json + + + + + +github.com/containers/image/v5/image->fmt + + + + + +github.com/containers/image/v5/image->io/ioutil + + + + + +github.com/containers/image/v5/image->strings + + + + + +github.com/containers/image/v5/image->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/image->github.com/containers/image/v5/manifest + + + + + +github.com/containers/image/v5/image->github.com/containers/image/v5/pkg/blobinfocache/none + + + + + +github.com/containers/image/v5/image->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/image->github.com/opencontainers/go-digest + + + + + +github.com/containers/image/v5/image->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containers/image/v5/image->github.com/pkg/errors + + + + + +github.com/containers/image/v5/image->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/manifest->encoding/json + + + + + +github.com/containers/image/v5/manifest->fmt + + + + + +github.com/containers/image/v5/manifest->strings + + + + + +github.com/containers/image/v5/manifest->time + + + + + +github.com/containers/image/v5/manifest->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/manifest->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/manifest->github.com/opencontainers/go-digest + + + + + +github.com/containers/image/v5/manifest->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containers/image/v5/manifest->github.com/pkg/errors + + + + + +github.com/containers/image/v5/manifest->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/manifest->regexp + + + + + +github.com/containers/image/v5/pkg/compression + + +github.com/containers/image/v5/pkg/compression + + + + + +github.com/containers/image/v5/manifest->github.com/containers/image/v5/pkg/compression + + + + + +github.com/containers/image/v5/pkg/strslice + + +github.com/containers/image/v5/pkg/strslice + + + + + +github.com/containers/image/v5/manifest->github.com/containers/image/v5/pkg/strslice + + + + + +github.com/containers/libtrust + + +github.com/containers/libtrust + + + + + +github.com/containers/image/v5/manifest->github.com/containers/libtrust + + + + + +github.com/containers/ocicrypt/spec + + +github.com/containers/ocicrypt/spec + + + + + +github.com/containers/image/v5/manifest->github.com/containers/ocicrypt/spec + + + + + +github.com/docker/docker/api/types/versions + + +github.com/docker/docker/api/types/versions + + + + + +github.com/containers/image/v5/manifest->github.com/docker/docker/api/types/versions + + + + + +github.com/opencontainers/image-spec/specs-go + + +github.com/opencontainers/image-spec/specs-go + + + + + +github.com/containers/image/v5/manifest->github.com/opencontainers/image-spec/specs-go + + + + + +runtime + + +runtime + + + + + +github.com/containers/image/v5/manifest->runtime + + + + + +github.com/containers/image/v5/pkg/blobinfocache/none->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/pkg/blobinfocache/none->github.com/opencontainers/go-digest + + + + + +github.com/containers/image/v5/pkg/docker/config->encoding/base64 + + + + + +github.com/containers/image/v5/pkg/docker/config->encoding/json + + + + + +github.com/containers/image/v5/pkg/docker/config->fmt + + + + + +github.com/containers/image/v5/pkg/docker/config->io/ioutil + + + + + +github.com/containers/image/v5/pkg/docker/config->strings + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/pkg/errors + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/pkg/docker/config->os + + + + + +github.com/containers/image/v5/pkg/docker/config->path/filepath + + + + + +github.com/containers/image/v5/internal/pkg/keyctl + + +github.com/containers/image/v5/internal/pkg/keyctl + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/containers/image/v5/internal/pkg/keyctl + + + + + +github.com/docker/docker-credential-helpers/client + + +github.com/docker/docker-credential-helpers/client + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/docker/docker-credential-helpers/client + + + + + +github.com/docker/docker-credential-helpers/credentials + + +github.com/docker/docker-credential-helpers/credentials + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/docker/docker-credential-helpers/credentials + + + + + +github.com/docker/docker/pkg/homedir + + +github.com/docker/docker/pkg/homedir + + + + + +github.com/containers/image/v5/pkg/docker/config->github.com/docker/docker/pkg/homedir + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->fmt + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->github.com/BurntSushi/toml + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->io/ioutil + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->strings + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->sync + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->github.com/pkg/errors + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->os + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->path/filepath + + + + + +github.com/containers/image/v5/pkg/sysregistriesv2->regexp + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->crypto/tls + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->io/ioutil + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->strings + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->time + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->github.com/docker/go-connections/tlsconfig + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->github.com/pkg/errors + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->net/http + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->os + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->path/filepath + + + + + +github.com/docker/go-connections/sockets + + +github.com/docker/go-connections/sockets + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->github.com/docker/go-connections/sockets + + + + + +net + + +net + + + + + +github.com/containers/image/v5/pkg/tlsclientconfig->net + + + + + +github.com/containers/image/v5/transports->fmt + + + + + +github.com/containers/image/v5/transports->sort + + + + + +github.com/containers/image/v5/transports->sync + + + + + +github.com/containers/image/v5/transports->github.com/containers/image/v5/types + + + + + +github.com/containers/image/v5/types->context + + + + + +github.com/containers/image/v5/types->io + + + + + +github.com/containers/image/v5/types->time + + + + + +github.com/containers/image/v5/types->github.com/containers/image/v5/docker/reference + + + + + +github.com/containers/image/v5/types->github.com/opencontainers/go-digest + + + + + +github.com/containers/image/v5/types->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/containers/image/v5/pkg/compression/types + + +github.com/containers/image/v5/pkg/compression/types + + + + + +github.com/containers/image/v5/types->github.com/containers/image/v5/pkg/compression/types + + + + + +github.com/docker/distribution/registry/api/errcode->encoding/json + + + + + +github.com/docker/distribution/registry/api/errcode->fmt + + + + + +github.com/docker/distribution/registry/api/errcode->sort + + + + + +github.com/docker/distribution/registry/api/errcode->strings + + + + + +github.com/docker/distribution/registry/api/errcode->sync + + + + + +github.com/docker/distribution/registry/api/errcode->net/http + + + + + +github.com/docker/distribution/registry/api/v2->fmt + + + + + +github.com/docker/distribution/registry/api/v2->strings + + + + + +github.com/docker/distribution/registry/api/v2->unicode + + + + + +github.com/docker/distribution/registry/api/v2->github.com/docker/distribution/registry/api/errcode + + + + + +github.com/docker/distribution/registry/api/v2->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/api/v2->net/http + + + + + +github.com/docker/distribution/registry/api/v2->net/url + + + + + +github.com/docker/distribution/registry/api/v2->regexp + + + + + +github.com/docker/distribution/reference + + +github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/api/v2->github.com/docker/distribution/reference + + + + + +github.com/gorilla/mux + + +github.com/gorilla/mux + + + + + +github.com/docker/distribution/registry/api/v2->github.com/gorilla/mux + + + + + +github.com/docker/distribution/registry/client->bytes + + + + + +github.com/docker/distribution/registry/client->context + + + + + +github.com/docker/distribution/registry/client->encoding/json + + + + + +github.com/docker/distribution/registry/client->errors + + + + + +github.com/docker/distribution/registry/client->fmt + + + + + +github.com/docker/distribution/registry/client->io + + + + + +github.com/docker/distribution/registry/client->io/ioutil + + + + + +github.com/docker/distribution/registry/client->strconv + + + + + +github.com/docker/distribution/registry/client->strings + + + + + +github.com/docker/distribution/registry/client->time + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/api/errcode + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/api/v2 + + + + + +github.com/docker/distribution/registry/client->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/client->net/http + + + + + +github.com/docker/distribution/registry/client->net/url + + + + + +github.com/docker/distribution + + +github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/client/auth/challenge + + +github.com/docker/distribution/registry/client/auth/challenge + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/client/auth/challenge + + + + + +github.com/docker/distribution/registry/client/transport + + +github.com/docker/distribution/registry/client/transport + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/client/transport + + + + + +github.com/docker/distribution/registry/storage/cache + + +github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/distribution/registry/storage/cache/memory + + +github.com/docker/distribution/registry/storage/cache/memory + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/storage/cache/memory + + + + + +github.com/docker/go-connections/tlsconfig->crypto/tls + + + + + +github.com/docker/go-connections/tlsconfig->crypto/x509 + + + + + +github.com/docker/go-connections/tlsconfig->encoding/pem + + + + + +github.com/docker/go-connections/tlsconfig->fmt + + + + + +github.com/docker/go-connections/tlsconfig->io/ioutil + + + + + +github.com/docker/go-connections/tlsconfig->github.com/pkg/errors + + + + + +github.com/docker/go-connections/tlsconfig->os + + + + + +github.com/docker/go-connections/tlsconfig->runtime + + + + + +github.com/ghodss/yaml->bytes + + + + + +github.com/ghodss/yaml->encoding + + + + + +github.com/ghodss/yaml->encoding/json + + + + + +github.com/ghodss/yaml->fmt + + + + + +github.com/ghodss/yaml->reflect + + + + + +github.com/ghodss/yaml->sort + + + + + +github.com/ghodss/yaml->strconv + + + + + +github.com/ghodss/yaml->strings + + + + + +github.com/ghodss/yaml->sync + + + + + +github.com/ghodss/yaml->unicode + + + + + +github.com/ghodss/yaml->unicode/utf8 + + + + + +gopkg.in/yaml.v2 + + +gopkg.in/yaml.v2 + + + + + +github.com/ghodss/yaml->gopkg.in/yaml.v2 + + + + + +github.com/opencontainers/go-digest->crypto + + + + + +github.com/opencontainers/go-digest->fmt + + + + + +github.com/opencontainers/go-digest->io + + + + + +github.com/opencontainers/go-digest->strings + + + + + +github.com/opencontainers/go-digest->regexp + + + + + +hash + + +hash + + + + + +github.com/opencontainers/go-digest->hash + + + + + +github.com/opencontainers/image-spec/specs-go/v1->time + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/image-spec/specs-go + + + + + +github.com/pkg/errors->fmt + + + + + +github.com/pkg/errors->io + + + + + +github.com/pkg/errors->strings + + + + + +github.com/pkg/errors->path + + + + + +github.com/pkg/errors->runtime + + + + + +github.com/sirupsen/logrus->bufio + + + + + +github.com/sirupsen/logrus->bytes + + + + + +github.com/sirupsen/logrus->context + + + + + +github.com/sirupsen/logrus->encoding/json + + + + + +github.com/sirupsen/logrus->fmt + + + + + +github.com/sirupsen/logrus->io + + + + + +github.com/sirupsen/logrus->reflect + + + + + +github.com/sirupsen/logrus->sort + + + + + +github.com/sirupsen/logrus->strings + + + + + +github.com/sirupsen/logrus->sync + + + + + +github.com/sirupsen/logrus->time + + + + + +github.com/sirupsen/logrus->os + + + + + +golang.org/x/sys/unix + + +golang.org/x/sys/unix + + + + + +github.com/sirupsen/logrus->golang.org/x/sys/unix + + + + + +github.com/sirupsen/logrus->runtime + + + + + +log + + +log + + + + + +github.com/sirupsen/logrus->log + + + + + +sync/atomic + + +sync/atomic + + + + + +github.com/sirupsen/logrus->sync/atomic + + + + + +github.com/containers/image/v5/internal/pkg/keyctl->unsafe + + + + + +github.com/containers/image/v5/internal/pkg/keyctl->golang.org/x/sys/unix + + + + + +golang.org/x/sys/unix->bytes + + + + + +golang.org/x/sys/unix->encoding/binary + + + + + +golang.org/x/sys/unix->sort + + + + + +golang.org/x/sys/unix->strings + + + + + +golang.org/x/sys/unix->sync + + + + + +golang.org/x/sys/unix->time + + + + + +golang.org/x/sys/unix->unsafe + + + + + +golang.org/x/sys/unix->runtime + + + + + +golang.org/x/sys/unix->net + + + + + +syscall + + +syscall + + + + + +golang.org/x/sys/unix->syscall + + + + + +github.com/containers/image/v5/pkg/compression->bytes + + + + + +github.com/containers/image/v5/pkg/compression->compress/bzip2 + + + + + +github.com/containers/image/v5/pkg/compression->fmt + + + + + +github.com/containers/image/v5/pkg/compression->io + + + + + +github.com/containers/image/v5/pkg/compression->io/ioutil + + + + + +github.com/containers/image/v5/pkg/compression->github.com/pkg/errors + + + + + +github.com/containers/image/v5/pkg/compression->github.com/sirupsen/logrus + + + + + +github.com/containers/image/v5/pkg/compression/internal + + +github.com/containers/image/v5/pkg/compression/internal + + + + + +github.com/containers/image/v5/pkg/compression->github.com/containers/image/v5/pkg/compression/internal + + + + + +github.com/containers/image/v5/pkg/compression->github.com/containers/image/v5/pkg/compression/types + + + + + +github.com/klauspost/compress/zstd + + +github.com/klauspost/compress/zstd + + + + + +github.com/containers/image/v5/pkg/compression->github.com/klauspost/compress/zstd + + + + + +github.com/klauspost/pgzip + + +github.com/klauspost/pgzip + + + + + +github.com/containers/image/v5/pkg/compression->github.com/klauspost/pgzip + + + + + +github.com/ulikunitz/xz + + +github.com/ulikunitz/xz + + + + + +github.com/containers/image/v5/pkg/compression->github.com/ulikunitz/xz + + + + + +github.com/containers/image/v5/pkg/strslice->encoding/json + + + + + +github.com/containers/libtrust->bytes + + + + + +github.com/containers/libtrust->crypto + + + + + +github.com/containers/libtrust->crypto/ecdsa + + + + + +github.com/containers/libtrust->crypto/elliptic + + + + + +github.com/containers/libtrust->crypto/rand + + + + + +github.com/containers/libtrust->crypto/rsa + + + + + +github.com/containers/libtrust->crypto/sha256 + + + + + +github.com/containers/libtrust->crypto/sha512 + + + + + +github.com/containers/libtrust->crypto/tls + + + + + +github.com/containers/libtrust->crypto/x509 + + + + + +github.com/containers/libtrust->crypto/x509/pkix + + + + + +github.com/containers/libtrust->encoding/base32 + + + + + +github.com/containers/libtrust->encoding/base64 + + + + + +github.com/containers/libtrust->encoding/binary + + + + + +github.com/containers/libtrust->encoding/json + + + + + +github.com/containers/libtrust->encoding/pem + + + + + +github.com/containers/libtrust->errors + + + + + +github.com/containers/libtrust->fmt + + + + + +github.com/containers/libtrust->io + + + + + +github.com/containers/libtrust->io/ioutil + + + + + +github.com/containers/libtrust->sort + + + + + +github.com/containers/libtrust->strings + + + + + +github.com/containers/libtrust->sync + + + + + +github.com/containers/libtrust->time + + + + + +github.com/containers/libtrust->unicode + + + + + +github.com/containers/libtrust->net/url + + + + + +github.com/containers/libtrust->os + + + + + +github.com/containers/libtrust->path + + + + + +github.com/containers/libtrust->path/filepath + + + + + +github.com/containers/libtrust->net + + + + + +math/big + + +math/big + + + + + +github.com/containers/libtrust->math/big + + + + + +github.com/docker/docker/api/types/versions->strconv + + + + + +github.com/docker/docker/api/types/versions->strings + + + + + +github.com/opencontainers/image-spec/specs-go->fmt + + + + + +github.com/containers/image/v5/pkg/compression/internal->io + + + + + +github.com/containers/image/v5/pkg/compression/types->github.com/containers/image/v5/pkg/compression/internal + + + + + +github.com/klauspost/compress/zstd->bytes + + + + + +github.com/klauspost/compress/zstd->crypto/rand + + + + + +github.com/klauspost/compress/zstd->encoding/binary + + + + + +github.com/klauspost/compress/zstd->encoding/hex + + + + + +github.com/klauspost/compress/zstd->errors + + + + + +github.com/klauspost/compress/zstd->fmt + + + + + +github.com/klauspost/compress/zstd->io + + + + + +github.com/klauspost/compress/zstd->io/ioutil + + + + + +github.com/klauspost/compress/zstd->math + + + + + +github.com/klauspost/compress/zstd->strconv + + + + + +github.com/klauspost/compress/zstd->strings + + + + + +github.com/klauspost/compress/zstd->sync + + + + + +github.com/klauspost/compress/zstd->math/bits + + + + + +github.com/klauspost/compress/zstd->runtime + + + + + +github.com/klauspost/compress/zstd->log + + + + + +github.com/klauspost/compress/huff0 + + +github.com/klauspost/compress/huff0 + + + + + +github.com/klauspost/compress/zstd->github.com/klauspost/compress/huff0 + + + + + +github.com/klauspost/compress/snappy + + +github.com/klauspost/compress/snappy + + + + + +github.com/klauspost/compress/zstd->github.com/klauspost/compress/snappy + + + + + +hash/crc32 + + +hash/crc32 + + + + + +github.com/klauspost/compress/zstd->hash/crc32 + + + + + +github.com/klauspost/compress/zstd/internal/xxhash + + +github.com/klauspost/compress/zstd/internal/xxhash + + + + + +github.com/klauspost/compress/zstd->github.com/klauspost/compress/zstd/internal/xxhash + + + + + +github.com/klauspost/compress/zstd->hash + + + + + +runtime/debug + + +runtime/debug + + + + + +github.com/klauspost/compress/zstd->runtime/debug + + + + + +github.com/klauspost/pgzip->bufio + + + + + +github.com/klauspost/pgzip->bytes + + + + + +github.com/klauspost/pgzip->errors + + + + + +github.com/klauspost/pgzip->fmt + + + + + +github.com/klauspost/pgzip->io + + + + + +github.com/klauspost/pgzip->sync + + + + + +github.com/klauspost/pgzip->time + + + + + +github.com/klauspost/compress/flate + + +github.com/klauspost/compress/flate + + + + + +github.com/klauspost/pgzip->github.com/klauspost/compress/flate + + + + + +github.com/klauspost/pgzip->hash/crc32 + + + + + +github.com/klauspost/pgzip->hash + + + + + +github.com/ulikunitz/xz->bytes + + + + + +github.com/ulikunitz/xz->crypto/sha256 + + + + + +github.com/ulikunitz/xz->errors + + + + + +github.com/ulikunitz/xz->fmt + + + + + +github.com/ulikunitz/xz->io + + + + + +github.com/ulikunitz/xz->hash/crc32 + + + + + +github.com/ulikunitz/xz->hash + + + + + +github.com/ulikunitz/xz/internal/xlog + + +github.com/ulikunitz/xz/internal/xlog + + + + + +github.com/ulikunitz/xz->github.com/ulikunitz/xz/internal/xlog + + + + + +github.com/ulikunitz/xz/lzma + + +github.com/ulikunitz/xz/lzma + + + + + +github.com/ulikunitz/xz->github.com/ulikunitz/xz/lzma + + + + + +hash/crc64 + + +hash/crc64 + + + + + +github.com/ulikunitz/xz->hash/crc64 + + + + + +github.com/docker/docker-credential-helpers/client->bytes + + + + + +github.com/docker/docker-credential-helpers/client->encoding/json + + + + + +github.com/docker/docker-credential-helpers/client->fmt + + + + + +github.com/docker/docker-credential-helpers/client->io + + + + + +github.com/docker/docker-credential-helpers/client->strings + + + + + +github.com/docker/docker-credential-helpers/client->os + + + + + +github.com/docker/docker-credential-helpers/client->github.com/docker/docker-credential-helpers/credentials + + + + + +os/exec + + +os/exec + + + + + +github.com/docker/docker-credential-helpers/client->os/exec + + + + + +github.com/docker/docker-credential-helpers/credentials->bufio + + + + + +github.com/docker/docker-credential-helpers/credentials->bytes + + + + + +github.com/docker/docker-credential-helpers/credentials->encoding/json + + + + + +github.com/docker/docker-credential-helpers/credentials->fmt + + + + + +github.com/docker/docker-credential-helpers/credentials->io + + + + + +github.com/docker/docker-credential-helpers/credentials->strings + + + + + +github.com/docker/docker-credential-helpers/credentials->os + + + + + +github.com/docker/docker/pkg/homedir->os + + + + + +github.com/docker/docker/pkg/idtools + + +github.com/docker/docker/pkg/idtools + + + + + +github.com/docker/docker/pkg/homedir->github.com/docker/docker/pkg/idtools + + + + + +github.com/opencontainers/runc/libcontainer/user + + +github.com/opencontainers/runc/libcontainer/user + + + + + +github.com/docker/docker/pkg/homedir->github.com/opencontainers/runc/libcontainer/user + + + + + +github.com/docker/go-connections/sockets->crypto/tls + + + + + +github.com/docker/go-connections/sockets->errors + + + + + +github.com/docker/go-connections/sockets->fmt + + + + + +github.com/docker/go-connections/sockets->strings + + + + + +github.com/docker/go-connections/sockets->sync + + + + + +github.com/docker/go-connections/sockets->time + + + + + +github.com/docker/go-connections/sockets->net/http + + + + + +github.com/docker/go-connections/sockets->net/url + + + + + +github.com/docker/go-connections/sockets->os + + + + + +github.com/docker/go-connections/sockets->net + + + + + +github.com/docker/go-connections/sockets->syscall + + + + + +golang.org/x/net/proxy + + +golang.org/x/net/proxy + + + + + +github.com/docker/go-connections/sockets->golang.org/x/net/proxy + + + + + +github.com/docker/distribution->context + + + + + +github.com/docker/distribution->errors + + + + + +github.com/docker/distribution->fmt + + + + + +github.com/docker/distribution->io + + + + + +github.com/docker/distribution->strings + + + + + +github.com/docker/distribution->time + + + + + +github.com/docker/distribution->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/docker/distribution->mime + + + + + +github.com/docker/distribution->net/http + + + + + +github.com/docker/distribution->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/reference->errors + + + + + +github.com/docker/distribution/reference->fmt + + + + + +github.com/docker/distribution/reference->strings + + + + + +github.com/docker/distribution/reference->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/reference->path + + + + + +github.com/docker/distribution/reference->regexp + + + + + +github.com/docker/distribution/digestset + + +github.com/docker/distribution/digestset + + + + + +github.com/docker/distribution/reference->github.com/docker/distribution/digestset + + + + + +github.com/docker/distribution/digestset->errors + + + + + +github.com/docker/distribution/digestset->sort + + + + + +github.com/docker/distribution/digestset->strings + + + + + +github.com/docker/distribution/digestset->sync + + + + + +github.com/docker/distribution/digestset->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/metrics + + +github.com/docker/distribution/metrics + + + + + +github.com/docker/go-metrics + + +github.com/docker/go-metrics + + + + + +github.com/docker/distribution/metrics->github.com/docker/go-metrics + + + + + +github.com/docker/go-metrics->fmt + + + + + +github.com/docker/go-metrics->sync + + + + + +github.com/docker/go-metrics->time + + + + + +github.com/docker/go-metrics->net/http + + + + + +github.com/prometheus/client_golang/prometheus + + +github.com/prometheus/client_golang/prometheus + + + + + +github.com/docker/go-metrics->github.com/prometheus/client_golang/prometheus + + + + + +github.com/prometheus/client_golang/prometheus/promhttp + + +github.com/prometheus/client_golang/prometheus/promhttp + + + + + +github.com/docker/go-metrics->github.com/prometheus/client_golang/prometheus/promhttp + + + + + +github.com/gorilla/mux->bytes + + + + + +github.com/gorilla/mux->context + + + + + +github.com/gorilla/mux->errors + + + + + +github.com/gorilla/mux->fmt + + + + + +github.com/gorilla/mux->strconv + + + + + +github.com/gorilla/mux->strings + + + + + +github.com/gorilla/mux->net/http + + + + + +github.com/gorilla/mux->net/url + + + + + +github.com/gorilla/mux->path + + + + + +github.com/gorilla/mux->regexp + + + + + +github.com/docker/distribution/registry/client/auth/challenge->fmt + + + + + +github.com/docker/distribution/registry/client/auth/challenge->strings + + + + + +github.com/docker/distribution/registry/client/auth/challenge->sync + + + + + +github.com/docker/distribution/registry/client/auth/challenge->net/http + + + + + +github.com/docker/distribution/registry/client/auth/challenge->net/url + + + + + +github.com/docker/distribution/registry/client/transport->errors + + + + + +github.com/docker/distribution/registry/client/transport->fmt + + + + + +github.com/docker/distribution/registry/client/transport->io + + + + + +github.com/docker/distribution/registry/client/transport->strconv + + + + + +github.com/docker/distribution/registry/client/transport->sync + + + + + +github.com/docker/distribution/registry/client/transport->net/http + + + + + +github.com/docker/distribution/registry/client/transport->regexp + + + + + +github.com/docker/distribution/registry/storage/cache->context + + + + + +github.com/docker/distribution/registry/storage/cache->fmt + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/docker/distribution/metrics + + + + + +github.com/docker/distribution/registry/storage/cache/memory->context + + + + + +github.com/docker/distribution/registry/storage/cache/memory->sync + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/docker/pkg/idtools->bufio + + + + + +github.com/docker/docker/pkg/idtools->bytes + + + + + +github.com/docker/docker/pkg/idtools->fmt + + + + + +github.com/docker/docker/pkg/idtools->io + + + + + +github.com/docker/docker/pkg/idtools->sort + + + + + +github.com/docker/docker/pkg/idtools->strconv + + + + + +github.com/docker/docker/pkg/idtools->strings + + + + + +github.com/docker/docker/pkg/idtools->sync + + + + + +github.com/docker/docker/pkg/idtools->os + + + + + +github.com/docker/docker/pkg/idtools->path/filepath + + + + + +github.com/docker/docker/pkg/idtools->regexp + + + + + +github.com/docker/docker/pkg/idtools->os/exec + + + + + +github.com/docker/docker/pkg/idtools->github.com/opencontainers/runc/libcontainer/user + + + + + +github.com/docker/docker/pkg/system + + +github.com/docker/docker/pkg/system + + + + + +github.com/docker/docker/pkg/idtools->github.com/docker/docker/pkg/system + + + + + +github.com/docker/docker/pkg/idtools->syscall + + + + + +github.com/opencontainers/runc/libcontainer/user->bufio + + + + + +github.com/opencontainers/runc/libcontainer/user->errors + + + + + +github.com/opencontainers/runc/libcontainer/user->fmt + + + + + +github.com/opencontainers/runc/libcontainer/user->io + + + + + +github.com/opencontainers/runc/libcontainer/user->strconv + + + + + +github.com/opencontainers/runc/libcontainer/user->strings + + + + + +github.com/opencontainers/runc/libcontainer/user->os + + + + + +github.com/opencontainers/runc/libcontainer/user->golang.org/x/sys/unix + + + + + +os/user + + +os/user + + + + + +github.com/opencontainers/runc/libcontainer/user->os/user + + + + + +github.com/docker/docker/pkg/system->bufio + + + + + +github.com/docker/docker/pkg/system->errors + + + + + +github.com/docker/docker/pkg/system->fmt + + + + + +github.com/docker/docker/pkg/system->io + + + + + +github.com/docker/docker/pkg/system->io/ioutil + + + + + +github.com/docker/docker/pkg/system->strconv + + + + + +github.com/docker/docker/pkg/system->strings + + + + + +github.com/docker/docker/pkg/system->time + + + + + +github.com/docker/docker/pkg/system->unsafe + + + + + +github.com/docker/docker/pkg/system->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/docker/docker/pkg/system->github.com/pkg/errors + + + + + +github.com/docker/docker/pkg/system->os + + + + + +github.com/docker/docker/pkg/system->path/filepath + + + + + +github.com/docker/docker/pkg/system->golang.org/x/sys/unix + + + + + +github.com/docker/docker/pkg/system->runtime + + + + + +github.com/docker/docker/pkg/system->os/exec + + + + + +github.com/docker/docker/pkg/system->syscall + + + + + +github.com/docker/docker/pkg/mount + + +github.com/docker/docker/pkg/mount + + + + + +github.com/docker/docker/pkg/system->github.com/docker/docker/pkg/mount + + + + + +github.com/docker/go-units + + +github.com/docker/go-units + + + + + +github.com/docker/docker/pkg/system->github.com/docker/go-units + + + + + +github.com/docker/docker/pkg/mount->bufio + + + + + +github.com/docker/docker/pkg/mount->fmt + + + + + +github.com/docker/docker/pkg/mount->io + + + + + +github.com/docker/docker/pkg/mount->sort + + + + + +github.com/docker/docker/pkg/mount->strconv + + + + + +github.com/docker/docker/pkg/mount->strings + + + + + +github.com/docker/docker/pkg/mount->github.com/pkg/errors + + + + + +github.com/docker/docker/pkg/mount->github.com/sirupsen/logrus + + + + + +github.com/docker/docker/pkg/mount->os + + + + + +github.com/docker/docker/pkg/mount->golang.org/x/sys/unix + + + + + +github.com/docker/go-units->fmt + + + + + +github.com/docker/go-units->strconv + + + + + +github.com/docker/go-units->strings + + + + + +github.com/docker/go-units->time + + + + + +github.com/docker/go-units->regexp + + + + + +golang.org/x/net/proxy->context + + + + + +golang.org/x/net/proxy->errors + + + + + +golang.org/x/net/proxy->strings + + + + + +golang.org/x/net/proxy->sync + + + + + +golang.org/x/net/proxy->net/url + + + + + +golang.org/x/net/proxy->os + + + + + +golang.org/x/net/proxy->net + + + + + +golang.org/x/net/internal/socks + + +golang.org/x/net/internal/socks + + + + + +golang.org/x/net/proxy->golang.org/x/net/internal/socks + + + + + +github.com/prometheus/client_golang/prometheus->bytes + + + + + +github.com/prometheus/client_golang/prometheus->encoding/json + + + + + +github.com/prometheus/client_golang/prometheus->errors + + + + + +github.com/prometheus/client_golang/prometheus->expvar + + + + + +github.com/prometheus/client_golang/prometheus->fmt + + + + + +github.com/prometheus/client_golang/prometheus->io/ioutil + + + + + +github.com/prometheus/client_golang/prometheus->math + + + + + +github.com/prometheus/client_golang/prometheus->sort + + + + + +github.com/prometheus/client_golang/prometheus->strings + + + + + +github.com/prometheus/client_golang/prometheus->sync + + + + + +github.com/prometheus/client_golang/prometheus->time + + + + + +github.com/prometheus/client_golang/prometheus->unicode/utf8 + + + + + +github.com/prometheus/client_golang/prometheus->github.com/beorn7/perks/quantile + + + + + +github.com/prometheus/client_golang/prometheus->github.com/cespare/xxhash/v2 + + + + + +github.com/prometheus/client_golang/prometheus->os + + + + + +github.com/prometheus/client_golang/prometheus->path/filepath + + + + + +github.com/prometheus/client_golang/prometheus->runtime + + + + + +github.com/golang/protobuf/proto + + +github.com/golang/protobuf/proto + + + + + +github.com/prometheus/client_golang/prometheus->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/client_golang/prometheus->sync/atomic + + + + + +github.com/prometheus/client_golang/prometheus->runtime/debug + + + + + +github.com/prometheus/client_golang/prometheus/internal + + +github.com/prometheus/client_golang/prometheus/internal + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/client_golang/prometheus/internal + + + + + +github.com/prometheus/client_model/go + + +github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/common/expfmt + + +github.com/prometheus/common/expfmt + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/common/expfmt + + + + + +github.com/prometheus/common/model + + +github.com/prometheus/common/model + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/common/model + + + + + +github.com/prometheus/procfs + + +github.com/prometheus/procfs + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/procfs + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->bufio + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->compress/gzip + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->crypto/tls + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->errors + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->fmt + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->io + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->strconv + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->strings + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->sync + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->time + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net/http + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/client_golang/prometheus + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/common/expfmt + + + + + +net/http/httptrace + + +net/http/httptrace + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net/http/httptrace + + + + + +gopkg.in/yaml.v2->bytes + + + + + +gopkg.in/yaml.v2->encoding + + + + + +gopkg.in/yaml.v2->encoding/base64 + + + + + +gopkg.in/yaml.v2->errors + + + + + +gopkg.in/yaml.v2->fmt + + + + + +gopkg.in/yaml.v2->io + + + + + +gopkg.in/yaml.v2->math + + + + + +gopkg.in/yaml.v2->reflect + + + + + +gopkg.in/yaml.v2->sort + + + + + +gopkg.in/yaml.v2->strconv + + + + + +gopkg.in/yaml.v2->strings + + + + + +gopkg.in/yaml.v2->sync + + + + + +gopkg.in/yaml.v2->time + + + + + +gopkg.in/yaml.v2->unicode + + + + + +gopkg.in/yaml.v2->unicode/utf8 + + + + + +gopkg.in/yaml.v2->regexp + + + + + +github.com/golang/protobuf/proto->bufio + + + + + +github.com/golang/protobuf/proto->bytes + + + + + +github.com/golang/protobuf/proto->encoding + + + + + +github.com/golang/protobuf/proto->encoding/json + + + + + +github.com/golang/protobuf/proto->errors + + + + + +github.com/golang/protobuf/proto->fmt + + + + + +github.com/golang/protobuf/proto->io + + + + + +github.com/golang/protobuf/proto->math + + + + + +github.com/golang/protobuf/proto->reflect + + + + + +github.com/golang/protobuf/proto->sort + + + + + +github.com/golang/protobuf/proto->strconv + + + + + +github.com/golang/protobuf/proto->strings + + + + + +github.com/golang/protobuf/proto->sync + + + + + +github.com/golang/protobuf/proto->unicode/utf8 + + + + + +github.com/golang/protobuf/proto->unsafe + + + + + +github.com/golang/protobuf/proto->log + + + + + +github.com/golang/protobuf/proto->sync/atomic + + + + + +github.com/klauspost/compress/flate->bufio + + + + + +github.com/klauspost/compress/flate->bytes + + + + + +github.com/klauspost/compress/flate->encoding/binary + + + + + +github.com/klauspost/compress/flate->fmt + + + + + +github.com/klauspost/compress/flate->io + + + + + +github.com/klauspost/compress/flate->math + + + + + +github.com/klauspost/compress/flate->sort + + + + + +github.com/klauspost/compress/flate->strconv + + + + + +github.com/klauspost/compress/flate->sync + + + + + +github.com/klauspost/compress/flate->math/bits + + + + + +github.com/klauspost/compress/fse + + +github.com/klauspost/compress/fse + + + + + +github.com/klauspost/compress/fse->errors + + + + + +github.com/klauspost/compress/fse->fmt + + + + + +github.com/klauspost/compress/fse->io + + + + + +github.com/klauspost/compress/fse->math/bits + + + + + +github.com/klauspost/compress/huff0->errors + + + + + +github.com/klauspost/compress/huff0->fmt + + + + + +github.com/klauspost/compress/huff0->io + + + + + +github.com/klauspost/compress/huff0->math + + + + + +github.com/klauspost/compress/huff0->sync + + + + + +github.com/klauspost/compress/huff0->math/bits + + + + + +github.com/klauspost/compress/huff0->runtime + + + + + +github.com/klauspost/compress/huff0->github.com/klauspost/compress/fse + + + + + +github.com/klauspost/compress/snappy->encoding/binary + + + + + +github.com/klauspost/compress/snappy->errors + + + + + +github.com/klauspost/compress/snappy->io + + + + + +github.com/klauspost/compress/snappy->hash/crc32 + + + + + +github.com/klauspost/compress/zstd/internal/xxhash->encoding/binary + + + + + +github.com/klauspost/compress/zstd/internal/xxhash->errors + + + + + +github.com/klauspost/compress/zstd/internal/xxhash->math/bits + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil + + +github.com/matttproud/golang_protobuf_extensions/pbutil + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->encoding/binary + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->errors + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->io + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/client_golang/prometheus/internal->sort + + + + + +github.com/prometheus/client_golang/prometheus/internal->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_model/go->fmt + + + + + +github.com/prometheus/client_model/go->math + + + + + +github.com/prometheus/client_model/go->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/common/expfmt->bufio + + + + + +github.com/prometheus/common/expfmt->bytes + + + + + +github.com/prometheus/common/expfmt->fmt + + + + + +github.com/prometheus/common/expfmt->io + + + + + +github.com/prometheus/common/expfmt->io/ioutil + + + + + +github.com/prometheus/common/expfmt->math + + + + + +github.com/prometheus/common/expfmt->strconv + + + + + +github.com/prometheus/common/expfmt->strings + + + + + +github.com/prometheus/common/expfmt->sync + + + + + +github.com/prometheus/common/expfmt->mime + + + + + +github.com/prometheus/common/expfmt->net/http + + + + + +github.com/prometheus/common/expfmt->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/common/expfmt->github.com/matttproud/golang_protobuf_extensions/pbutil + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/common/model + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + + + + +github.com/prometheus/common/model->encoding/json + + + + + +github.com/prometheus/common/model->fmt + + + + + +github.com/prometheus/common/model->math + + + + + +github.com/prometheus/common/model->sort + + + + + +github.com/prometheus/common/model->strconv + + + + + +github.com/prometheus/common/model->strings + + + + + +github.com/prometheus/common/model->time + + + + + +github.com/prometheus/common/model->unicode/utf8 + + + + + +github.com/prometheus/common/model->regexp + + + + + +github.com/prometheus/procfs->bufio + + + + + +github.com/prometheus/procfs->bytes + + + + + +github.com/prometheus/procfs->encoding/hex + + + + + +github.com/prometheus/procfs->errors + + + + + +github.com/prometheus/procfs->fmt + + + + + +github.com/prometheus/procfs->io + + + + + +github.com/prometheus/procfs->io/ioutil + + + + + +github.com/prometheus/procfs->sort + + + + + +github.com/prometheus/procfs->strconv + + + + + +github.com/prometheus/procfs->strings + + + + + +github.com/prometheus/procfs->time + + + + + +github.com/prometheus/procfs->os + + + + + +github.com/prometheus/procfs->path/filepath + + + + + +github.com/prometheus/procfs->regexp + + + + + +github.com/prometheus/procfs->net + + + + + +github.com/prometheus/procfs/internal/fs + + +github.com/prometheus/procfs/internal/fs + + + + + +github.com/prometheus/procfs->github.com/prometheus/procfs/internal/fs + + + + + +github.com/prometheus/procfs/internal/util + + +github.com/prometheus/procfs/internal/util + + + + + +github.com/prometheus/procfs->github.com/prometheus/procfs/internal/util + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->sort + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->strconv + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->strings + + + + + +github.com/prometheus/procfs/internal/fs->fmt + + + + + +github.com/prometheus/procfs/internal/fs->os + + + + + +github.com/prometheus/procfs/internal/fs->path/filepath + + + + + +github.com/prometheus/procfs/internal/util->bytes + + + + + +github.com/prometheus/procfs/internal/util->io/ioutil + + + + + +github.com/prometheus/procfs/internal/util->strconv + + + + + +github.com/prometheus/procfs/internal/util->strings + + + + + +github.com/prometheus/procfs/internal/util->os + + + + + +github.com/prometheus/procfs/internal/util->syscall + + + + + +github.com/ulikunitz/xz/internal/xlog->fmt + + + + + +github.com/ulikunitz/xz/internal/xlog->io + + + + + +github.com/ulikunitz/xz/internal/xlog->sync + + + + + +github.com/ulikunitz/xz/internal/xlog->time + + + + + +github.com/ulikunitz/xz/internal/xlog->os + + + + + +github.com/ulikunitz/xz/internal/xlog->runtime + + + + + +github.com/ulikunitz/xz/lzma->bufio + + + + + +github.com/ulikunitz/xz/lzma->bytes + + + + + +github.com/ulikunitz/xz/lzma->errors + + + + + +github.com/ulikunitz/xz/lzma->fmt + + + + + +github.com/ulikunitz/xz/lzma->io + + + + + +github.com/ulikunitz/xz/lzma->unicode + + + + + +github.com/ulikunitz/xz/lzma->github.com/ulikunitz/xz/internal/xlog + + + + + +github.com/ulikunitz/xz/internal/hash + + +github.com/ulikunitz/xz/internal/hash + + + + + +github.com/ulikunitz/xz/lzma->github.com/ulikunitz/xz/internal/hash + + + + + +golang.org/x/net/internal/socks->context + + + + + +golang.org/x/net/internal/socks->errors + + + + + +golang.org/x/net/internal/socks->io + + + + + +golang.org/x/net/internal/socks->strconv + + + + + +golang.org/x/net/internal/socks->time + + + + + +golang.org/x/net/internal/socks->net + + + + + diff --git a/images/crane.png b/images/crane.png new file mode 100644 index 0000000..ffd95af Binary files /dev/null and b/images/crane.png differ diff --git a/images/credhelper-basic.svg b/images/credhelper-basic.svg new file mode 100644 index 0000000..44d4d0e --- /dev/null +++ b/images/credhelper-basic.svg @@ -0,0 +1 @@ +Created with Raphaël 2.2.0Credential helper flow - Basic authggcrggcrregistryregistryconfigconfighelperhelpergcloudgcloudGET /v2/401 UnauthorizedWww-Authenticate: Bearer realm="<rlm>",service="<svc>"GetAuthConfig("gcr.io")~/.docker/config.json:{"credHelpers":{"gcr.io": "gcr"}}$ echo gcr.io | docker-credential-gcr get$ gcloud auth print-access-token --format=json{"access_token":"hunter2","token_expiry":"..."}{"Username":"_token","Secret":"hunter2"}{"username":"_token","password":"hunter2"}note: base64("_token:hunter2") == "X3Rva2VuOmh1bnRlcjI="GET <rlm>?service=<svc>&scope=...Authorization: Basic X3Rva2VuOmh1bnRlcjI=200 OK{"token":"<bearer token>"}GET /v2/_catalogAuthorization: Bearer <bearer token>{"repositories":["foo", "bar"]} \ No newline at end of file diff --git a/images/credhelper-oauth.svg b/images/credhelper-oauth.svg new file mode 100644 index 0000000..a88e1b8 --- /dev/null +++ b/images/credhelper-oauth.svg @@ -0,0 +1 @@ +Created with Raphaël 2.2.0Credential helper flow - OauthggcrggcrregistryregistryconfigconfighelperhelperGET /v2/401 UnauthorizedWww-Authenticate: Bearer realm="<rlm>",service="<svc>"GetAuthConfig("example.com")~/.docker/config.json:{"credHelpers":{"example.com": "foo"}}$ echo example.com | docker-credential-foo get{"Username":"<token>","Secret":"hunter2"}the "<token>" username indicates this is an IdentityToken{"identitytoken":"hunter2"}the IdentityToken indicates we should use oauth2POST <rlm> service=<svc>&grant_type=refresh_token&refresh_token=hunter2&client_id=go-containerregistry&scope=...200 OK{"token":"<bearer token>"}GET /v2/_catalogAuthorization: Bearer <bearer token>{"repositories":["foo", "bar"]} \ No newline at end of file diff --git a/images/docker.dot.svg b/images/docker.dot.svg new file mode 100644 index 0000000..f031ddf --- /dev/null +++ b/images/docker.dot.svg @@ -0,0 +1,2155 @@ + + + + + + +godep + + + +bufio + + +bufio + + + + + +bytes + + +bytes + + + + + +compress/gzip + + +compress/gzip + + + + + +context + + +context + + + + + +crypto + + +crypto + + + + + +crypto/tls + + +crypto/tls + + + + + +encoding + + +encoding + + + + + +encoding/binary + + +encoding/binary + + + + + +encoding/hex + + +encoding/hex + + + + + +encoding/json + + +encoding/json + + + + + +errors + + +errors + + + + + +expvar + + +expvar + + + + + +fmt + + +fmt + + + + + +github.com/beorn7/perks/quantile + + +github.com/beorn7/perks/quantile + + + + + +math + + +math + + + + + +github.com/beorn7/perks/quantile->math + + + + + +sort + + +sort + + + + + +github.com/beorn7/perks/quantile->sort + + + + + +github.com/cespare/xxhash/v2 + + +github.com/cespare/xxhash/v2 + + + + + +github.com/cespare/xxhash/v2->encoding/binary + + + + + +github.com/cespare/xxhash/v2->errors + + + + + +math/bits + + +math/bits + + + + + +github.com/cespare/xxhash/v2->math/bits + + + + + +reflect + + +reflect + + + + + +github.com/cespare/xxhash/v2->reflect + + + + + +unsafe + + +unsafe + + + + + +github.com/cespare/xxhash/v2->unsafe + + + + + +github.com/docker/distribution + + +github.com/docker/distribution + + + + + +github.com/docker/distribution->context + + + + + +github.com/docker/distribution->errors + + + + + +github.com/docker/distribution->fmt + + + + + +github.com/docker/distribution/reference + + +github.com/docker/distribution/reference + + + + + +github.com/docker/distribution->github.com/docker/distribution/reference + + + + + +github.com/opencontainers/go-digest + + +github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1 + + +github.com/opencontainers/image-spec/specs-go/v1 + + + + + +github.com/docker/distribution->github.com/opencontainers/image-spec/specs-go/v1 + + + + + +io + + +io + + + + + +github.com/docker/distribution->io + + + + + +mime + + +mime + + + + + +github.com/docker/distribution->mime + + + + + +net/http + + +net/http + + + + + +github.com/docker/distribution->net/http + + + + + +strings + + +strings + + + + + +github.com/docker/distribution->strings + + + + + +time + + +time + + + + + +github.com/docker/distribution->time + + + + + +github.com/docker/distribution/reference->errors + + + + + +github.com/docker/distribution/reference->fmt + + + + + +github.com/docker/distribution/reference->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/reference->strings + + + + + +github.com/docker/distribution/digestset + + +github.com/docker/distribution/digestset + + + + + +github.com/docker/distribution/reference->github.com/docker/distribution/digestset + + + + + +path + + +path + + + + + +github.com/docker/distribution/reference->path + + + + + +regexp + + +regexp + + + + + +github.com/docker/distribution/reference->regexp + + + + + +github.com/opencontainers/go-digest->crypto + + + + + +github.com/opencontainers/go-digest->fmt + + + + + +github.com/opencontainers/go-digest->io + + + + + +github.com/opencontainers/go-digest->strings + + + + + +github.com/opencontainers/go-digest->regexp + + + + + +hash + + +hash + + + + + +github.com/opencontainers/go-digest->hash + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/go-digest + + + + + +github.com/opencontainers/image-spec/specs-go/v1->time + + + + + +github.com/opencontainers/image-spec/specs-go + + +github.com/opencontainers/image-spec/specs-go + + + + + +github.com/opencontainers/image-spec/specs-go/v1->github.com/opencontainers/image-spec/specs-go + + + + + +github.com/docker/distribution/digestset->errors + + + + + +github.com/docker/distribution/digestset->sort + + + + + +github.com/docker/distribution/digestset->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/digestset->strings + + + + + +sync + + +sync + + + + + +github.com/docker/distribution/digestset->sync + + + + + +github.com/docker/distribution/metrics + + +github.com/docker/distribution/metrics + + + + + +github.com/docker/go-metrics + + +github.com/docker/go-metrics + + + + + +github.com/docker/distribution/metrics->github.com/docker/go-metrics + + + + + +github.com/docker/go-metrics->fmt + + + + + +github.com/docker/go-metrics->net/http + + + + + +github.com/docker/go-metrics->time + + + + + +github.com/docker/go-metrics->sync + + + + + +github.com/prometheus/client_golang/prometheus + + +github.com/prometheus/client_golang/prometheus + + + + + +github.com/docker/go-metrics->github.com/prometheus/client_golang/prometheus + + + + + +github.com/prometheus/client_golang/prometheus/promhttp + + +github.com/prometheus/client_golang/prometheus/promhttp + + + + + +github.com/docker/go-metrics->github.com/prometheus/client_golang/prometheus/promhttp + + + + + +github.com/docker/distribution/registry/api/errcode + + +github.com/docker/distribution/registry/api/errcode + + + + + +github.com/docker/distribution/registry/api/errcode->encoding/json + + + + + +github.com/docker/distribution/registry/api/errcode->fmt + + + + + +github.com/docker/distribution/registry/api/errcode->sort + + + + + +github.com/docker/distribution/registry/api/errcode->net/http + + + + + +github.com/docker/distribution/registry/api/errcode->strings + + + + + +github.com/docker/distribution/registry/api/errcode->sync + + + + + +github.com/docker/distribution/registry/api/v2 + + +github.com/docker/distribution/registry/api/v2 + + + + + +github.com/docker/distribution/registry/api/v2->fmt + + + + + +github.com/docker/distribution/registry/api/v2->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/api/v2->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/api/v2->net/http + + + + + +github.com/docker/distribution/registry/api/v2->strings + + + + + +github.com/docker/distribution/registry/api/v2->regexp + + + + + +github.com/docker/distribution/registry/api/v2->github.com/docker/distribution/registry/api/errcode + + + + + +github.com/gorilla/mux + + +github.com/gorilla/mux + + + + + +github.com/docker/distribution/registry/api/v2->github.com/gorilla/mux + + + + + +net/url + + +net/url + + + + + +github.com/docker/distribution/registry/api/v2->net/url + + + + + +unicode + + +unicode + + + + + +github.com/docker/distribution/registry/api/v2->unicode + + + + + +github.com/gorilla/mux->bytes + + + + + +github.com/gorilla/mux->context + + + + + +github.com/gorilla/mux->errors + + + + + +github.com/gorilla/mux->fmt + + + + + +github.com/gorilla/mux->net/http + + + + + +github.com/gorilla/mux->strings + + + + + +github.com/gorilla/mux->path + + + + + +github.com/gorilla/mux->regexp + + + + + +github.com/gorilla/mux->net/url + + + + + +strconv + + +strconv + + + + + +github.com/gorilla/mux->strconv + + + + + +github.com/docker/distribution/registry/client + + +github.com/docker/distribution/registry/client + + + + + +github.com/docker/distribution/registry/client->bytes + + + + + +github.com/docker/distribution/registry/client->context + + + + + +github.com/docker/distribution/registry/client->encoding/json + + + + + +github.com/docker/distribution/registry/client->errors + + + + + +github.com/docker/distribution/registry/client->fmt + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/client->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/client->io + + + + + +github.com/docker/distribution/registry/client->net/http + + + + + +github.com/docker/distribution/registry/client->strings + + + + + +github.com/docker/distribution/registry/client->time + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/api/errcode + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/api/v2 + + + + + +github.com/docker/distribution/registry/client->net/url + + + + + +github.com/docker/distribution/registry/client/auth/challenge + + +github.com/docker/distribution/registry/client/auth/challenge + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/client/auth/challenge + + + + + +github.com/docker/distribution/registry/client/transport + + +github.com/docker/distribution/registry/client/transport + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/client/transport + + + + + +github.com/docker/distribution/registry/storage/cache + + +github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/distribution/registry/storage/cache/memory + + +github.com/docker/distribution/registry/storage/cache/memory + + + + + +github.com/docker/distribution/registry/client->github.com/docker/distribution/registry/storage/cache/memory + + + + + +io/ioutil + + +io/ioutil + + + + + +github.com/docker/distribution/registry/client->io/ioutil + + + + + +github.com/docker/distribution/registry/client->strconv + + + + + +github.com/docker/distribution/registry/client/auth/challenge->fmt + + + + + +github.com/docker/distribution/registry/client/auth/challenge->net/http + + + + + +github.com/docker/distribution/registry/client/auth/challenge->strings + + + + + +github.com/docker/distribution/registry/client/auth/challenge->sync + + + + + +github.com/docker/distribution/registry/client/auth/challenge->net/url + + + + + +github.com/docker/distribution/registry/client/transport->errors + + + + + +github.com/docker/distribution/registry/client/transport->fmt + + + + + +github.com/docker/distribution/registry/client/transport->io + + + + + +github.com/docker/distribution/registry/client/transport->net/http + + + + + +github.com/docker/distribution/registry/client/transport->sync + + + + + +github.com/docker/distribution/registry/client/transport->regexp + + + + + +github.com/docker/distribution/registry/client/transport->strconv + + + + + +github.com/docker/distribution/registry/storage/cache->context + + + + + +github.com/docker/distribution/registry/storage/cache->fmt + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/storage/cache->github.com/docker/distribution/metrics + + + + + +github.com/docker/distribution/registry/storage/cache/memory->context + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution/reference + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/opencontainers/go-digest + + + + + +github.com/docker/distribution/registry/storage/cache/memory->sync + + + + + +github.com/docker/distribution/registry/storage/cache/memory->github.com/docker/distribution/registry/storage/cache + + + + + +github.com/docker/distribution/registry/client/auth + + +github.com/docker/distribution/registry/client/auth + + + + + +github.com/docker/distribution/registry/client/auth->encoding/json + + + + + +github.com/docker/distribution/registry/client/auth->errors + + + + + +github.com/docker/distribution/registry/client/auth->fmt + + + + + +github.com/docker/distribution/registry/client/auth->net/http + + + + + +github.com/docker/distribution/registry/client/auth->strings + + + + + +github.com/docker/distribution/registry/client/auth->time + + + + + +github.com/docker/distribution/registry/client/auth->sync + + + + + +github.com/docker/distribution/registry/client/auth->net/url + + + + + +github.com/docker/distribution/registry/client/auth->github.com/docker/distribution/registry/client + + + + + +github.com/docker/distribution/registry/client/auth->github.com/docker/distribution/registry/client/auth/challenge + + + + + +github.com/docker/distribution/registry/client/auth->github.com/docker/distribution/registry/client/transport + + + + + +github.com/prometheus/client_golang/prometheus->bytes + + + + + +github.com/prometheus/client_golang/prometheus->encoding/json + + + + + +github.com/prometheus/client_golang/prometheus->errors + + + + + +github.com/prometheus/client_golang/prometheus->expvar + + + + + +github.com/prometheus/client_golang/prometheus->fmt + + + + + +github.com/prometheus/client_golang/prometheus->github.com/beorn7/perks/quantile + + + + + +github.com/prometheus/client_golang/prometheus->math + + + + + +github.com/prometheus/client_golang/prometheus->sort + + + + + +github.com/prometheus/client_golang/prometheus->github.com/cespare/xxhash/v2 + + + + + +github.com/prometheus/client_golang/prometheus->strings + + + + + +github.com/prometheus/client_golang/prometheus->time + + + + + +github.com/prometheus/client_golang/prometheus->sync + + + + + +github.com/prometheus/client_golang/prometheus->io/ioutil + + + + + +github.com/golang/protobuf/proto + + +github.com/golang/protobuf/proto + + + + + +github.com/prometheus/client_golang/prometheus->github.com/golang/protobuf/proto + + + + + +sync/atomic + + +sync/atomic + + + + + +github.com/prometheus/client_golang/prometheus->sync/atomic + + + + + +unicode/utf8 + + +unicode/utf8 + + + + + +github.com/prometheus/client_golang/prometheus->unicode/utf8 + + + + + +github.com/prometheus/client_golang/prometheus/internal + + +github.com/prometheus/client_golang/prometheus/internal + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/client_golang/prometheus/internal + + + + + +github.com/prometheus/client_model/go + + +github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/common/expfmt + + +github.com/prometheus/common/expfmt + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/common/expfmt + + + + + +github.com/prometheus/common/model + + +github.com/prometheus/common/model + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/common/model + + + + + +github.com/prometheus/procfs + + +github.com/prometheus/procfs + + + + + +github.com/prometheus/client_golang/prometheus->github.com/prometheus/procfs + + + + + +os + + +os + + + + + +github.com/prometheus/client_golang/prometheus->os + + + + + +path/filepath + + +path/filepath + + + + + +github.com/prometheus/client_golang/prometheus->path/filepath + + + + + +runtime + + +runtime + + + + + +github.com/prometheus/client_golang/prometheus->runtime + + + + + +runtime/debug + + +runtime/debug + + + + + +github.com/prometheus/client_golang/prometheus->runtime/debug + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->bufio + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->compress/gzip + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->crypto/tls + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->errors + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->fmt + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->io + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net/http + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->strings + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->time + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->sync + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->strconv + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/client_golang/prometheus + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->github.com/prometheus/common/expfmt + + + + + +net + + +net + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net + + + + + +net/http/httptrace + + +net/http/httptrace + + + + + +github.com/prometheus/client_golang/prometheus/promhttp->net/http/httptrace + + + + + +github.com/golang/protobuf/proto->bufio + + + + + +github.com/golang/protobuf/proto->bytes + + + + + +github.com/golang/protobuf/proto->encoding + + + + + +github.com/golang/protobuf/proto->encoding/json + + + + + +github.com/golang/protobuf/proto->errors + + + + + +github.com/golang/protobuf/proto->fmt + + + + + +github.com/golang/protobuf/proto->math + + + + + +github.com/golang/protobuf/proto->sort + + + + + +github.com/golang/protobuf/proto->reflect + + + + + +github.com/golang/protobuf/proto->unsafe + + + + + +github.com/golang/protobuf/proto->io + + + + + +github.com/golang/protobuf/proto->strings + + + + + +github.com/golang/protobuf/proto->sync + + + + + +github.com/golang/protobuf/proto->strconv + + + + + +log + + +log + + + + + +github.com/golang/protobuf/proto->log + + + + + +github.com/golang/protobuf/proto->sync/atomic + + + + + +github.com/golang/protobuf/proto->unicode/utf8 + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil + + +github.com/matttproud/golang_protobuf_extensions/pbutil + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->encoding/binary + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->errors + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->io + + + + + +github.com/matttproud/golang_protobuf_extensions/pbutil->github.com/golang/protobuf/proto + + + + + +github.com/opencontainers/image-spec/specs-go->fmt + + + + + +github.com/prometheus/client_golang/prometheus/internal->sort + + + + + +github.com/prometheus/client_golang/prometheus/internal->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/client_model/go->fmt + + + + + +github.com/prometheus/client_model/go->math + + + + + +github.com/prometheus/client_model/go->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/common/expfmt->bufio + + + + + +github.com/prometheus/common/expfmt->bytes + + + + + +github.com/prometheus/common/expfmt->fmt + + + + + +github.com/prometheus/common/expfmt->math + + + + + +github.com/prometheus/common/expfmt->io + + + + + +github.com/prometheus/common/expfmt->mime + + + + + +github.com/prometheus/common/expfmt->net/http + + + + + +github.com/prometheus/common/expfmt->strings + + + + + +github.com/prometheus/common/expfmt->sync + + + + + +github.com/prometheus/common/expfmt->io/ioutil + + + + + +github.com/prometheus/common/expfmt->strconv + + + + + +github.com/prometheus/common/expfmt->github.com/golang/protobuf/proto + + + + + +github.com/prometheus/common/expfmt->github.com/matttproud/golang_protobuf_extensions/pbutil + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/client_model/go + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/common/model + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + + + + +github.com/prometheus/common/expfmt->github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + + + + + +github.com/prometheus/common/model->encoding/json + + + + + +github.com/prometheus/common/model->fmt + + + + + +github.com/prometheus/common/model->math + + + + + +github.com/prometheus/common/model->sort + + + + + +github.com/prometheus/common/model->strings + + + + + +github.com/prometheus/common/model->time + + + + + +github.com/prometheus/common/model->regexp + + + + + +github.com/prometheus/common/model->strconv + + + + + +github.com/prometheus/common/model->unicode/utf8 + + + + + +github.com/prometheus/procfs->bufio + + + + + +github.com/prometheus/procfs->bytes + + + + + +github.com/prometheus/procfs->encoding/hex + + + + + +github.com/prometheus/procfs->errors + + + + + +github.com/prometheus/procfs->fmt + + + + + +github.com/prometheus/procfs->sort + + + + + +github.com/prometheus/procfs->io + + + + + +github.com/prometheus/procfs->strings + + + + + +github.com/prometheus/procfs->time + + + + + +github.com/prometheus/procfs->regexp + + + + + +github.com/prometheus/procfs->io/ioutil + + + + + +github.com/prometheus/procfs->strconv + + + + + +github.com/prometheus/procfs->os + + + + + +github.com/prometheus/procfs->path/filepath + + + + + +github.com/prometheus/procfs->net + + + + + +github.com/prometheus/procfs/internal/fs + + +github.com/prometheus/procfs/internal/fs + + + + + +github.com/prometheus/procfs->github.com/prometheus/procfs/internal/fs + + + + + +github.com/prometheus/procfs/internal/util + + +github.com/prometheus/procfs/internal/util + + + + + +github.com/prometheus/procfs->github.com/prometheus/procfs/internal/util + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->sort + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->strings + + + + + +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg->strconv + + + + + +github.com/prometheus/procfs/internal/fs->fmt + + + + + +github.com/prometheus/procfs/internal/fs->os + + + + + +github.com/prometheus/procfs/internal/fs->path/filepath + + + + + +github.com/prometheus/procfs/internal/util->bytes + + + + + +github.com/prometheus/procfs/internal/util->strings + + + + + +github.com/prometheus/procfs/internal/util->io/ioutil + + + + + +github.com/prometheus/procfs/internal/util->strconv + + + + + +github.com/prometheus/procfs/internal/util->os + + + + + +syscall + + +syscall + + + + + +github.com/prometheus/procfs/internal/util->syscall + + + + + diff --git a/images/dot/containerd.dot b/images/dot/containerd.dot new file mode 100644 index 0000000..f674396 --- /dev/null +++ b/images/dot/containerd.dot @@ -0,0 +1,316 @@ +digraph godep { +nodesep=0.4 +ranksep=0.8 +node [shape="box",style="rounded,filled"] +edge [arrowsize="0.5"] +"bufio" [label="bufio" color="palegreen" URL="https://godoc.org/bufio" target="_blank"]; +"bytes" [label="bytes" color="palegreen" URL="https://godoc.org/bytes" target="_blank"]; +"compress/gzip" [label="compress/gzip" color="palegreen" URL="https://godoc.org/compress/gzip" target="_blank"]; +"container/list" [label="container/list" color="palegreen" URL="https://godoc.org/container/list" target="_blank"]; +"context" [label="context" color="palegreen" URL="https://godoc.org/context" target="_blank"]; +"crypto" [label="crypto" color="palegreen" URL="https://godoc.org/crypto" target="_blank"]; +"encoding" [label="encoding" color="palegreen" URL="https://godoc.org/encoding" target="_blank"]; +"encoding/base64" [label="encoding/base64" color="palegreen" URL="https://godoc.org/encoding/base64" target="_blank"]; +"encoding/json" [label="encoding/json" color="palegreen" URL="https://godoc.org/encoding/json" target="_blank"]; +"errors" [label="errors" color="palegreen" URL="https://godoc.org/errors" target="_blank"]; +"fmt" [label="fmt" color="palegreen" URL="https://godoc.org/fmt" target="_blank"]; +"github.com/containerd/containerd/archive/compression" [label="github.com/containerd/containerd/archive/compression" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/archive/compression" target="_blank"]; +"github.com/containerd/containerd/archive/compression" -> "bufio"; +"github.com/containerd/containerd/archive/compression" -> "bytes"; +"github.com/containerd/containerd/archive/compression" -> "compress/gzip"; +"github.com/containerd/containerd/archive/compression" -> "context"; +"github.com/containerd/containerd/archive/compression" -> "fmt"; +"github.com/containerd/containerd/archive/compression" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/archive/compression" -> "io"; +"github.com/containerd/containerd/archive/compression" -> "os"; +"github.com/containerd/containerd/archive/compression" -> "os/exec"; +"github.com/containerd/containerd/archive/compression" -> "strconv"; +"github.com/containerd/containerd/archive/compression" -> "sync"; +"github.com/containerd/containerd/content" [label="github.com/containerd/containerd/content" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/content" target="_blank"]; +"github.com/containerd/containerd/content" -> "context"; +"github.com/containerd/containerd/content" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/content" -> "github.com/opencontainers/go-digest"; +"github.com/containerd/containerd/content" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/content" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/content" -> "io"; +"github.com/containerd/containerd/content" -> "io/ioutil"; +"github.com/containerd/containerd/content" -> "math/rand"; +"github.com/containerd/containerd/content" -> "sync"; +"github.com/containerd/containerd/content" -> "time"; +"github.com/containerd/containerd/errdefs" [label="github.com/containerd/containerd/errdefs" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/errdefs" target="_blank"]; +"github.com/containerd/containerd/errdefs" -> "context"; +"github.com/containerd/containerd/errdefs" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/errdefs" -> "google.golang.org/grpc/codes"; +"github.com/containerd/containerd/errdefs" -> "google.golang.org/grpc/status"; +"github.com/containerd/containerd/errdefs" -> "strings"; +"github.com/containerd/containerd/images" [label="github.com/containerd/containerd/images" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/images" target="_blank"]; +"github.com/containerd/containerd/images" -> "context"; +"github.com/containerd/containerd/images" -> "encoding/json"; +"github.com/containerd/containerd/images" -> "fmt"; +"github.com/containerd/containerd/images" -> "github.com/containerd/containerd/content"; +"github.com/containerd/containerd/images" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/images" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/images" -> "github.com/containerd/containerd/platforms"; +"github.com/containerd/containerd/images" -> "github.com/opencontainers/go-digest"; +"github.com/containerd/containerd/images" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/images" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/images" -> "golang.org/x/sync/errgroup"; +"github.com/containerd/containerd/images" -> "golang.org/x/sync/semaphore"; +"github.com/containerd/containerd/images" -> "io"; +"github.com/containerd/containerd/images" -> "sort"; +"github.com/containerd/containerd/images" -> "strings"; +"github.com/containerd/containerd/images" -> "time"; +"github.com/containerd/containerd/labels" [label="github.com/containerd/containerd/labels" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/labels" target="_blank"]; +"github.com/containerd/containerd/labels" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/labels" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/log" [label="github.com/containerd/containerd/log" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/log" target="_blank"]; +"github.com/containerd/containerd/log" -> "context"; +"github.com/containerd/containerd/log" -> "github.com/sirupsen/logrus"; +"github.com/containerd/containerd/log" -> "sync/atomic"; +"github.com/containerd/containerd/platforms" [label="github.com/containerd/containerd/platforms" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/platforms" target="_blank"]; +"github.com/containerd/containerd/platforms" -> "bufio"; +"github.com/containerd/containerd/platforms" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/platforms" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/platforms" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/platforms" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/platforms" -> "os"; +"github.com/containerd/containerd/platforms" -> "regexp"; +"github.com/containerd/containerd/platforms" -> "runtime"; +"github.com/containerd/containerd/platforms" -> "strconv"; +"github.com/containerd/containerd/platforms" -> "strings"; +"github.com/containerd/containerd/reference" [label="github.com/containerd/containerd/reference" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/reference" target="_blank"]; +"github.com/containerd/containerd/reference" -> "errors"; +"github.com/containerd/containerd/reference" -> "fmt"; +"github.com/containerd/containerd/reference" -> "github.com/opencontainers/go-digest"; +"github.com/containerd/containerd/reference" -> "net/url"; +"github.com/containerd/containerd/reference" -> "path"; +"github.com/containerd/containerd/reference" -> "regexp"; +"github.com/containerd/containerd/reference" -> "strings"; +"github.com/containerd/containerd/remotes" [label="github.com/containerd/containerd/remotes" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/remotes" target="_blank"]; +"github.com/containerd/containerd/remotes" -> "context"; +"github.com/containerd/containerd/remotes" -> "fmt"; +"github.com/containerd/containerd/remotes" -> "github.com/containerd/containerd/content"; +"github.com/containerd/containerd/remotes" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/remotes" -> "github.com/containerd/containerd/images"; +"github.com/containerd/containerd/remotes" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/remotes" -> "github.com/containerd/containerd/platforms"; +"github.com/containerd/containerd/remotes" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/remotes" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/remotes" -> "github.com/sirupsen/logrus"; +"github.com/containerd/containerd/remotes" -> "io"; +"github.com/containerd/containerd/remotes" -> "strings"; +"github.com/containerd/containerd/remotes" -> "sync"; +"github.com/containerd/containerd/remotes/docker" [label="github.com/containerd/containerd/remotes/docker" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/remotes/docker" target="_blank"]; +"github.com/containerd/containerd/remotes/docker" -> "bytes"; +"github.com/containerd/containerd/remotes/docker" -> "context"; +"github.com/containerd/containerd/remotes/docker" -> "encoding/base64"; +"github.com/containerd/containerd/remotes/docker" -> "encoding/json"; +"github.com/containerd/containerd/remotes/docker" -> "fmt"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/content"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/images"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/labels"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/reference"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/remotes"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/remotes/docker/schema1"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/containerd/containerd/version"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/opencontainers/go-digest"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/remotes/docker" -> "github.com/sirupsen/logrus"; +"github.com/containerd/containerd/remotes/docker" -> "golang.org/x/net/context/ctxhttp"; +"github.com/containerd/containerd/remotes/docker" -> "io"; +"github.com/containerd/containerd/remotes/docker" -> "io/ioutil"; +"github.com/containerd/containerd/remotes/docker" -> "net/http"; +"github.com/containerd/containerd/remotes/docker" -> "net/url"; +"github.com/containerd/containerd/remotes/docker" -> "path"; +"github.com/containerd/containerd/remotes/docker" -> "sort"; +"github.com/containerd/containerd/remotes/docker" -> "strings"; +"github.com/containerd/containerd/remotes/docker" -> "sync"; +"github.com/containerd/containerd/remotes/docker" -> "time"; +"github.com/containerd/containerd/remotes/docker/schema1" [label="github.com/containerd/containerd/remotes/docker/schema1" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/remotes/docker/schema1" target="_blank"]; +"github.com/containerd/containerd/remotes/docker/schema1" -> "bytes"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "context"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "encoding/base64"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "encoding/json"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "fmt"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/archive/compression"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/content"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/errdefs"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/images"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/log"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/containerd/containerd/remotes"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/opencontainers/go-digest"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/opencontainers/image-spec/specs-go"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "github.com/pkg/errors"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "golang.org/x/sync/errgroup"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "io"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "io/ioutil"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "strconv"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "strings"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "sync"; +"github.com/containerd/containerd/remotes/docker/schema1" -> "time"; +"github.com/containerd/containerd/version" [label="github.com/containerd/containerd/version" color="paleturquoise" URL="https://godoc.org/github.com/containerd/containerd/version" target="_blank"]; +"github.com/docker/distribution/registry/api/errcode" [label="github.com/docker/distribution/registry/api/errcode" color="palegoldenrod" URL="https://godoc.org/github.com/docker/distribution/registry/api/errcode" target="_blank"]; +"github.com/docker/distribution/registry/api/errcode" -> "encoding/json"; +"github.com/docker/distribution/registry/api/errcode" -> "fmt"; +"github.com/docker/distribution/registry/api/errcode" -> "net/http"; +"github.com/docker/distribution/registry/api/errcode" -> "sort"; +"github.com/docker/distribution/registry/api/errcode" -> "strings"; +"github.com/docker/distribution/registry/api/errcode" -> "sync"; +"github.com/golang/protobuf/proto" [label="github.com/golang/protobuf/proto" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/proto" target="_blank"]; +"github.com/golang/protobuf/proto" -> "bufio"; +"github.com/golang/protobuf/proto" -> "bytes"; +"github.com/golang/protobuf/proto" -> "encoding"; +"github.com/golang/protobuf/proto" -> "encoding/json"; +"github.com/golang/protobuf/proto" -> "errors"; +"github.com/golang/protobuf/proto" -> "fmt"; +"github.com/golang/protobuf/proto" -> "io"; +"github.com/golang/protobuf/proto" -> "log"; +"github.com/golang/protobuf/proto" -> "math"; +"github.com/golang/protobuf/proto" -> "os"; +"github.com/golang/protobuf/proto" -> "reflect"; +"github.com/golang/protobuf/proto" -> "sort"; +"github.com/golang/protobuf/proto" -> "strconv"; +"github.com/golang/protobuf/proto" -> "strings"; +"github.com/golang/protobuf/proto" -> "sync"; +"github.com/golang/protobuf/proto" -> "sync/atomic"; +"github.com/golang/protobuf/proto" -> "unicode/utf8"; +"github.com/golang/protobuf/proto" -> "unsafe"; +"github.com/golang/protobuf/ptypes" [label="github.com/golang/protobuf/ptypes" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/ptypes" target="_blank"]; +"github.com/golang/protobuf/ptypes" -> "errors"; +"github.com/golang/protobuf/ptypes" -> "fmt"; +"github.com/golang/protobuf/ptypes" -> "github.com/golang/protobuf/proto"; +"github.com/golang/protobuf/ptypes" -> "github.com/golang/protobuf/ptypes/any"; +"github.com/golang/protobuf/ptypes" -> "github.com/golang/protobuf/ptypes/duration"; +"github.com/golang/protobuf/ptypes" -> "github.com/golang/protobuf/ptypes/timestamp"; +"github.com/golang/protobuf/ptypes" -> "reflect"; +"github.com/golang/protobuf/ptypes" -> "strings"; +"github.com/golang/protobuf/ptypes" -> "time"; +"github.com/golang/protobuf/ptypes/any" [label="github.com/golang/protobuf/ptypes/any" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/ptypes/any" target="_blank"]; +"github.com/golang/protobuf/ptypes/any" -> "fmt"; +"github.com/golang/protobuf/ptypes/any" -> "github.com/golang/protobuf/proto"; +"github.com/golang/protobuf/ptypes/any" -> "math"; +"github.com/golang/protobuf/ptypes/duration" [label="github.com/golang/protobuf/ptypes/duration" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/ptypes/duration" target="_blank"]; +"github.com/golang/protobuf/ptypes/duration" -> "fmt"; +"github.com/golang/protobuf/ptypes/duration" -> "github.com/golang/protobuf/proto"; +"github.com/golang/protobuf/ptypes/duration" -> "math"; +"github.com/golang/protobuf/ptypes/timestamp" [label="github.com/golang/protobuf/ptypes/timestamp" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/ptypes/timestamp" target="_blank"]; +"github.com/golang/protobuf/ptypes/timestamp" -> "fmt"; +"github.com/golang/protobuf/ptypes/timestamp" -> "github.com/golang/protobuf/proto"; +"github.com/golang/protobuf/ptypes/timestamp" -> "math"; +"github.com/opencontainers/go-digest" [label="github.com/opencontainers/go-digest" color="palegoldenrod" URL="https://godoc.org/github.com/opencontainers/go-digest" target="_blank"]; +"github.com/opencontainers/go-digest" -> "crypto"; +"github.com/opencontainers/go-digest" -> "fmt"; +"github.com/opencontainers/go-digest" -> "hash"; +"github.com/opencontainers/go-digest" -> "io"; +"github.com/opencontainers/go-digest" -> "regexp"; +"github.com/opencontainers/go-digest" -> "strings"; +"github.com/opencontainers/image-spec/specs-go" [label="github.com/opencontainers/image-spec/specs-go" color="palegoldenrod" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go" -> "fmt"; +"github.com/opencontainers/image-spec/specs-go/v1" [label="github.com/opencontainers/image-spec/specs-go/v1" color="palegoldenrod" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go/v1" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/go-digest"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/image-spec/specs-go"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "time"; +"github.com/pkg/errors" [label="github.com/pkg/errors" color="palegoldenrod" URL="https://godoc.org/github.com/pkg/errors" target="_blank"]; +"github.com/pkg/errors" -> "fmt"; +"github.com/pkg/errors" -> "io"; +"github.com/pkg/errors" -> "path"; +"github.com/pkg/errors" -> "runtime"; +"github.com/pkg/errors" -> "strings"; +"github.com/sirupsen/logrus" [label="github.com/sirupsen/logrus" color="palegoldenrod" URL="https://godoc.org/github.com/sirupsen/logrus" target="_blank"]; +"github.com/sirupsen/logrus" -> "bufio"; +"github.com/sirupsen/logrus" -> "bytes"; +"github.com/sirupsen/logrus" -> "context"; +"github.com/sirupsen/logrus" -> "encoding/json"; +"github.com/sirupsen/logrus" -> "fmt"; +"github.com/sirupsen/logrus" -> "golang.org/x/sys/unix"; +"github.com/sirupsen/logrus" -> "io"; +"github.com/sirupsen/logrus" -> "log"; +"github.com/sirupsen/logrus" -> "os"; +"github.com/sirupsen/logrus" -> "reflect"; +"github.com/sirupsen/logrus" -> "runtime"; +"github.com/sirupsen/logrus" -> "sort"; +"github.com/sirupsen/logrus" -> "strings"; +"github.com/sirupsen/logrus" -> "sync"; +"github.com/sirupsen/logrus" -> "sync/atomic"; +"github.com/sirupsen/logrus" -> "time"; +"golang.org/x/net/context/ctxhttp" [label="golang.org/x/net/context/ctxhttp" color="palegoldenrod" URL="https://godoc.org/golang.org/x/net/context/ctxhttp" target="_blank"]; +"golang.org/x/net/context/ctxhttp" -> "context"; +"golang.org/x/net/context/ctxhttp" -> "io"; +"golang.org/x/net/context/ctxhttp" -> "net/http"; +"golang.org/x/net/context/ctxhttp" -> "net/url"; +"golang.org/x/net/context/ctxhttp" -> "strings"; +"golang.org/x/sync/errgroup" [label="golang.org/x/sync/errgroup" color="palegoldenrod" URL="https://godoc.org/golang.org/x/sync/errgroup" target="_blank"]; +"golang.org/x/sync/errgroup" -> "context"; +"golang.org/x/sync/errgroup" -> "sync"; +"golang.org/x/sync/semaphore" [label="golang.org/x/sync/semaphore" color="palegoldenrod" URL="https://godoc.org/golang.org/x/sync/semaphore" target="_blank"]; +"golang.org/x/sync/semaphore" -> "container/list"; +"golang.org/x/sync/semaphore" -> "context"; +"golang.org/x/sync/semaphore" -> "sync"; +"golang.org/x/sys/unix" [label="golang.org/x/sys/unix" color="paleturquoise" URL="https://godoc.org/golang.org/x/sys/unix" target="_blank"]; +"golang.org/x/sys/unix" -> "bytes"; +"golang.org/x/sys/unix" -> "runtime"; +"golang.org/x/sys/unix" -> "sort"; +"golang.org/x/sys/unix" -> "strings"; +"golang.org/x/sys/unix" -> "sync"; +"golang.org/x/sys/unix" -> "syscall"; +"golang.org/x/sys/unix" -> "time"; +"golang.org/x/sys/unix" -> "unsafe"; +"google.golang.org/genproto/googleapis/rpc/status" [label="google.golang.org/genproto/googleapis/rpc/status" color="paleturquoise" URL="https://godoc.org/google.golang.org/genproto/googleapis/rpc/status" target="_blank"]; +"google.golang.org/genproto/googleapis/rpc/status" -> "fmt"; +"google.golang.org/genproto/googleapis/rpc/status" -> "github.com/golang/protobuf/proto"; +"google.golang.org/genproto/googleapis/rpc/status" -> "github.com/golang/protobuf/ptypes/any"; +"google.golang.org/genproto/googleapis/rpc/status" -> "math"; +"google.golang.org/grpc/codes" [label="google.golang.org/grpc/codes" color="palegoldenrod" URL="https://godoc.org/google.golang.org/grpc/codes" target="_blank"]; +"google.golang.org/grpc/codes" -> "fmt"; +"google.golang.org/grpc/codes" -> "strconv"; +"google.golang.org/grpc/connectivity" [label="google.golang.org/grpc/connectivity" color="paleturquoise" URL="https://godoc.org/google.golang.org/grpc/connectivity" target="_blank"]; +"google.golang.org/grpc/connectivity" -> "context"; +"google.golang.org/grpc/connectivity" -> "google.golang.org/grpc/grpclog"; +"google.golang.org/grpc/grpclog" [label="google.golang.org/grpc/grpclog" color="paleturquoise" URL="https://godoc.org/google.golang.org/grpc/grpclog" target="_blank"]; +"google.golang.org/grpc/grpclog" -> "io"; +"google.golang.org/grpc/grpclog" -> "io/ioutil"; +"google.golang.org/grpc/grpclog" -> "log"; +"google.golang.org/grpc/grpclog" -> "os"; +"google.golang.org/grpc/grpclog" -> "strconv"; +"google.golang.org/grpc/internal" [label="google.golang.org/grpc/internal" color="paleturquoise" URL="https://godoc.org/google.golang.org/grpc/internal" target="_blank"]; +"google.golang.org/grpc/internal" -> "context"; +"google.golang.org/grpc/internal" -> "google.golang.org/grpc/connectivity"; +"google.golang.org/grpc/internal" -> "time"; +"google.golang.org/grpc/status" [label="google.golang.org/grpc/status" color="palegoldenrod" URL="https://godoc.org/google.golang.org/grpc/status" target="_blank"]; +"google.golang.org/grpc/status" -> "context"; +"google.golang.org/grpc/status" -> "errors"; +"google.golang.org/grpc/status" -> "fmt"; +"google.golang.org/grpc/status" -> "github.com/golang/protobuf/proto"; +"google.golang.org/grpc/status" -> "github.com/golang/protobuf/ptypes"; +"google.golang.org/grpc/status" -> "google.golang.org/genproto/googleapis/rpc/status"; +"google.golang.org/grpc/status" -> "google.golang.org/grpc/codes"; +"google.golang.org/grpc/status" -> "google.golang.org/grpc/internal"; +"hash" [label="hash" color="palegreen" URL="https://godoc.org/hash" target="_blank"]; +"io" [label="io" color="palegreen" URL="https://godoc.org/io" target="_blank"]; +"io/ioutil" [label="io/ioutil" color="palegreen" URL="https://godoc.org/io/ioutil" target="_blank"]; +"log" [label="log" color="palegreen" URL="https://godoc.org/log" target="_blank"]; +"math" [label="math" color="palegreen" URL="https://godoc.org/math" target="_blank"]; +"math/rand" [label="math/rand" color="palegreen" URL="https://godoc.org/math/rand" target="_blank"]; +"net/http" [label="net/http" color="palegreen" URL="https://godoc.org/net/http" target="_blank"]; +"net/url" [label="net/url" color="palegreen" URL="https://godoc.org/net/url" target="_blank"]; +"os" [label="os" color="palegreen" URL="https://godoc.org/os" target="_blank"]; +"os/exec" [label="os/exec" color="palegreen" URL="https://godoc.org/os/exec" target="_blank"]; +"path" [label="path" color="palegreen" URL="https://godoc.org/path" target="_blank"]; +"reflect" [label="reflect" color="palegreen" URL="https://godoc.org/reflect" target="_blank"]; +"regexp" [label="regexp" color="palegreen" URL="https://godoc.org/regexp" target="_blank"]; +"runtime" [label="runtime" color="palegreen" URL="https://godoc.org/runtime" target="_blank"]; +"sort" [label="sort" color="palegreen" URL="https://godoc.org/sort" target="_blank"]; +"strconv" [label="strconv" color="palegreen" URL="https://godoc.org/strconv" target="_blank"]; +"strings" [label="strings" color="palegreen" URL="https://godoc.org/strings" target="_blank"]; +"sync" [label="sync" color="palegreen" URL="https://godoc.org/sync" target="_blank"]; +"sync/atomic" [label="sync/atomic" color="palegreen" URL="https://godoc.org/sync/atomic" target="_blank"]; +"syscall" [label="syscall" color="palegreen" URL="https://godoc.org/syscall" target="_blank"]; +"time" [label="time" color="palegreen" URL="https://godoc.org/time" target="_blank"]; +"unicode/utf8" [label="unicode/utf8" color="palegreen" URL="https://godoc.org/unicode/utf8" target="_blank"]; +"unsafe" [label="unsafe" color="palegreen" URL="https://godoc.org/unsafe" target="_blank"]; +} diff --git a/images/dot/containers.dot b/images/dot/containers.dot new file mode 100644 index 0000000..3e53f84 --- /dev/null +++ b/images/dot/containers.dot @@ -0,0 +1,831 @@ +digraph godep { +rankdir="LR" +nodesep=0.4 +ranksep=0.8 +node [shape="box",style="rounded,filled"] +edge [arrowsize="0.5"] +"bufio" [label="bufio" color="palegreen" URL="https://godoc.org/bufio" target="_blank"]; +"bytes" [label="bytes" color="palegreen" URL="https://godoc.org/bytes" target="_blank"]; +"compress/bzip2" [label="compress/bzip2" color="palegreen" URL="https://godoc.org/compress/bzip2" target="_blank"]; +"compress/gzip" [label="compress/gzip" color="palegreen" URL="https://godoc.org/compress/gzip" target="_blank"]; +"context" [label="context" color="palegreen" URL="https://godoc.org/context" target="_blank"]; +"crypto" [label="crypto" color="palegreen" URL="https://godoc.org/crypto" target="_blank"]; +"crypto/ecdsa" [label="crypto/ecdsa" color="palegreen" URL="https://godoc.org/crypto/ecdsa" target="_blank"]; +"crypto/elliptic" [label="crypto/elliptic" color="palegreen" URL="https://godoc.org/crypto/elliptic" target="_blank"]; +"crypto/rand" [label="crypto/rand" color="palegreen" URL="https://godoc.org/crypto/rand" target="_blank"]; +"crypto/rsa" [label="crypto/rsa" color="palegreen" URL="https://godoc.org/crypto/rsa" target="_blank"]; +"crypto/sha256" [label="crypto/sha256" color="palegreen" URL="https://godoc.org/crypto/sha256" target="_blank"]; +"crypto/sha512" [label="crypto/sha512" color="palegreen" URL="https://godoc.org/crypto/sha512" target="_blank"]; +"crypto/tls" [label="crypto/tls" color="palegreen" URL="https://godoc.org/crypto/tls" target="_blank"]; +"crypto/x509" [label="crypto/x509" color="palegreen" URL="https://godoc.org/crypto/x509" target="_blank"]; +"crypto/x509/pkix" [label="crypto/x509/pkix" color="palegreen" URL="https://godoc.org/crypto/x509/pkix" target="_blank"]; +"encoding" [label="encoding" color="palegreen" URL="https://godoc.org/encoding" target="_blank"]; +"encoding/base32" [label="encoding/base32" color="palegreen" URL="https://godoc.org/encoding/base32" target="_blank"]; +"encoding/base64" [label="encoding/base64" color="palegreen" URL="https://godoc.org/encoding/base64" target="_blank"]; +"encoding/binary" [label="encoding/binary" color="palegreen" URL="https://godoc.org/encoding/binary" target="_blank"]; +"encoding/hex" [label="encoding/hex" color="palegreen" URL="https://godoc.org/encoding/hex" target="_blank"]; +"encoding/json" [label="encoding/json" color="palegreen" URL="https://godoc.org/encoding/json" target="_blank"]; +"encoding/pem" [label="encoding/pem" color="palegreen" URL="https://godoc.org/encoding/pem" target="_blank"]; +"errors" [label="errors" color="palegreen" URL="https://godoc.org/errors" target="_blank"]; +"expvar" [label="expvar" color="palegreen" URL="https://godoc.org/expvar" target="_blank"]; +"fmt" [label="fmt" color="palegreen" URL="https://godoc.org/fmt" target="_blank"]; +"github.com/BurntSushi/toml" [label="github.com/BurntSushi/toml" color="paleturquoise" URL="https://godoc.org/github.com/BurntSushi/toml" target="_blank"]; +"github.com/BurntSushi/toml" -> "bufio"; +"github.com/BurntSushi/toml" -> "encoding"; +"github.com/BurntSushi/toml" -> "errors"; +"github.com/BurntSushi/toml" -> "fmt"; +"github.com/BurntSushi/toml" -> "io"; +"github.com/BurntSushi/toml" -> "io/ioutil"; +"github.com/BurntSushi/toml" -> "math"; +"github.com/BurntSushi/toml" -> "reflect"; +"github.com/BurntSushi/toml" -> "sort"; +"github.com/BurntSushi/toml" -> "strconv"; +"github.com/BurntSushi/toml" -> "strings"; +"github.com/BurntSushi/toml" -> "sync"; +"github.com/BurntSushi/toml" -> "time"; +"github.com/BurntSushi/toml" -> "unicode"; +"github.com/BurntSushi/toml" -> "unicode/utf8"; +"github.com/beorn7/perks/quantile" [label="github.com/beorn7/perks/quantile" color="paleturquoise" URL="https://godoc.org/github.com/beorn7/perks/quantile" target="_blank"]; +"github.com/beorn7/perks/quantile" -> "math"; +"github.com/beorn7/perks/quantile" -> "sort"; +"github.com/cespare/xxhash/v2" [label="github.com/cespare/xxhash/v2" color="paleturquoise" URL="https://godoc.org/github.com/cespare/xxhash/v2" target="_blank"]; +"github.com/cespare/xxhash/v2" -> "encoding/binary"; +"github.com/cespare/xxhash/v2" -> "errors"; +"github.com/cespare/xxhash/v2" -> "math/bits"; +"github.com/cespare/xxhash/v2" -> "reflect"; +"github.com/cespare/xxhash/v2" -> "unsafe"; +"github.com/containers/image/docker" [label="github.com/containers/image/docker" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/docker" target="_blank"]; +"github.com/containers/image/docker" -> "bytes"; +"github.com/containers/image/docker" -> "context"; +"github.com/containers/image/docker" -> "crypto/rand"; +"github.com/containers/image/docker" -> "crypto/tls"; +"github.com/containers/image/docker" -> "encoding/json"; +"github.com/containers/image/docker" -> "errors"; +"github.com/containers/image/docker" -> "fmt"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/docker/policyconfiguration"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/image"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/manifest"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/pkg/blobinfocache/none"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/pkg/docker/config"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/pkg/sysregistriesv2"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/pkg/tlsclientconfig"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/transports"; +"github.com/containers/image/docker" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/docker" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/containers/image/docker" -> "github.com/docker/distribution/registry/api/v2"; +"github.com/containers/image/docker" -> "github.com/docker/distribution/registry/client"; +"github.com/containers/image/docker" -> "github.com/docker/go-connections/tlsconfig"; +"github.com/containers/image/docker" -> "github.com/ghodss/yaml"; +"github.com/containers/image/docker" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/docker" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containers/image/docker" -> "github.com/pkg/errors"; +"github.com/containers/image/docker" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/docker" -> "io"; +"github.com/containers/image/docker" -> "io/ioutil"; +"github.com/containers/image/docker" -> "mime"; +"github.com/containers/image/docker" -> "net/http"; +"github.com/containers/image/docker" -> "net/url"; +"github.com/containers/image/docker" -> "os"; +"github.com/containers/image/docker" -> "path"; +"github.com/containers/image/docker" -> "path/filepath"; +"github.com/containers/image/docker" -> "strconv"; +"github.com/containers/image/docker" -> "strings"; +"github.com/containers/image/docker" -> "sync"; +"github.com/containers/image/docker" -> "time"; +"github.com/containers/image/v5/docker/policyconfiguration" [label="github.com/containers/image/v5/docker/policyconfiguration" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/docker/policyconfiguration" target="_blank"]; +"github.com/containers/image/v5/docker/policyconfiguration" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/v5/docker/policyconfiguration" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/docker/policyconfiguration" -> "strings"; +"github.com/containers/image/v5/docker/reference" [label="github.com/containers/image/v5/docker/reference" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/docker/reference" target="_blank"]; +"github.com/containers/image/v5/docker/reference" -> "errors"; +"github.com/containers/image/v5/docker/reference" -> "fmt"; +"github.com/containers/image/v5/docker/reference" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/v5/docker/reference" -> "path"; +"github.com/containers/image/v5/docker/reference" -> "regexp"; +"github.com/containers/image/v5/docker/reference" -> "strings"; +"github.com/containers/image/v5/image" [label="github.com/containers/image/v5/image" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/image" target="_blank"]; +"github.com/containers/image/v5/image" -> "bytes"; +"github.com/containers/image/v5/image" -> "context"; +"github.com/containers/image/v5/image" -> "crypto/sha256"; +"github.com/containers/image/v5/image" -> "encoding/hex"; +"github.com/containers/image/v5/image" -> "encoding/json"; +"github.com/containers/image/v5/image" -> "fmt"; +"github.com/containers/image/v5/image" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/v5/image" -> "github.com/containers/image/v5/manifest"; +"github.com/containers/image/v5/image" -> "github.com/containers/image/v5/pkg/blobinfocache/none"; +"github.com/containers/image/v5/image" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/image" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/v5/image" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containers/image/v5/image" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/image" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/image" -> "io/ioutil"; +"github.com/containers/image/v5/image" -> "strings"; +"github.com/containers/image/v5/internal/pkg/keyctl" [label="github.com/containers/image/v5/internal/pkg/keyctl" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/internal/pkg/keyctl" target="_blank"]; +"github.com/containers/image/v5/internal/pkg/keyctl" -> "golang.org/x/sys/unix"; +"github.com/containers/image/v5/internal/pkg/keyctl" -> "unsafe"; +"github.com/containers/image/v5/manifest" [label="github.com/containers/image/v5/manifest" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/manifest" target="_blank"]; +"github.com/containers/image/v5/manifest" -> "encoding/json"; +"github.com/containers/image/v5/manifest" -> "fmt"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/image/v5/pkg/compression"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/image/v5/pkg/strslice"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/libtrust"; +"github.com/containers/image/v5/manifest" -> "github.com/containers/ocicrypt/spec"; +"github.com/containers/image/v5/manifest" -> "github.com/docker/docker/api/types/versions"; +"github.com/containers/image/v5/manifest" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/v5/manifest" -> "github.com/opencontainers/image-spec/specs-go"; +"github.com/containers/image/v5/manifest" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containers/image/v5/manifest" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/manifest" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/manifest" -> "regexp"; +"github.com/containers/image/v5/manifest" -> "runtime"; +"github.com/containers/image/v5/manifest" -> "strings"; +"github.com/containers/image/v5/manifest" -> "time"; +"github.com/containers/image/v5/pkg/blobinfocache/none" [label="github.com/containers/image/v5/pkg/blobinfocache/none" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/blobinfocache/none" target="_blank"]; +"github.com/containers/image/v5/pkg/blobinfocache/none" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/pkg/blobinfocache/none" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/v5/pkg/compression" [label="github.com/containers/image/v5/pkg/compression" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/compression" target="_blank"]; +"github.com/containers/image/v5/pkg/compression" -> "bytes"; +"github.com/containers/image/v5/pkg/compression" -> "compress/bzip2"; +"github.com/containers/image/v5/pkg/compression" -> "fmt"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/containers/image/v5/pkg/compression/internal"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/containers/image/v5/pkg/compression/types"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/klauspost/compress/zstd"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/klauspost/pgzip"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/pkg/compression" -> "github.com/ulikunitz/xz"; +"github.com/containers/image/v5/pkg/compression" -> "io"; +"github.com/containers/image/v5/pkg/compression" -> "io/ioutil"; +"github.com/containers/image/v5/pkg/compression/internal" [label="github.com/containers/image/v5/pkg/compression/internal" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/compression/internal" target="_blank"]; +"github.com/containers/image/v5/pkg/compression/internal" -> "io"; +"github.com/containers/image/v5/pkg/compression/types" [label="github.com/containers/image/v5/pkg/compression/types" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/compression/types" target="_blank"]; +"github.com/containers/image/v5/pkg/compression/types" -> "github.com/containers/image/v5/pkg/compression/internal"; +"github.com/containers/image/v5/pkg/docker/config" [label="github.com/containers/image/v5/pkg/docker/config" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/docker/config" target="_blank"]; +"github.com/containers/image/v5/pkg/docker/config" -> "encoding/base64"; +"github.com/containers/image/v5/pkg/docker/config" -> "encoding/json"; +"github.com/containers/image/v5/pkg/docker/config" -> "fmt"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/containers/image/v5/internal/pkg/keyctl"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/docker/docker-credential-helpers/client"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/docker/docker-credential-helpers/credentials"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/docker/docker/pkg/homedir"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/pkg/docker/config" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/pkg/docker/config" -> "io/ioutil"; +"github.com/containers/image/v5/pkg/docker/config" -> "os"; +"github.com/containers/image/v5/pkg/docker/config" -> "path/filepath"; +"github.com/containers/image/v5/pkg/docker/config" -> "strings"; +"github.com/containers/image/v5/pkg/strslice" [label="github.com/containers/image/v5/pkg/strslice" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/strslice" target="_blank"]; +"github.com/containers/image/v5/pkg/strslice" -> "encoding/json"; +"github.com/containers/image/v5/pkg/sysregistriesv2" [label="github.com/containers/image/v5/pkg/sysregistriesv2" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/sysregistriesv2" target="_blank"]; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "fmt"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "github.com/BurntSushi/toml"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "io/ioutil"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "os"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "path/filepath"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "regexp"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "strings"; +"github.com/containers/image/v5/pkg/sysregistriesv2" -> "sync"; +"github.com/containers/image/v5/pkg/tlsclientconfig" [label="github.com/containers/image/v5/pkg/tlsclientconfig" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/pkg/tlsclientconfig" target="_blank"]; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "crypto/tls"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "github.com/docker/go-connections/sockets"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "github.com/docker/go-connections/tlsconfig"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "github.com/pkg/errors"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "github.com/sirupsen/logrus"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "io/ioutil"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "net"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "net/http"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "os"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "path/filepath"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "strings"; +"github.com/containers/image/v5/pkg/tlsclientconfig" -> "time"; +"github.com/containers/image/v5/transports" [label="github.com/containers/image/v5/transports" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/transports" target="_blank"]; +"github.com/containers/image/v5/transports" -> "fmt"; +"github.com/containers/image/v5/transports" -> "github.com/containers/image/v5/types"; +"github.com/containers/image/v5/transports" -> "sort"; +"github.com/containers/image/v5/transports" -> "sync"; +"github.com/containers/image/v5/types" [label="github.com/containers/image/v5/types" color="paleturquoise" URL="https://godoc.org/github.com/containers/image/v5/types" target="_blank"]; +"github.com/containers/image/v5/types" -> "context"; +"github.com/containers/image/v5/types" -> "github.com/containers/image/v5/docker/reference"; +"github.com/containers/image/v5/types" -> "github.com/containers/image/v5/pkg/compression/types"; +"github.com/containers/image/v5/types" -> "github.com/opencontainers/go-digest"; +"github.com/containers/image/v5/types" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/containers/image/v5/types" -> "io"; +"github.com/containers/image/v5/types" -> "time"; +"github.com/containers/libtrust" [label="github.com/containers/libtrust" color="paleturquoise" URL="https://godoc.org/github.com/containers/libtrust" target="_blank"]; +"github.com/containers/libtrust" -> "bytes"; +"github.com/containers/libtrust" -> "crypto"; +"github.com/containers/libtrust" -> "crypto/ecdsa"; +"github.com/containers/libtrust" -> "crypto/elliptic"; +"github.com/containers/libtrust" -> "crypto/rand"; +"github.com/containers/libtrust" -> "crypto/rsa"; +"github.com/containers/libtrust" -> "crypto/sha256"; +"github.com/containers/libtrust" -> "crypto/sha512"; +"github.com/containers/libtrust" -> "crypto/tls"; +"github.com/containers/libtrust" -> "crypto/x509"; +"github.com/containers/libtrust" -> "crypto/x509/pkix"; +"github.com/containers/libtrust" -> "encoding/base32"; +"github.com/containers/libtrust" -> "encoding/base64"; +"github.com/containers/libtrust" -> "encoding/binary"; +"github.com/containers/libtrust" -> "encoding/json"; +"github.com/containers/libtrust" -> "encoding/pem"; +"github.com/containers/libtrust" -> "errors"; +"github.com/containers/libtrust" -> "fmt"; +"github.com/containers/libtrust" -> "io"; +"github.com/containers/libtrust" -> "io/ioutil"; +"github.com/containers/libtrust" -> "math/big"; +"github.com/containers/libtrust" -> "net"; +"github.com/containers/libtrust" -> "net/url"; +"github.com/containers/libtrust" -> "os"; +"github.com/containers/libtrust" -> "path"; +"github.com/containers/libtrust" -> "path/filepath"; +"github.com/containers/libtrust" -> "sort"; +"github.com/containers/libtrust" -> "strings"; +"github.com/containers/libtrust" -> "sync"; +"github.com/containers/libtrust" -> "time"; +"github.com/containers/libtrust" -> "unicode"; +"github.com/containers/ocicrypt/spec" [label="github.com/containers/ocicrypt/spec" color="paleturquoise" URL="https://godoc.org/github.com/containers/ocicrypt/spec" target="_blank"]; +"github.com/docker/distribution" [label="github.com/docker/distribution" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution" target="_blank"]; +"github.com/docker/distribution" -> "context"; +"github.com/docker/distribution" -> "errors"; +"github.com/docker/distribution" -> "fmt"; +"github.com/docker/distribution" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/docker/distribution" -> "io"; +"github.com/docker/distribution" -> "mime"; +"github.com/docker/distribution" -> "net/http"; +"github.com/docker/distribution" -> "strings"; +"github.com/docker/distribution" -> "time"; +"github.com/docker/distribution/digestset" [label="github.com/docker/distribution/digestset" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/digestset" target="_blank"]; +"github.com/docker/distribution/digestset" -> "errors"; +"github.com/docker/distribution/digestset" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/digestset" -> "sort"; +"github.com/docker/distribution/digestset" -> "strings"; +"github.com/docker/distribution/digestset" -> "sync"; +"github.com/docker/distribution/metrics" [label="github.com/docker/distribution/metrics" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/metrics" target="_blank"]; +"github.com/docker/distribution/metrics" -> "github.com/docker/go-metrics"; +"github.com/docker/distribution/reference" [label="github.com/docker/distribution/reference" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/reference" target="_blank"]; +"github.com/docker/distribution/reference" -> "errors"; +"github.com/docker/distribution/reference" -> "fmt"; +"github.com/docker/distribution/reference" -> "github.com/docker/distribution/digestset"; +"github.com/docker/distribution/reference" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/reference" -> "path"; +"github.com/docker/distribution/reference" -> "regexp"; +"github.com/docker/distribution/reference" -> "strings"; +"github.com/docker/distribution/registry/api/errcode" [label="github.com/docker/distribution/registry/api/errcode" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/api/errcode" target="_blank"]; +"github.com/docker/distribution/registry/api/errcode" -> "encoding/json"; +"github.com/docker/distribution/registry/api/errcode" -> "fmt"; +"github.com/docker/distribution/registry/api/errcode" -> "net/http"; +"github.com/docker/distribution/registry/api/errcode" -> "sort"; +"github.com/docker/distribution/registry/api/errcode" -> "strings"; +"github.com/docker/distribution/registry/api/errcode" -> "sync"; +"github.com/docker/distribution/registry/api/v2" [label="github.com/docker/distribution/registry/api/v2" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/api/v2" target="_blank"]; +"github.com/docker/distribution/registry/api/v2" -> "fmt"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/gorilla/mux"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/api/v2" -> "net/http"; +"github.com/docker/distribution/registry/api/v2" -> "net/url"; +"github.com/docker/distribution/registry/api/v2" -> "regexp"; +"github.com/docker/distribution/registry/api/v2" -> "strings"; +"github.com/docker/distribution/registry/api/v2" -> "unicode"; +"github.com/docker/distribution/registry/client" [label="github.com/docker/distribution/registry/client" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client" target="_blank"]; +"github.com/docker/distribution/registry/client" -> "bytes"; +"github.com/docker/distribution/registry/client" -> "context"; +"github.com/docker/distribution/registry/client" -> "encoding/json"; +"github.com/docker/distribution/registry/client" -> "errors"; +"github.com/docker/distribution/registry/client" -> "fmt"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/api/v2"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/client/auth/challenge"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/client/transport"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/storage/cache"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/storage/cache/memory"; +"github.com/docker/distribution/registry/client" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/client" -> "io"; +"github.com/docker/distribution/registry/client" -> "io/ioutil"; +"github.com/docker/distribution/registry/client" -> "net/http"; +"github.com/docker/distribution/registry/client" -> "net/url"; +"github.com/docker/distribution/registry/client" -> "strconv"; +"github.com/docker/distribution/registry/client" -> "strings"; +"github.com/docker/distribution/registry/client" -> "time"; +"github.com/docker/distribution/registry/client/auth/challenge" [label="github.com/docker/distribution/registry/client/auth/challenge" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client/auth/challenge" target="_blank"]; +"github.com/docker/distribution/registry/client/auth/challenge" -> "fmt"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "net/http"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "net/url"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "strings"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "sync"; +"github.com/docker/distribution/registry/client/transport" [label="github.com/docker/distribution/registry/client/transport" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client/transport" target="_blank"]; +"github.com/docker/distribution/registry/client/transport" -> "errors"; +"github.com/docker/distribution/registry/client/transport" -> "fmt"; +"github.com/docker/distribution/registry/client/transport" -> "io"; +"github.com/docker/distribution/registry/client/transport" -> "net/http"; +"github.com/docker/distribution/registry/client/transport" -> "regexp"; +"github.com/docker/distribution/registry/client/transport" -> "strconv"; +"github.com/docker/distribution/registry/client/transport" -> "sync"; +"github.com/docker/distribution/registry/storage/cache" [label="github.com/docker/distribution/registry/storage/cache" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/storage/cache" target="_blank"]; +"github.com/docker/distribution/registry/storage/cache" -> "context"; +"github.com/docker/distribution/registry/storage/cache" -> "fmt"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/docker/distribution/metrics"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/storage/cache/memory" [label="github.com/docker/distribution/registry/storage/cache/memory" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/storage/cache/memory" target="_blank"]; +"github.com/docker/distribution/registry/storage/cache/memory" -> "context"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution/registry/storage/cache"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "sync"; +"github.com/docker/docker-credential-helpers/client" [label="github.com/docker/docker-credential-helpers/client" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker-credential-helpers/client" target="_blank"]; +"github.com/docker/docker-credential-helpers/client" -> "bytes"; +"github.com/docker/docker-credential-helpers/client" -> "encoding/json"; +"github.com/docker/docker-credential-helpers/client" -> "fmt"; +"github.com/docker/docker-credential-helpers/client" -> "github.com/docker/docker-credential-helpers/credentials"; +"github.com/docker/docker-credential-helpers/client" -> "io"; +"github.com/docker/docker-credential-helpers/client" -> "os"; +"github.com/docker/docker-credential-helpers/client" -> "os/exec"; +"github.com/docker/docker-credential-helpers/client" -> "strings"; +"github.com/docker/docker-credential-helpers/credentials" [label="github.com/docker/docker-credential-helpers/credentials" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker-credential-helpers/credentials" target="_blank"]; +"github.com/docker/docker-credential-helpers/credentials" -> "bufio"; +"github.com/docker/docker-credential-helpers/credentials" -> "bytes"; +"github.com/docker/docker-credential-helpers/credentials" -> "encoding/json"; +"github.com/docker/docker-credential-helpers/credentials" -> "fmt"; +"github.com/docker/docker-credential-helpers/credentials" -> "io"; +"github.com/docker/docker-credential-helpers/credentials" -> "os"; +"github.com/docker/docker-credential-helpers/credentials" -> "strings"; +"github.com/docker/docker/api/types/versions" [label="github.com/docker/docker/api/types/versions" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/api/types/versions" target="_blank"]; +"github.com/docker/docker/api/types/versions" -> "strconv"; +"github.com/docker/docker/api/types/versions" -> "strings"; +"github.com/docker/docker/pkg/homedir" [label="github.com/docker/docker/pkg/homedir" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/pkg/homedir" target="_blank"]; +"github.com/docker/docker/pkg/homedir" -> "github.com/docker/docker/pkg/idtools"; +"github.com/docker/docker/pkg/homedir" -> "github.com/opencontainers/runc/libcontainer/user"; +"github.com/docker/docker/pkg/homedir" -> "os"; +"github.com/docker/docker/pkg/idtools" [label="github.com/docker/docker/pkg/idtools" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/pkg/idtools" target="_blank"]; +"github.com/docker/docker/pkg/idtools" -> "bufio"; +"github.com/docker/docker/pkg/idtools" -> "bytes"; +"github.com/docker/docker/pkg/idtools" -> "fmt"; +"github.com/docker/docker/pkg/idtools" -> "github.com/docker/docker/pkg/system"; +"github.com/docker/docker/pkg/idtools" -> "github.com/opencontainers/runc/libcontainer/user"; +"github.com/docker/docker/pkg/idtools" -> "io"; +"github.com/docker/docker/pkg/idtools" -> "os"; +"github.com/docker/docker/pkg/idtools" -> "os/exec"; +"github.com/docker/docker/pkg/idtools" -> "path/filepath"; +"github.com/docker/docker/pkg/idtools" -> "regexp"; +"github.com/docker/docker/pkg/idtools" -> "sort"; +"github.com/docker/docker/pkg/idtools" -> "strconv"; +"github.com/docker/docker/pkg/idtools" -> "strings"; +"github.com/docker/docker/pkg/idtools" -> "sync"; +"github.com/docker/docker/pkg/idtools" -> "syscall"; +"github.com/docker/docker/pkg/mount" [label="github.com/docker/docker/pkg/mount" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/pkg/mount" target="_blank"]; +"github.com/docker/docker/pkg/mount" -> "bufio"; +"github.com/docker/docker/pkg/mount" -> "fmt"; +"github.com/docker/docker/pkg/mount" -> "github.com/pkg/errors"; +"github.com/docker/docker/pkg/mount" -> "github.com/sirupsen/logrus"; +"github.com/docker/docker/pkg/mount" -> "golang.org/x/sys/unix"; +"github.com/docker/docker/pkg/mount" -> "io"; +"github.com/docker/docker/pkg/mount" -> "os"; +"github.com/docker/docker/pkg/mount" -> "sort"; +"github.com/docker/docker/pkg/mount" -> "strconv"; +"github.com/docker/docker/pkg/mount" -> "strings"; +"github.com/docker/docker/pkg/system" [label="github.com/docker/docker/pkg/system" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/pkg/system" target="_blank"]; +"github.com/docker/docker/pkg/system" -> "bufio"; +"github.com/docker/docker/pkg/system" -> "errors"; +"github.com/docker/docker/pkg/system" -> "fmt"; +"github.com/docker/docker/pkg/system" -> "github.com/docker/docker/pkg/mount"; +"github.com/docker/docker/pkg/system" -> "github.com/docker/go-units"; +"github.com/docker/docker/pkg/system" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/docker/docker/pkg/system" -> "github.com/pkg/errors"; +"github.com/docker/docker/pkg/system" -> "golang.org/x/sys/unix"; +"github.com/docker/docker/pkg/system" -> "io"; +"github.com/docker/docker/pkg/system" -> "io/ioutil"; +"github.com/docker/docker/pkg/system" -> "os"; +"github.com/docker/docker/pkg/system" -> "os/exec"; +"github.com/docker/docker/pkg/system" -> "path/filepath"; +"github.com/docker/docker/pkg/system" -> "runtime"; +"github.com/docker/docker/pkg/system" -> "strconv"; +"github.com/docker/docker/pkg/system" -> "strings"; +"github.com/docker/docker/pkg/system" -> "syscall"; +"github.com/docker/docker/pkg/system" -> "time"; +"github.com/docker/docker/pkg/system" -> "unsafe"; +"github.com/docker/go-connections/sockets" [label="github.com/docker/go-connections/sockets" color="paleturquoise" URL="https://godoc.org/github.com/docker/go-connections/sockets" target="_blank"]; +"github.com/docker/go-connections/sockets" -> "crypto/tls"; +"github.com/docker/go-connections/sockets" -> "errors"; +"github.com/docker/go-connections/sockets" -> "fmt"; +"github.com/docker/go-connections/sockets" -> "golang.org/x/net/proxy"; +"github.com/docker/go-connections/sockets" -> "net"; +"github.com/docker/go-connections/sockets" -> "net/http"; +"github.com/docker/go-connections/sockets" -> "net/url"; +"github.com/docker/go-connections/sockets" -> "os"; +"github.com/docker/go-connections/sockets" -> "strings"; +"github.com/docker/go-connections/sockets" -> "sync"; +"github.com/docker/go-connections/sockets" -> "syscall"; +"github.com/docker/go-connections/sockets" -> "time"; +"github.com/docker/go-connections/tlsconfig" [label="github.com/docker/go-connections/tlsconfig" color="paleturquoise" URL="https://godoc.org/github.com/docker/go-connections/tlsconfig" target="_blank"]; +"github.com/docker/go-connections/tlsconfig" -> "crypto/tls"; +"github.com/docker/go-connections/tlsconfig" -> "crypto/x509"; +"github.com/docker/go-connections/tlsconfig" -> "encoding/pem"; +"github.com/docker/go-connections/tlsconfig" -> "fmt"; +"github.com/docker/go-connections/tlsconfig" -> "github.com/pkg/errors"; +"github.com/docker/go-connections/tlsconfig" -> "io/ioutil"; +"github.com/docker/go-connections/tlsconfig" -> "os"; +"github.com/docker/go-connections/tlsconfig" -> "runtime"; +"github.com/docker/go-metrics" [label="github.com/docker/go-metrics" color="paleturquoise" URL="https://godoc.org/github.com/docker/go-metrics" target="_blank"]; +"github.com/docker/go-metrics" -> "fmt"; +"github.com/docker/go-metrics" -> "github.com/prometheus/client_golang/prometheus"; +"github.com/docker/go-metrics" -> "github.com/prometheus/client_golang/prometheus/promhttp"; +"github.com/docker/go-metrics" -> "net/http"; +"github.com/docker/go-metrics" -> "sync"; +"github.com/docker/go-metrics" -> "time"; +"github.com/docker/go-units" [label="github.com/docker/go-units" color="palegoldenrod" URL="https://godoc.org/github.com/docker/go-units" target="_blank"]; +"github.com/docker/go-units" -> "fmt"; +"github.com/docker/go-units" -> "regexp"; +"github.com/docker/go-units" -> "strconv"; +"github.com/docker/go-units" -> "strings"; +"github.com/docker/go-units" -> "time"; +"github.com/ghodss/yaml" [label="github.com/ghodss/yaml" color="paleturquoise" URL="https://godoc.org/github.com/ghodss/yaml" target="_blank"]; +"github.com/ghodss/yaml" -> "bytes"; +"github.com/ghodss/yaml" -> "encoding"; +"github.com/ghodss/yaml" -> "encoding/json"; +"github.com/ghodss/yaml" -> "fmt"; +"github.com/ghodss/yaml" -> "gopkg.in/yaml.v2"; +"github.com/ghodss/yaml" -> "reflect"; +"github.com/ghodss/yaml" -> "sort"; +"github.com/ghodss/yaml" -> "strconv"; +"github.com/ghodss/yaml" -> "strings"; +"github.com/ghodss/yaml" -> "sync"; +"github.com/ghodss/yaml" -> "unicode"; +"github.com/ghodss/yaml" -> "unicode/utf8"; +"github.com/golang/protobuf/proto" [label="github.com/golang/protobuf/proto" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/proto" target="_blank"]; +"github.com/golang/protobuf/proto" -> "bufio"; +"github.com/golang/protobuf/proto" -> "bytes"; +"github.com/golang/protobuf/proto" -> "encoding"; +"github.com/golang/protobuf/proto" -> "encoding/json"; +"github.com/golang/protobuf/proto" -> "errors"; +"github.com/golang/protobuf/proto" -> "fmt"; +"github.com/golang/protobuf/proto" -> "io"; +"github.com/golang/protobuf/proto" -> "log"; +"github.com/golang/protobuf/proto" -> "math"; +"github.com/golang/protobuf/proto" -> "reflect"; +"github.com/golang/protobuf/proto" -> "sort"; +"github.com/golang/protobuf/proto" -> "strconv"; +"github.com/golang/protobuf/proto" -> "strings"; +"github.com/golang/protobuf/proto" -> "sync"; +"github.com/golang/protobuf/proto" -> "sync/atomic"; +"github.com/golang/protobuf/proto" -> "unicode/utf8"; +"github.com/golang/protobuf/proto" -> "unsafe"; +"github.com/gorilla/mux" [label="github.com/gorilla/mux" color="paleturquoise" URL="https://godoc.org/github.com/gorilla/mux" target="_blank"]; +"github.com/gorilla/mux" -> "bytes"; +"github.com/gorilla/mux" -> "context"; +"github.com/gorilla/mux" -> "errors"; +"github.com/gorilla/mux" -> "fmt"; +"github.com/gorilla/mux" -> "net/http"; +"github.com/gorilla/mux" -> "net/url"; +"github.com/gorilla/mux" -> "path"; +"github.com/gorilla/mux" -> "regexp"; +"github.com/gorilla/mux" -> "strconv"; +"github.com/gorilla/mux" -> "strings"; +"github.com/klauspost/compress/flate" [label="github.com/klauspost/compress/flate" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/flate" target="_blank"]; +"github.com/klauspost/compress/flate" -> "bufio"; +"github.com/klauspost/compress/flate" -> "bytes"; +"github.com/klauspost/compress/flate" -> "encoding/binary"; +"github.com/klauspost/compress/flate" -> "fmt"; +"github.com/klauspost/compress/flate" -> "io"; +"github.com/klauspost/compress/flate" -> "math"; +"github.com/klauspost/compress/flate" -> "math/bits"; +"github.com/klauspost/compress/flate" -> "sort"; +"github.com/klauspost/compress/flate" -> "strconv"; +"github.com/klauspost/compress/flate" -> "sync"; +"github.com/klauspost/compress/fse" [label="github.com/klauspost/compress/fse" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/fse" target="_blank"]; +"github.com/klauspost/compress/fse" -> "errors"; +"github.com/klauspost/compress/fse" -> "fmt"; +"github.com/klauspost/compress/fse" -> "io"; +"github.com/klauspost/compress/fse" -> "math/bits"; +"github.com/klauspost/compress/huff0" [label="github.com/klauspost/compress/huff0" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/huff0" target="_blank"]; +"github.com/klauspost/compress/huff0" -> "errors"; +"github.com/klauspost/compress/huff0" -> "fmt"; +"github.com/klauspost/compress/huff0" -> "github.com/klauspost/compress/fse"; +"github.com/klauspost/compress/huff0" -> "io"; +"github.com/klauspost/compress/huff0" -> "math"; +"github.com/klauspost/compress/huff0" -> "math/bits"; +"github.com/klauspost/compress/huff0" -> "runtime"; +"github.com/klauspost/compress/huff0" -> "sync"; +"github.com/klauspost/compress/snappy" [label="github.com/klauspost/compress/snappy" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/snappy" target="_blank"]; +"github.com/klauspost/compress/snappy" -> "encoding/binary"; +"github.com/klauspost/compress/snappy" -> "errors"; +"github.com/klauspost/compress/snappy" -> "hash/crc32"; +"github.com/klauspost/compress/snappy" -> "io"; +"github.com/klauspost/compress/zstd" [label="github.com/klauspost/compress/zstd" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/zstd" target="_blank"]; +"github.com/klauspost/compress/zstd" -> "bytes"; +"github.com/klauspost/compress/zstd" -> "crypto/rand"; +"github.com/klauspost/compress/zstd" -> "encoding/binary"; +"github.com/klauspost/compress/zstd" -> "encoding/hex"; +"github.com/klauspost/compress/zstd" -> "errors"; +"github.com/klauspost/compress/zstd" -> "fmt"; +"github.com/klauspost/compress/zstd" -> "github.com/klauspost/compress/huff0"; +"github.com/klauspost/compress/zstd" -> "github.com/klauspost/compress/snappy"; +"github.com/klauspost/compress/zstd" -> "github.com/klauspost/compress/zstd/internal/xxhash"; +"github.com/klauspost/compress/zstd" -> "hash"; +"github.com/klauspost/compress/zstd" -> "hash/crc32"; +"github.com/klauspost/compress/zstd" -> "io"; +"github.com/klauspost/compress/zstd" -> "io/ioutil"; +"github.com/klauspost/compress/zstd" -> "log"; +"github.com/klauspost/compress/zstd" -> "math"; +"github.com/klauspost/compress/zstd" -> "math/bits"; +"github.com/klauspost/compress/zstd" -> "runtime"; +"github.com/klauspost/compress/zstd" -> "runtime/debug"; +"github.com/klauspost/compress/zstd" -> "strconv"; +"github.com/klauspost/compress/zstd" -> "strings"; +"github.com/klauspost/compress/zstd" -> "sync"; +"github.com/klauspost/compress/zstd/internal/xxhash" [label="github.com/klauspost/compress/zstd/internal/xxhash" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/compress/zstd/internal/xxhash" target="_blank"]; +"github.com/klauspost/compress/zstd/internal/xxhash" -> "encoding/binary"; +"github.com/klauspost/compress/zstd/internal/xxhash" -> "errors"; +"github.com/klauspost/compress/zstd/internal/xxhash" -> "math/bits"; +"github.com/klauspost/pgzip" [label="github.com/klauspost/pgzip" color="paleturquoise" URL="https://godoc.org/github.com/klauspost/pgzip" target="_blank"]; +"github.com/klauspost/pgzip" -> "bufio"; +"github.com/klauspost/pgzip" -> "bytes"; +"github.com/klauspost/pgzip" -> "errors"; +"github.com/klauspost/pgzip" -> "fmt"; +"github.com/klauspost/pgzip" -> "github.com/klauspost/compress/flate"; +"github.com/klauspost/pgzip" -> "hash"; +"github.com/klauspost/pgzip" -> "hash/crc32"; +"github.com/klauspost/pgzip" -> "io"; +"github.com/klauspost/pgzip" -> "sync"; +"github.com/klauspost/pgzip" -> "time"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" [label="github.com/matttproud/golang_protobuf_extensions/pbutil" color="paleturquoise" URL="https://godoc.org/github.com/matttproud/golang_protobuf_extensions/pbutil" target="_blank"]; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "encoding/binary"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "errors"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "github.com/golang/protobuf/proto"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "io"; +"github.com/opencontainers/go-digest" [label="github.com/opencontainers/go-digest" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/go-digest" target="_blank"]; +"github.com/opencontainers/go-digest" -> "crypto"; +"github.com/opencontainers/go-digest" -> "fmt"; +"github.com/opencontainers/go-digest" -> "hash"; +"github.com/opencontainers/go-digest" -> "io"; +"github.com/opencontainers/go-digest" -> "regexp"; +"github.com/opencontainers/go-digest" -> "strings"; +"github.com/opencontainers/image-spec/specs-go" [label="github.com/opencontainers/image-spec/specs-go" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go" -> "fmt"; +"github.com/opencontainers/image-spec/specs-go/v1" [label="github.com/opencontainers/image-spec/specs-go/v1" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go/v1" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/go-digest"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/image-spec/specs-go"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "time"; +"github.com/opencontainers/runc/libcontainer/user" [label="github.com/opencontainers/runc/libcontainer/user" color="palegoldenrod" URL="https://godoc.org/github.com/opencontainers/runc/libcontainer/user" target="_blank"]; +"github.com/opencontainers/runc/libcontainer/user" -> "bufio"; +"github.com/opencontainers/runc/libcontainer/user" -> "errors"; +"github.com/opencontainers/runc/libcontainer/user" -> "fmt"; +"github.com/opencontainers/runc/libcontainer/user" -> "golang.org/x/sys/unix"; +"github.com/opencontainers/runc/libcontainer/user" -> "io"; +"github.com/opencontainers/runc/libcontainer/user" -> "os"; +"github.com/opencontainers/runc/libcontainer/user" -> "os/user"; +"github.com/opencontainers/runc/libcontainer/user" -> "strconv"; +"github.com/opencontainers/runc/libcontainer/user" -> "strings"; +"github.com/pkg/errors" [label="github.com/pkg/errors" color="paleturquoise" URL="https://godoc.org/github.com/pkg/errors" target="_blank"]; +"github.com/pkg/errors" -> "fmt"; +"github.com/pkg/errors" -> "io"; +"github.com/pkg/errors" -> "path"; +"github.com/pkg/errors" -> "runtime"; +"github.com/pkg/errors" -> "strings"; +"github.com/prometheus/client_golang/prometheus" [label="github.com/prometheus/client_golang/prometheus" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus" target="_blank"]; +"github.com/prometheus/client_golang/prometheus" -> "bytes"; +"github.com/prometheus/client_golang/prometheus" -> "encoding/json"; +"github.com/prometheus/client_golang/prometheus" -> "errors"; +"github.com/prometheus/client_golang/prometheus" -> "expvar"; +"github.com/prometheus/client_golang/prometheus" -> "fmt"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/beorn7/perks/quantile"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/cespare/xxhash/v2"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/client_golang/prometheus/internal"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/common/expfmt"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/common/model"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/procfs"; +"github.com/prometheus/client_golang/prometheus" -> "io/ioutil"; +"github.com/prometheus/client_golang/prometheus" -> "math"; +"github.com/prometheus/client_golang/prometheus" -> "os"; +"github.com/prometheus/client_golang/prometheus" -> "path/filepath"; +"github.com/prometheus/client_golang/prometheus" -> "runtime"; +"github.com/prometheus/client_golang/prometheus" -> "runtime/debug"; +"github.com/prometheus/client_golang/prometheus" -> "sort"; +"github.com/prometheus/client_golang/prometheus" -> "strings"; +"github.com/prometheus/client_golang/prometheus" -> "sync"; +"github.com/prometheus/client_golang/prometheus" -> "sync/atomic"; +"github.com/prometheus/client_golang/prometheus" -> "time"; +"github.com/prometheus/client_golang/prometheus" -> "unicode/utf8"; +"github.com/prometheus/client_golang/prometheus/internal" [label="github.com/prometheus/client_golang/prometheus/internal" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus/internal" target="_blank"]; +"github.com/prometheus/client_golang/prometheus/internal" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus/internal" -> "sort"; +"github.com/prometheus/client_golang/prometheus/promhttp" [label="github.com/prometheus/client_golang/prometheus/promhttp" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp" target="_blank"]; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "bufio"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "compress/gzip"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "crypto/tls"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "errors"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "fmt"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/client_golang/prometheus"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/common/expfmt"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "io"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net/http"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net/http/httptrace"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "strconv"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "strings"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "sync"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "time"; +"github.com/prometheus/client_model/go" [label="github.com/prometheus/client_model/go" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_model/go" target="_blank"]; +"github.com/prometheus/client_model/go" -> "fmt"; +"github.com/prometheus/client_model/go" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/client_model/go" -> "math"; +"github.com/prometheus/common/expfmt" [label="github.com/prometheus/common/expfmt" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/expfmt" target="_blank"]; +"github.com/prometheus/common/expfmt" -> "bufio"; +"github.com/prometheus/common/expfmt" -> "bytes"; +"github.com/prometheus/common/expfmt" -> "fmt"; +"github.com/prometheus/common/expfmt" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/common/expfmt" -> "github.com/matttproud/golang_protobuf_extensions/pbutil"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/common/model"; +"github.com/prometheus/common/expfmt" -> "io"; +"github.com/prometheus/common/expfmt" -> "io/ioutil"; +"github.com/prometheus/common/expfmt" -> "math"; +"github.com/prometheus/common/expfmt" -> "mime"; +"github.com/prometheus/common/expfmt" -> "net/http"; +"github.com/prometheus/common/expfmt" -> "strconv"; +"github.com/prometheus/common/expfmt" -> "strings"; +"github.com/prometheus/common/expfmt" -> "sync"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" [label="github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" target="_blank"]; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "sort"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "strconv"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "strings"; +"github.com/prometheus/common/model" [label="github.com/prometheus/common/model" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/model" target="_blank"]; +"github.com/prometheus/common/model" -> "encoding/json"; +"github.com/prometheus/common/model" -> "fmt"; +"github.com/prometheus/common/model" -> "math"; +"github.com/prometheus/common/model" -> "regexp"; +"github.com/prometheus/common/model" -> "sort"; +"github.com/prometheus/common/model" -> "strconv"; +"github.com/prometheus/common/model" -> "strings"; +"github.com/prometheus/common/model" -> "time"; +"github.com/prometheus/common/model" -> "unicode/utf8"; +"github.com/prometheus/procfs" [label="github.com/prometheus/procfs" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs" target="_blank"]; +"github.com/prometheus/procfs" -> "bufio"; +"github.com/prometheus/procfs" -> "bytes"; +"github.com/prometheus/procfs" -> "encoding/hex"; +"github.com/prometheus/procfs" -> "errors"; +"github.com/prometheus/procfs" -> "fmt"; +"github.com/prometheus/procfs" -> "github.com/prometheus/procfs/internal/fs"; +"github.com/prometheus/procfs" -> "github.com/prometheus/procfs/internal/util"; +"github.com/prometheus/procfs" -> "io"; +"github.com/prometheus/procfs" -> "io/ioutil"; +"github.com/prometheus/procfs" -> "net"; +"github.com/prometheus/procfs" -> "os"; +"github.com/prometheus/procfs" -> "path/filepath"; +"github.com/prometheus/procfs" -> "regexp"; +"github.com/prometheus/procfs" -> "sort"; +"github.com/prometheus/procfs" -> "strconv"; +"github.com/prometheus/procfs" -> "strings"; +"github.com/prometheus/procfs" -> "time"; +"github.com/prometheus/procfs/internal/fs" [label="github.com/prometheus/procfs/internal/fs" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs/internal/fs" target="_blank"]; +"github.com/prometheus/procfs/internal/fs" -> "fmt"; +"github.com/prometheus/procfs/internal/fs" -> "os"; +"github.com/prometheus/procfs/internal/fs" -> "path/filepath"; +"github.com/prometheus/procfs/internal/util" [label="github.com/prometheus/procfs/internal/util" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs/internal/util" target="_blank"]; +"github.com/prometheus/procfs/internal/util" -> "bytes"; +"github.com/prometheus/procfs/internal/util" -> "io/ioutil"; +"github.com/prometheus/procfs/internal/util" -> "os"; +"github.com/prometheus/procfs/internal/util" -> "strconv"; +"github.com/prometheus/procfs/internal/util" -> "strings"; +"github.com/prometheus/procfs/internal/util" -> "syscall"; +"github.com/sirupsen/logrus" [label="github.com/sirupsen/logrus" color="paleturquoise" URL="https://godoc.org/github.com/sirupsen/logrus" target="_blank"]; +"github.com/sirupsen/logrus" -> "bufio"; +"github.com/sirupsen/logrus" -> "bytes"; +"github.com/sirupsen/logrus" -> "context"; +"github.com/sirupsen/logrus" -> "encoding/json"; +"github.com/sirupsen/logrus" -> "fmt"; +"github.com/sirupsen/logrus" -> "golang.org/x/sys/unix"; +"github.com/sirupsen/logrus" -> "io"; +"github.com/sirupsen/logrus" -> "log"; +"github.com/sirupsen/logrus" -> "os"; +"github.com/sirupsen/logrus" -> "reflect"; +"github.com/sirupsen/logrus" -> "runtime"; +"github.com/sirupsen/logrus" -> "sort"; +"github.com/sirupsen/logrus" -> "strings"; +"github.com/sirupsen/logrus" -> "sync"; +"github.com/sirupsen/logrus" -> "sync/atomic"; +"github.com/sirupsen/logrus" -> "time"; +"github.com/ulikunitz/xz" [label="github.com/ulikunitz/xz" color="paleturquoise" URL="https://godoc.org/github.com/ulikunitz/xz" target="_blank"]; +"github.com/ulikunitz/xz" -> "bytes"; +"github.com/ulikunitz/xz" -> "crypto/sha256"; +"github.com/ulikunitz/xz" -> "errors"; +"github.com/ulikunitz/xz" -> "fmt"; +"github.com/ulikunitz/xz" -> "github.com/ulikunitz/xz/internal/xlog"; +"github.com/ulikunitz/xz" -> "github.com/ulikunitz/xz/lzma"; +"github.com/ulikunitz/xz" -> "hash"; +"github.com/ulikunitz/xz" -> "hash/crc32"; +"github.com/ulikunitz/xz" -> "hash/crc64"; +"github.com/ulikunitz/xz" -> "io"; +"github.com/ulikunitz/xz/internal/hash" [label="github.com/ulikunitz/xz/internal/hash" color="paleturquoise" URL="https://godoc.org/github.com/ulikunitz/xz/internal/hash" target="_blank"]; +"github.com/ulikunitz/xz/internal/xlog" [label="github.com/ulikunitz/xz/internal/xlog" color="paleturquoise" URL="https://godoc.org/github.com/ulikunitz/xz/internal/xlog" target="_blank"]; +"github.com/ulikunitz/xz/internal/xlog" -> "fmt"; +"github.com/ulikunitz/xz/internal/xlog" -> "io"; +"github.com/ulikunitz/xz/internal/xlog" -> "os"; +"github.com/ulikunitz/xz/internal/xlog" -> "runtime"; +"github.com/ulikunitz/xz/internal/xlog" -> "sync"; +"github.com/ulikunitz/xz/internal/xlog" -> "time"; +"github.com/ulikunitz/xz/lzma" [label="github.com/ulikunitz/xz/lzma" color="paleturquoise" URL="https://godoc.org/github.com/ulikunitz/xz/lzma" target="_blank"]; +"github.com/ulikunitz/xz/lzma" -> "bufio"; +"github.com/ulikunitz/xz/lzma" -> "bytes"; +"github.com/ulikunitz/xz/lzma" -> "errors"; +"github.com/ulikunitz/xz/lzma" -> "fmt"; +"github.com/ulikunitz/xz/lzma" -> "github.com/ulikunitz/xz/internal/hash"; +"github.com/ulikunitz/xz/lzma" -> "github.com/ulikunitz/xz/internal/xlog"; +"github.com/ulikunitz/xz/lzma" -> "io"; +"github.com/ulikunitz/xz/lzma" -> "unicode"; +"golang.org/x/net/internal/socks" [label="golang.org/x/net/internal/socks" color="paleturquoise" URL="https://godoc.org/golang.org/x/net/internal/socks" target="_blank"]; +"golang.org/x/net/internal/socks" -> "context"; +"golang.org/x/net/internal/socks" -> "errors"; +"golang.org/x/net/internal/socks" -> "io"; +"golang.org/x/net/internal/socks" -> "net"; +"golang.org/x/net/internal/socks" -> "strconv"; +"golang.org/x/net/internal/socks" -> "time"; +"golang.org/x/net/proxy" [label="golang.org/x/net/proxy" color="paleturquoise" URL="https://godoc.org/golang.org/x/net/proxy" target="_blank"]; +"golang.org/x/net/proxy" -> "context"; +"golang.org/x/net/proxy" -> "errors"; +"golang.org/x/net/proxy" -> "golang.org/x/net/internal/socks"; +"golang.org/x/net/proxy" -> "net"; +"golang.org/x/net/proxy" -> "net/url"; +"golang.org/x/net/proxy" -> "os"; +"golang.org/x/net/proxy" -> "strings"; +"golang.org/x/net/proxy" -> "sync"; +"golang.org/x/sys/unix" [label="golang.org/x/sys/unix" color="paleturquoise" URL="https://godoc.org/golang.org/x/sys/unix" target="_blank"]; +"golang.org/x/sys/unix" -> "bytes"; +"golang.org/x/sys/unix" -> "encoding/binary"; +"golang.org/x/sys/unix" -> "net"; +"golang.org/x/sys/unix" -> "runtime"; +"golang.org/x/sys/unix" -> "sort"; +"golang.org/x/sys/unix" -> "strings"; +"golang.org/x/sys/unix" -> "sync"; +"golang.org/x/sys/unix" -> "syscall"; +"golang.org/x/sys/unix" -> "time"; +"golang.org/x/sys/unix" -> "unsafe"; +"gopkg.in/yaml.v2" [label="gopkg.in/yaml.v2" color="paleturquoise" URL="https://godoc.org/gopkg.in/yaml.v2" target="_blank"]; +"gopkg.in/yaml.v2" -> "bytes"; +"gopkg.in/yaml.v2" -> "encoding"; +"gopkg.in/yaml.v2" -> "encoding/base64"; +"gopkg.in/yaml.v2" -> "errors"; +"gopkg.in/yaml.v2" -> "fmt"; +"gopkg.in/yaml.v2" -> "io"; +"gopkg.in/yaml.v2" -> "math"; +"gopkg.in/yaml.v2" -> "reflect"; +"gopkg.in/yaml.v2" -> "regexp"; +"gopkg.in/yaml.v2" -> "sort"; +"gopkg.in/yaml.v2" -> "strconv"; +"gopkg.in/yaml.v2" -> "strings"; +"gopkg.in/yaml.v2" -> "sync"; +"gopkg.in/yaml.v2" -> "time"; +"gopkg.in/yaml.v2" -> "unicode"; +"gopkg.in/yaml.v2" -> "unicode/utf8"; +"hash" [label="hash" color="palegreen" URL="https://godoc.org/hash" target="_blank"]; +"hash/crc32" [label="hash/crc32" color="palegreen" URL="https://godoc.org/hash/crc32" target="_blank"]; +"hash/crc64" [label="hash/crc64" color="palegreen" URL="https://godoc.org/hash/crc64" target="_blank"]; +"io" [label="io" color="palegreen" URL="https://godoc.org/io" target="_blank"]; +"io/ioutil" [label="io/ioutil" color="palegreen" URL="https://godoc.org/io/ioutil" target="_blank"]; +"log" [label="log" color="palegreen" URL="https://godoc.org/log" target="_blank"]; +"math" [label="math" color="palegreen" URL="https://godoc.org/math" target="_blank"]; +"math/big" [label="math/big" color="palegreen" URL="https://godoc.org/math/big" target="_blank"]; +"math/bits" [label="math/bits" color="palegreen" URL="https://godoc.org/math/bits" target="_blank"]; +"mime" [label="mime" color="palegreen" URL="https://godoc.org/mime" target="_blank"]; +"net" [label="net" color="palegreen" URL="https://godoc.org/net" target="_blank"]; +"net/http" [label="net/http" color="palegreen" URL="https://godoc.org/net/http" target="_blank"]; +"net/http/httptrace" [label="net/http/httptrace" color="palegreen" URL="https://godoc.org/net/http/httptrace" target="_blank"]; +"net/url" [label="net/url" color="palegreen" URL="https://godoc.org/net/url" target="_blank"]; +"os" [label="os" color="palegreen" URL="https://godoc.org/os" target="_blank"]; +"os/exec" [label="os/exec" color="palegreen" URL="https://godoc.org/os/exec" target="_blank"]; +"os/user" [label="os/user" color="palegreen" URL="https://godoc.org/os/user" target="_blank"]; +"path" [label="path" color="palegreen" URL="https://godoc.org/path" target="_blank"]; +"path/filepath" [label="path/filepath" color="palegreen" URL="https://godoc.org/path/filepath" target="_blank"]; +"reflect" [label="reflect" color="palegreen" URL="https://godoc.org/reflect" target="_blank"]; +"regexp" [label="regexp" color="palegreen" URL="https://godoc.org/regexp" target="_blank"]; +"runtime" [label="runtime" color="palegreen" URL="https://godoc.org/runtime" target="_blank"]; +"runtime/debug" [label="runtime/debug" color="palegreen" URL="https://godoc.org/runtime/debug" target="_blank"]; +"sort" [label="sort" color="palegreen" URL="https://godoc.org/sort" target="_blank"]; +"strconv" [label="strconv" color="palegreen" URL="https://godoc.org/strconv" target="_blank"]; +"strings" [label="strings" color="palegreen" URL="https://godoc.org/strings" target="_blank"]; +"sync" [label="sync" color="palegreen" URL="https://godoc.org/sync" target="_blank"]; +"sync/atomic" [label="sync/atomic" color="palegreen" URL="https://godoc.org/sync/atomic" target="_blank"]; +"syscall" [label="syscall" color="palegreen" URL="https://godoc.org/syscall" target="_blank"]; +"time" [label="time" color="palegreen" URL="https://godoc.org/time" target="_blank"]; +"unicode" [label="unicode" color="palegreen" URL="https://godoc.org/unicode" target="_blank"]; +"unicode/utf8" [label="unicode/utf8" color="palegreen" URL="https://godoc.org/unicode/utf8" target="_blank"]; +"unsafe" [label="unsafe" color="palegreen" URL="https://godoc.org/unsafe" target="_blank"]; +} diff --git a/images/dot/docker.dot b/images/dot/docker.dot new file mode 100644 index 0000000..90dc677 --- /dev/null +++ b/images/dot/docker.dot @@ -0,0 +1,327 @@ +digraph godep { +nodesep=0.4 +ranksep=0.8 +node [shape="box",style="rounded,filled"] +edge [arrowsize="0.5"] +"bufio" [label="bufio" color="palegreen" URL="https://godoc.org/bufio" target="_blank"]; +"bytes" [label="bytes" color="palegreen" URL="https://godoc.org/bytes" target="_blank"]; +"compress/gzip" [label="compress/gzip" color="palegreen" URL="https://godoc.org/compress/gzip" target="_blank"]; +"context" [label="context" color="palegreen" URL="https://godoc.org/context" target="_blank"]; +"crypto" [label="crypto" color="palegreen" URL="https://godoc.org/crypto" target="_blank"]; +"crypto/tls" [label="crypto/tls" color="palegreen" URL="https://godoc.org/crypto/tls" target="_blank"]; +"encoding" [label="encoding" color="palegreen" URL="https://godoc.org/encoding" target="_blank"]; +"encoding/binary" [label="encoding/binary" color="palegreen" URL="https://godoc.org/encoding/binary" target="_blank"]; +"encoding/hex" [label="encoding/hex" color="palegreen" URL="https://godoc.org/encoding/hex" target="_blank"]; +"encoding/json" [label="encoding/json" color="palegreen" URL="https://godoc.org/encoding/json" target="_blank"]; +"errors" [label="errors" color="palegreen" URL="https://godoc.org/errors" target="_blank"]; +"expvar" [label="expvar" color="palegreen" URL="https://godoc.org/expvar" target="_blank"]; +"fmt" [label="fmt" color="palegreen" URL="https://godoc.org/fmt" target="_blank"]; +"github.com/beorn7/perks/quantile" [label="github.com/beorn7/perks/quantile" color="paleturquoise" URL="https://godoc.org/github.com/beorn7/perks/quantile" target="_blank"]; +"github.com/beorn7/perks/quantile" -> "math"; +"github.com/beorn7/perks/quantile" -> "sort"; +"github.com/cespare/xxhash/v2" [label="github.com/cespare/xxhash/v2" color="paleturquoise" URL="https://godoc.org/github.com/cespare/xxhash/v2" target="_blank"]; +"github.com/cespare/xxhash/v2" -> "encoding/binary"; +"github.com/cespare/xxhash/v2" -> "errors"; +"github.com/cespare/xxhash/v2" -> "math/bits"; +"github.com/cespare/xxhash/v2" -> "reflect"; +"github.com/cespare/xxhash/v2" -> "unsafe"; +"github.com/docker/distribution" [label="github.com/docker/distribution" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution" target="_blank"]; +"github.com/docker/distribution" -> "context"; +"github.com/docker/distribution" -> "errors"; +"github.com/docker/distribution" -> "fmt"; +"github.com/docker/distribution" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution" -> "github.com/opencontainers/image-spec/specs-go/v1"; +"github.com/docker/distribution" -> "io"; +"github.com/docker/distribution" -> "mime"; +"github.com/docker/distribution" -> "net/http"; +"github.com/docker/distribution" -> "strings"; +"github.com/docker/distribution" -> "time"; +"github.com/docker/distribution/digestset" [label="github.com/docker/distribution/digestset" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/digestset" target="_blank"]; +"github.com/docker/distribution/digestset" -> "errors"; +"github.com/docker/distribution/digestset" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/digestset" -> "sort"; +"github.com/docker/distribution/digestset" -> "strings"; +"github.com/docker/distribution/digestset" -> "sync"; +"github.com/docker/distribution/metrics" [label="github.com/docker/distribution/metrics" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/metrics" target="_blank"]; +"github.com/docker/distribution/metrics" -> "github.com/docker/go-metrics"; +"github.com/docker/distribution/reference" [label="github.com/docker/distribution/reference" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/reference" target="_blank"]; +"github.com/docker/distribution/reference" -> "errors"; +"github.com/docker/distribution/reference" -> "fmt"; +"github.com/docker/distribution/reference" -> "github.com/docker/distribution/digestset"; +"github.com/docker/distribution/reference" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/reference" -> "path"; +"github.com/docker/distribution/reference" -> "regexp"; +"github.com/docker/distribution/reference" -> "strings"; +"github.com/docker/distribution/registry/api/errcode" [label="github.com/docker/distribution/registry/api/errcode" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/api/errcode" target="_blank"]; +"github.com/docker/distribution/registry/api/errcode" -> "encoding/json"; +"github.com/docker/distribution/registry/api/errcode" -> "fmt"; +"github.com/docker/distribution/registry/api/errcode" -> "net/http"; +"github.com/docker/distribution/registry/api/errcode" -> "sort"; +"github.com/docker/distribution/registry/api/errcode" -> "strings"; +"github.com/docker/distribution/registry/api/errcode" -> "sync"; +"github.com/docker/distribution/registry/api/v2" [label="github.com/docker/distribution/registry/api/v2" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/api/v2" target="_blank"]; +"github.com/docker/distribution/registry/api/v2" -> "fmt"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/gorilla/mux"; +"github.com/docker/distribution/registry/api/v2" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/api/v2" -> "net/http"; +"github.com/docker/distribution/registry/api/v2" -> "net/url"; +"github.com/docker/distribution/registry/api/v2" -> "regexp"; +"github.com/docker/distribution/registry/api/v2" -> "strings"; +"github.com/docker/distribution/registry/api/v2" -> "unicode"; +"github.com/docker/distribution/registry/client" [label="github.com/docker/distribution/registry/client" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client" target="_blank"]; +"github.com/docker/distribution/registry/client" -> "bytes"; +"github.com/docker/distribution/registry/client" -> "context"; +"github.com/docker/distribution/registry/client" -> "encoding/json"; +"github.com/docker/distribution/registry/client" -> "errors"; +"github.com/docker/distribution/registry/client" -> "fmt"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/api/errcode"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/api/v2"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/client/auth/challenge"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/client/transport"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/storage/cache"; +"github.com/docker/distribution/registry/client" -> "github.com/docker/distribution/registry/storage/cache/memory"; +"github.com/docker/distribution/registry/client" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/client" -> "io"; +"github.com/docker/distribution/registry/client" -> "io/ioutil"; +"github.com/docker/distribution/registry/client" -> "net/http"; +"github.com/docker/distribution/registry/client" -> "net/url"; +"github.com/docker/distribution/registry/client" -> "strconv"; +"github.com/docker/distribution/registry/client" -> "strings"; +"github.com/docker/distribution/registry/client" -> "time"; +"github.com/docker/distribution/registry/client/auth" [label="github.com/docker/distribution/registry/client/auth" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client/auth" target="_blank"]; +"github.com/docker/distribution/registry/client/auth" -> "encoding/json"; +"github.com/docker/distribution/registry/client/auth" -> "errors"; +"github.com/docker/distribution/registry/client/auth" -> "fmt"; +"github.com/docker/distribution/registry/client/auth" -> "github.com/docker/distribution/registry/client"; +"github.com/docker/distribution/registry/client/auth" -> "github.com/docker/distribution/registry/client/auth/challenge"; +"github.com/docker/distribution/registry/client/auth" -> "github.com/docker/distribution/registry/client/transport"; +"github.com/docker/distribution/registry/client/auth" -> "net/http"; +"github.com/docker/distribution/registry/client/auth" -> "net/url"; +"github.com/docker/distribution/registry/client/auth" -> "strings"; +"github.com/docker/distribution/registry/client/auth" -> "sync"; +"github.com/docker/distribution/registry/client/auth" -> "time"; +"github.com/docker/distribution/registry/client/auth/challenge" [label="github.com/docker/distribution/registry/client/auth/challenge" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client/auth/challenge" target="_blank"]; +"github.com/docker/distribution/registry/client/auth/challenge" -> "fmt"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "net/http"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "net/url"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "strings"; +"github.com/docker/distribution/registry/client/auth/challenge" -> "sync"; +"github.com/docker/distribution/registry/client/transport" [label="github.com/docker/distribution/registry/client/transport" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/client/transport" target="_blank"]; +"github.com/docker/distribution/registry/client/transport" -> "errors"; +"github.com/docker/distribution/registry/client/transport" -> "fmt"; +"github.com/docker/distribution/registry/client/transport" -> "io"; +"github.com/docker/distribution/registry/client/transport" -> "net/http"; +"github.com/docker/distribution/registry/client/transport" -> "regexp"; +"github.com/docker/distribution/registry/client/transport" -> "strconv"; +"github.com/docker/distribution/registry/client/transport" -> "sync"; +"github.com/docker/distribution/registry/storage/cache" [label="github.com/docker/distribution/registry/storage/cache" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/storage/cache" target="_blank"]; +"github.com/docker/distribution/registry/storage/cache" -> "context"; +"github.com/docker/distribution/registry/storage/cache" -> "fmt"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/docker/distribution/metrics"; +"github.com/docker/distribution/registry/storage/cache" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/storage/cache/memory" [label="github.com/docker/distribution/registry/storage/cache/memory" color="paleturquoise" URL="https://godoc.org/github.com/docker/distribution/registry/storage/cache/memory" target="_blank"]; +"github.com/docker/distribution/registry/storage/cache/memory" -> "context"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution/reference"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/docker/distribution/registry/storage/cache"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "github.com/opencontainers/go-digest"; +"github.com/docker/distribution/registry/storage/cache/memory" -> "sync"; +"github.com/docker/go-metrics" [label="github.com/docker/go-metrics" color="paleturquoise" URL="https://godoc.org/github.com/docker/go-metrics" target="_blank"]; +"github.com/docker/go-metrics" -> "fmt"; +"github.com/docker/go-metrics" -> "github.com/prometheus/client_golang/prometheus"; +"github.com/docker/go-metrics" -> "github.com/prometheus/client_golang/prometheus/promhttp"; +"github.com/docker/go-metrics" -> "net/http"; +"github.com/docker/go-metrics" -> "sync"; +"github.com/docker/go-metrics" -> "time"; +"github.com/golang/protobuf/proto" [label="github.com/golang/protobuf/proto" color="paleturquoise" URL="https://godoc.org/github.com/golang/protobuf/proto" target="_blank"]; +"github.com/golang/protobuf/proto" -> "bufio"; +"github.com/golang/protobuf/proto" -> "bytes"; +"github.com/golang/protobuf/proto" -> "encoding"; +"github.com/golang/protobuf/proto" -> "encoding/json"; +"github.com/golang/protobuf/proto" -> "errors"; +"github.com/golang/protobuf/proto" -> "fmt"; +"github.com/golang/protobuf/proto" -> "io"; +"github.com/golang/protobuf/proto" -> "log"; +"github.com/golang/protobuf/proto" -> "math"; +"github.com/golang/protobuf/proto" -> "reflect"; +"github.com/golang/protobuf/proto" -> "sort"; +"github.com/golang/protobuf/proto" -> "strconv"; +"github.com/golang/protobuf/proto" -> "strings"; +"github.com/golang/protobuf/proto" -> "sync"; +"github.com/golang/protobuf/proto" -> "sync/atomic"; +"github.com/golang/protobuf/proto" -> "unicode/utf8"; +"github.com/golang/protobuf/proto" -> "unsafe"; +"github.com/gorilla/mux" [label="github.com/gorilla/mux" color="paleturquoise" URL="https://godoc.org/github.com/gorilla/mux" target="_blank"]; +"github.com/gorilla/mux" -> "bytes"; +"github.com/gorilla/mux" -> "context"; +"github.com/gorilla/mux" -> "errors"; +"github.com/gorilla/mux" -> "fmt"; +"github.com/gorilla/mux" -> "net/http"; +"github.com/gorilla/mux" -> "net/url"; +"github.com/gorilla/mux" -> "path"; +"github.com/gorilla/mux" -> "regexp"; +"github.com/gorilla/mux" -> "strconv"; +"github.com/gorilla/mux" -> "strings"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" [label="github.com/matttproud/golang_protobuf_extensions/pbutil" color="paleturquoise" URL="https://godoc.org/github.com/matttproud/golang_protobuf_extensions/pbutil" target="_blank"]; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "encoding/binary"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "errors"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "github.com/golang/protobuf/proto"; +"github.com/matttproud/golang_protobuf_extensions/pbutil" -> "io"; +"github.com/opencontainers/go-digest" [label="github.com/opencontainers/go-digest" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/go-digest" target="_blank"]; +"github.com/opencontainers/go-digest" -> "crypto"; +"github.com/opencontainers/go-digest" -> "fmt"; +"github.com/opencontainers/go-digest" -> "hash"; +"github.com/opencontainers/go-digest" -> "io"; +"github.com/opencontainers/go-digest" -> "regexp"; +"github.com/opencontainers/go-digest" -> "strings"; +"github.com/opencontainers/image-spec/specs-go" [label="github.com/opencontainers/image-spec/specs-go" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go" -> "fmt"; +"github.com/opencontainers/image-spec/specs-go/v1" [label="github.com/opencontainers/image-spec/specs-go/v1" color="paleturquoise" URL="https://godoc.org/github.com/opencontainers/image-spec/specs-go/v1" target="_blank"]; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/go-digest"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "github.com/opencontainers/image-spec/specs-go"; +"github.com/opencontainers/image-spec/specs-go/v1" -> "time"; +"github.com/prometheus/client_golang/prometheus" [label="github.com/prometheus/client_golang/prometheus" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus" target="_blank"]; +"github.com/prometheus/client_golang/prometheus" -> "bytes"; +"github.com/prometheus/client_golang/prometheus" -> "encoding/json"; +"github.com/prometheus/client_golang/prometheus" -> "errors"; +"github.com/prometheus/client_golang/prometheus" -> "expvar"; +"github.com/prometheus/client_golang/prometheus" -> "fmt"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/beorn7/perks/quantile"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/cespare/xxhash/v2"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/client_golang/prometheus/internal"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/common/expfmt"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/common/model"; +"github.com/prometheus/client_golang/prometheus" -> "github.com/prometheus/procfs"; +"github.com/prometheus/client_golang/prometheus" -> "io/ioutil"; +"github.com/prometheus/client_golang/prometheus" -> "math"; +"github.com/prometheus/client_golang/prometheus" -> "os"; +"github.com/prometheus/client_golang/prometheus" -> "path/filepath"; +"github.com/prometheus/client_golang/prometheus" -> "runtime"; +"github.com/prometheus/client_golang/prometheus" -> "runtime/debug"; +"github.com/prometheus/client_golang/prometheus" -> "sort"; +"github.com/prometheus/client_golang/prometheus" -> "strings"; +"github.com/prometheus/client_golang/prometheus" -> "sync"; +"github.com/prometheus/client_golang/prometheus" -> "sync/atomic"; +"github.com/prometheus/client_golang/prometheus" -> "time"; +"github.com/prometheus/client_golang/prometheus" -> "unicode/utf8"; +"github.com/prometheus/client_golang/prometheus/internal" [label="github.com/prometheus/client_golang/prometheus/internal" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus/internal" target="_blank"]; +"github.com/prometheus/client_golang/prometheus/internal" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus/internal" -> "sort"; +"github.com/prometheus/client_golang/prometheus/promhttp" [label="github.com/prometheus/client_golang/prometheus/promhttp" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_golang/prometheus/promhttp" target="_blank"]; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "bufio"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "compress/gzip"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "crypto/tls"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "errors"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "fmt"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/client_golang/prometheus"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "github.com/prometheus/common/expfmt"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "io"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net/http"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "net/http/httptrace"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "strconv"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "strings"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "sync"; +"github.com/prometheus/client_golang/prometheus/promhttp" -> "time"; +"github.com/prometheus/client_model/go" [label="github.com/prometheus/client_model/go" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/client_model/go" target="_blank"]; +"github.com/prometheus/client_model/go" -> "fmt"; +"github.com/prometheus/client_model/go" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/client_model/go" -> "math"; +"github.com/prometheus/common/expfmt" [label="github.com/prometheus/common/expfmt" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/expfmt" target="_blank"]; +"github.com/prometheus/common/expfmt" -> "bufio"; +"github.com/prometheus/common/expfmt" -> "bytes"; +"github.com/prometheus/common/expfmt" -> "fmt"; +"github.com/prometheus/common/expfmt" -> "github.com/golang/protobuf/proto"; +"github.com/prometheus/common/expfmt" -> "github.com/matttproud/golang_protobuf_extensions/pbutil"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/client_model/go"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg"; +"github.com/prometheus/common/expfmt" -> "github.com/prometheus/common/model"; +"github.com/prometheus/common/expfmt" -> "io"; +"github.com/prometheus/common/expfmt" -> "io/ioutil"; +"github.com/prometheus/common/expfmt" -> "math"; +"github.com/prometheus/common/expfmt" -> "mime"; +"github.com/prometheus/common/expfmt" -> "net/http"; +"github.com/prometheus/common/expfmt" -> "strconv"; +"github.com/prometheus/common/expfmt" -> "strings"; +"github.com/prometheus/common/expfmt" -> "sync"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" [label="github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" target="_blank"]; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "sort"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "strconv"; +"github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" -> "strings"; +"github.com/prometheus/common/model" [label="github.com/prometheus/common/model" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/common/model" target="_blank"]; +"github.com/prometheus/common/model" -> "encoding/json"; +"github.com/prometheus/common/model" -> "fmt"; +"github.com/prometheus/common/model" -> "math"; +"github.com/prometheus/common/model" -> "regexp"; +"github.com/prometheus/common/model" -> "sort"; +"github.com/prometheus/common/model" -> "strconv"; +"github.com/prometheus/common/model" -> "strings"; +"github.com/prometheus/common/model" -> "time"; +"github.com/prometheus/common/model" -> "unicode/utf8"; +"github.com/prometheus/procfs" [label="github.com/prometheus/procfs" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs" target="_blank"]; +"github.com/prometheus/procfs" -> "bufio"; +"github.com/prometheus/procfs" -> "bytes"; +"github.com/prometheus/procfs" -> "encoding/hex"; +"github.com/prometheus/procfs" -> "errors"; +"github.com/prometheus/procfs" -> "fmt"; +"github.com/prometheus/procfs" -> "github.com/prometheus/procfs/internal/fs"; +"github.com/prometheus/procfs" -> "github.com/prometheus/procfs/internal/util"; +"github.com/prometheus/procfs" -> "io"; +"github.com/prometheus/procfs" -> "io/ioutil"; +"github.com/prometheus/procfs" -> "net"; +"github.com/prometheus/procfs" -> "os"; +"github.com/prometheus/procfs" -> "path/filepath"; +"github.com/prometheus/procfs" -> "regexp"; +"github.com/prometheus/procfs" -> "sort"; +"github.com/prometheus/procfs" -> "strconv"; +"github.com/prometheus/procfs" -> "strings"; +"github.com/prometheus/procfs" -> "time"; +"github.com/prometheus/procfs/internal/fs" [label="github.com/prometheus/procfs/internal/fs" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs/internal/fs" target="_blank"]; +"github.com/prometheus/procfs/internal/fs" -> "fmt"; +"github.com/prometheus/procfs/internal/fs" -> "os"; +"github.com/prometheus/procfs/internal/fs" -> "path/filepath"; +"github.com/prometheus/procfs/internal/util" [label="github.com/prometheus/procfs/internal/util" color="paleturquoise" URL="https://godoc.org/github.com/prometheus/procfs/internal/util" target="_blank"]; +"github.com/prometheus/procfs/internal/util" -> "bytes"; +"github.com/prometheus/procfs/internal/util" -> "io/ioutil"; +"github.com/prometheus/procfs/internal/util" -> "os"; +"github.com/prometheus/procfs/internal/util" -> "strconv"; +"github.com/prometheus/procfs/internal/util" -> "strings"; +"github.com/prometheus/procfs/internal/util" -> "syscall"; +"hash" [label="hash" color="palegreen" URL="https://godoc.org/hash" target="_blank"]; +"io" [label="io" color="palegreen" URL="https://godoc.org/io" target="_blank"]; +"io/ioutil" [label="io/ioutil" color="palegreen" URL="https://godoc.org/io/ioutil" target="_blank"]; +"log" [label="log" color="palegreen" URL="https://godoc.org/log" target="_blank"]; +"math" [label="math" color="palegreen" URL="https://godoc.org/math" target="_blank"]; +"math/bits" [label="math/bits" color="palegreen" URL="https://godoc.org/math/bits" target="_blank"]; +"mime" [label="mime" color="palegreen" URL="https://godoc.org/mime" target="_blank"]; +"net" [label="net" color="palegreen" URL="https://godoc.org/net" target="_blank"]; +"net/http" [label="net/http" color="palegreen" URL="https://godoc.org/net/http" target="_blank"]; +"net/http/httptrace" [label="net/http/httptrace" color="palegreen" URL="https://godoc.org/net/http/httptrace" target="_blank"]; +"net/url" [label="net/url" color="palegreen" URL="https://godoc.org/net/url" target="_blank"]; +"os" [label="os" color="palegreen" URL="https://godoc.org/os" target="_blank"]; +"path" [label="path" color="palegreen" URL="https://godoc.org/path" target="_blank"]; +"path/filepath" [label="path/filepath" color="palegreen" URL="https://godoc.org/path/filepath" target="_blank"]; +"reflect" [label="reflect" color="palegreen" URL="https://godoc.org/reflect" target="_blank"]; +"regexp" [label="regexp" color="palegreen" URL="https://godoc.org/regexp" target="_blank"]; +"runtime" [label="runtime" color="palegreen" URL="https://godoc.org/runtime" target="_blank"]; +"runtime/debug" [label="runtime/debug" color="palegreen" URL="https://godoc.org/runtime/debug" target="_blank"]; +"sort" [label="sort" color="palegreen" URL="https://godoc.org/sort" target="_blank"]; +"strconv" [label="strconv" color="palegreen" URL="https://godoc.org/strconv" target="_blank"]; +"strings" [label="strings" color="palegreen" URL="https://godoc.org/strings" target="_blank"]; +"sync" [label="sync" color="palegreen" URL="https://godoc.org/sync" target="_blank"]; +"sync/atomic" [label="sync/atomic" color="palegreen" URL="https://godoc.org/sync/atomic" target="_blank"]; +"syscall" [label="syscall" color="palegreen" URL="https://godoc.org/syscall" target="_blank"]; +"time" [label="time" color="palegreen" URL="https://godoc.org/time" target="_blank"]; +"unicode" [label="unicode" color="palegreen" URL="https://godoc.org/unicode" target="_blank"]; +"unicode/utf8" [label="unicode/utf8" color="palegreen" URL="https://godoc.org/unicode/utf8" target="_blank"]; +"unsafe" [label="unsafe" color="palegreen" URL="https://godoc.org/unsafe" target="_blank"]; +} diff --git a/images/dot/ggcr.dot b/images/dot/ggcr.dot new file mode 100644 index 0000000..459ba6d --- /dev/null +++ b/images/dot/ggcr.dot @@ -0,0 +1,130 @@ +digraph godep { +nodesep=0.4 +ranksep=0.8 +node [shape="box",style="rounded,filled"] +edge [arrowsize="0.5"] +"bufio" [label="bufio" color="palegreen" URL="https://godoc.org/bufio" target="_blank"]; +"bytes" [label="bytes" color="palegreen" URL="https://godoc.org/bytes" target="_blank"]; +"context" [label="context" color="palegreen" URL="https://godoc.org/context" target="_blank"]; +"encoding/base64" [label="encoding/base64" color="palegreen" URL="https://godoc.org/encoding/base64" target="_blank"]; +"encoding/json" [label="encoding/json" color="palegreen" URL="https://godoc.org/encoding/json" target="_blank"]; +"errors" [label="errors" color="palegreen" URL="https://godoc.org/errors" target="_blank"]; +"fmt" [label="fmt" color="palegreen" URL="https://godoc.org/fmt" target="_blank"]; +"github.com/docker/cli/cli/config" [label="github.com/docker/cli/cli/config" color="paleturquoise" URL="https://godoc.org/github.com/docker/cli/cli/config" target="_blank"]; +"github.com/docker/cli/cli/config" -> "fmt"; +"github.com/docker/cli/cli/config" -> "github.com/docker/cli/cli/config/configfile"; +"github.com/docker/cli/cli/config" -> "github.com/docker/cli/cli/config/credentials"; +"github.com/docker/cli/cli/config" -> "github.com/docker/cli/cli/config/types"; +"github.com/docker/cli/cli/config" -> "github.com/docker/docker/pkg/homedir"; +"github.com/docker/cli/cli/config" -> "github.com/pkg/errors"; +"github.com/docker/cli/cli/config" -> "io"; +"github.com/docker/cli/cli/config" -> "os"; +"github.com/docker/cli/cli/config" -> "path/filepath"; +"github.com/docker/cli/cli/config" -> "strings"; +"github.com/docker/cli/cli/config/configfile" [label="github.com/docker/cli/cli/config/configfile" color="paleturquoise" URL="https://godoc.org/github.com/docker/cli/cli/config/configfile" target="_blank"]; +"github.com/docker/cli/cli/config/configfile" -> "encoding/base64"; +"github.com/docker/cli/cli/config/configfile" -> "encoding/json"; +"github.com/docker/cli/cli/config/configfile" -> "fmt"; +"github.com/docker/cli/cli/config/configfile" -> "github.com/docker/cli/cli/config/credentials"; +"github.com/docker/cli/cli/config/configfile" -> "github.com/docker/cli/cli/config/types"; +"github.com/docker/cli/cli/config/configfile" -> "github.com/pkg/errors"; +"github.com/docker/cli/cli/config/configfile" -> "io"; +"github.com/docker/cli/cli/config/configfile" -> "io/ioutil"; +"github.com/docker/cli/cli/config/configfile" -> "os"; +"github.com/docker/cli/cli/config/configfile" -> "path/filepath"; +"github.com/docker/cli/cli/config/configfile" -> "strings"; +"github.com/docker/cli/cli/config/credentials" [label="github.com/docker/cli/cli/config/credentials" color="paleturquoise" URL="https://godoc.org/github.com/docker/cli/cli/config/credentials" target="_blank"]; +"github.com/docker/cli/cli/config/credentials" -> "github.com/docker/cli/cli/config/types"; +"github.com/docker/cli/cli/config/credentials" -> "github.com/docker/docker-credential-helpers/client"; +"github.com/docker/cli/cli/config/credentials" -> "github.com/docker/docker-credential-helpers/credentials"; +"github.com/docker/cli/cli/config/credentials" -> "os/exec"; +"github.com/docker/cli/cli/config/credentials" -> "strings"; +"github.com/docker/cli/cli/config/types" [label="github.com/docker/cli/cli/config/types" color="paleturquoise" URL="https://godoc.org/github.com/docker/cli/cli/config/types" target="_blank"]; +"github.com/docker/docker-credential-helpers/client" [label="github.com/docker/docker-credential-helpers/client" color="palegoldenrod" URL="https://godoc.org/github.com/docker/docker-credential-helpers/client" target="_blank"]; +"github.com/docker/docker-credential-helpers/client" -> "bytes"; +"github.com/docker/docker-credential-helpers/client" -> "encoding/json"; +"github.com/docker/docker-credential-helpers/client" -> "fmt"; +"github.com/docker/docker-credential-helpers/client" -> "github.com/docker/docker-credential-helpers/credentials"; +"github.com/docker/docker-credential-helpers/client" -> "io"; +"github.com/docker/docker-credential-helpers/client" -> "os"; +"github.com/docker/docker-credential-helpers/client" -> "os/exec"; +"github.com/docker/docker-credential-helpers/client" -> "strings"; +"github.com/docker/docker-credential-helpers/credentials" [label="github.com/docker/docker-credential-helpers/credentials" color="palegoldenrod" URL="https://godoc.org/github.com/docker/docker-credential-helpers/credentials" target="_blank"]; +"github.com/docker/docker-credential-helpers/credentials" -> "bufio"; +"github.com/docker/docker-credential-helpers/credentials" -> "bytes"; +"github.com/docker/docker-credential-helpers/credentials" -> "encoding/json"; +"github.com/docker/docker-credential-helpers/credentials" -> "fmt"; +"github.com/docker/docker-credential-helpers/credentials" -> "io"; +"github.com/docker/docker-credential-helpers/credentials" -> "os"; +"github.com/docker/docker-credential-helpers/credentials" -> "strings"; +"github.com/docker/docker/pkg/homedir" [label="github.com/docker/docker/pkg/homedir" color="paleturquoise" URL="https://godoc.org/github.com/docker/docker/pkg/homedir" target="_blank"]; +"github.com/docker/docker/pkg/homedir" -> "errors"; +"github.com/docker/docker/pkg/homedir" -> "os"; +"github.com/docker/docker/pkg/homedir" -> "os/user"; +"github.com/docker/docker/pkg/homedir" -> "path/filepath"; +"github.com/docker/docker/pkg/homedir" -> "strings"; +"github.com/google/go-containerregistry/pkg/authn" [label="github.com/google/go-containerregistry/pkg/authn" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/authn" target="_blank"]; +"github.com/google/go-containerregistry/pkg/authn" -> "encoding/json"; +"github.com/google/go-containerregistry/pkg/authn" -> "github.com/docker/cli/cli/config"; +"github.com/google/go-containerregistry/pkg/authn" -> "github.com/docker/cli/cli/config/types"; +"github.com/google/go-containerregistry/pkg/authn" -> "github.com/google/go-containerregistry/pkg/logs"; +"github.com/google/go-containerregistry/pkg/authn" -> "github.com/google/go-containerregistry/pkg/name"; +"github.com/google/go-containerregistry/pkg/authn" -> "os"; +"github.com/google/go-containerregistry/pkg/internal/retry" [label="github.com/google/go-containerregistry/pkg/internal/retry" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/internal/retry" target="_blank"]; +"github.com/google/go-containerregistry/pkg/internal/retry" -> "context"; +"github.com/google/go-containerregistry/pkg/internal/retry" -> "fmt"; +"github.com/google/go-containerregistry/pkg/internal/retry" -> "github.com/google/go-containerregistry/pkg/internal/retry/wait"; +"github.com/google/go-containerregistry/pkg/internal/retry/wait" [label="github.com/google/go-containerregistry/pkg/internal/retry/wait" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/internal/retry/wait" target="_blank"]; +"github.com/google/go-containerregistry/pkg/internal/retry/wait" -> "errors"; +"github.com/google/go-containerregistry/pkg/internal/retry/wait" -> "math/rand"; +"github.com/google/go-containerregistry/pkg/internal/retry/wait" -> "time"; +"github.com/google/go-containerregistry/pkg/logs" [label="github.com/google/go-containerregistry/pkg/logs" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/logs" target="_blank"]; +"github.com/google/go-containerregistry/pkg/logs" -> "io/ioutil"; +"github.com/google/go-containerregistry/pkg/logs" -> "log"; +"github.com/google/go-containerregistry/pkg/name" [label="github.com/google/go-containerregistry/pkg/name" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/name" target="_blank"]; +"github.com/google/go-containerregistry/pkg/name" -> "fmt"; +"github.com/google/go-containerregistry/pkg/name" -> "net"; +"github.com/google/go-containerregistry/pkg/name" -> "net/url"; +"github.com/google/go-containerregistry/pkg/name" -> "regexp"; +"github.com/google/go-containerregistry/pkg/name" -> "strings"; +"github.com/google/go-containerregistry/pkg/name" -> "unicode/utf8"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" [label="github.com/google/go-containerregistry/pkg/v1/remote/transport" color="paleturquoise" URL="https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote/transport" target="_blank"]; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "encoding/base64"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "encoding/json"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "fmt"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "github.com/google/go-containerregistry/pkg/authn"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "github.com/google/go-containerregistry/pkg/internal/retry"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "github.com/google/go-containerregistry/pkg/logs"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "github.com/google/go-containerregistry/pkg/name"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "io/ioutil"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "net"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "net/http"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "net/http/httputil"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "net/url"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "strings"; +"github.com/google/go-containerregistry/pkg/v1/remote/transport" -> "time"; +"github.com/pkg/errors" [label="github.com/pkg/errors" color="palegoldenrod" URL="https://godoc.org/github.com/pkg/errors" target="_blank"]; +"github.com/pkg/errors" -> "fmt"; +"github.com/pkg/errors" -> "io"; +"github.com/pkg/errors" -> "path"; +"github.com/pkg/errors" -> "runtime"; +"github.com/pkg/errors" -> "strings"; +"io" [label="io" color="palegreen" URL="https://godoc.org/io" target="_blank"]; +"io/ioutil" [label="io/ioutil" color="palegreen" URL="https://godoc.org/io/ioutil" target="_blank"]; +"log" [label="log" color="palegreen" URL="https://godoc.org/log" target="_blank"]; +"math/rand" [label="math/rand" color="palegreen" URL="https://godoc.org/math/rand" target="_blank"]; +"net" [label="net" color="palegreen" URL="https://godoc.org/net" target="_blank"]; +"net/http" [label="net/http" color="palegreen" URL="https://godoc.org/net/http" target="_blank"]; +"net/http/httputil" [label="net/http/httputil" color="palegreen" URL="https://godoc.org/net/http/httputil" target="_blank"]; +"net/url" [label="net/url" color="palegreen" URL="https://godoc.org/net/url" target="_blank"]; +"os" [label="os" color="palegreen" URL="https://godoc.org/os" target="_blank"]; +"os/exec" [label="os/exec" color="palegreen" URL="https://godoc.org/os/exec" target="_blank"]; +"os/user" [label="os/user" color="palegreen" URL="https://godoc.org/os/user" target="_blank"]; +"path" [label="path" color="palegreen" URL="https://godoc.org/path" target="_blank"]; +"path/filepath" [label="path/filepath" color="palegreen" URL="https://godoc.org/path/filepath" target="_blank"]; +"regexp" [label="regexp" color="palegreen" URL="https://godoc.org/regexp" target="_blank"]; +"runtime" [label="runtime" color="palegreen" URL="https://godoc.org/runtime" target="_blank"]; +"strings" [label="strings" color="palegreen" URL="https://godoc.org/strings" target="_blank"]; +"time" [label="time" color="palegreen" URL="https://godoc.org/time" target="_blank"]; +"unicode/utf8" [label="unicode/utf8" color="palegreen" URL="https://godoc.org/unicode/utf8" target="_blank"]; +} diff --git a/images/dot/image-anatomy.dot b/images/dot/image-anatomy.dot new file mode 100644 index 0000000..179e311 --- /dev/null +++ b/images/dot/image-anatomy.dot @@ -0,0 +1,26 @@ +digraph { + compound=true; + rankdir="LR"; + + tag [label="", shape="circle", width=0.1, style="filled", color="black"]; + manifest [shape="note"]; + config [shape="note"]; + + tag -> manifest [label="digest", taillabel="tag", tailport=head, labeldistance=2.1, labelangle=108]; + manifest -> config [label="(image id)"]; + config -> l1 [label="diffid"]; + config -> l2 [label="diffid"]; + manifest -> l1 [lhead=cluster_layer1, label="layer digest"]; + manifest -> l2 [lhead=cluster_layer2, label="layer digest"]; + + subgraph cluster_layer1 { + label = "layer.tar.gz"; + margin = 20.0; + l1 [label="layer.tar", shape="folder"]; + } + subgraph cluster_layer2 { + label = "layer.tar.gz"; + margin = 20.0; + l2 [label="layer.tar", shape="folder"]; + } +} diff --git a/images/dot/index-anatomy-strange.dot b/images/dot/index-anatomy-strange.dot new file mode 100644 index 0000000..2bccba3 --- /dev/null +++ b/images/dot/index-anatomy-strange.dot @@ -0,0 +1,24 @@ +digraph { + ordering = out; + compound=true; + rankdir="LR"; + + tag [label="", shape="circle", width=0.1, style="filled", color="black"]; + tag2 [label="", shape="circle", width=0.1, style="filled", color="black"]; + tag3 [label="", shape="circle", width=0.1, style="filled", color="black"]; + index [shape="note"]; + index2 [label="index", shape="note"]; + image [shape="note"]; + image2 [label="image", shape="note"]; + image3 [label="image", shape="note"]; + xml; + + tag -> index [taillabel="r124356", tailport=head, labeldistance=2.1, labelangle=108]; + tag2 -> index2 [taillabel="stable-release", tailport=head, labeldistance=2.1, labelangle=108]; + tag3 -> image [taillabel="v1.0", tailport=head, labeldistance=2.1, labelangle=108]; + index -> image; + index -> xml; + index -> index2; + index2 -> image2; + index2 -> image3; +} diff --git a/images/dot/index-anatomy.dot b/images/dot/index-anatomy.dot new file mode 100644 index 0000000..9155af0 --- /dev/null +++ b/images/dot/index-anatomy.dot @@ -0,0 +1,18 @@ +digraph { + ordering = out; + compound=true; + rankdir="LR"; + + tag [label="", shape="circle", width=0.1, style="filled", color="black"]; + tag2 [label="", shape="circle", width=0.1, style="filled", color="black"]; + tag3 [label="", shape="circle", width=0.1, style="filled", color="black"]; + index [shape="note"]; + image [shape="note"]; + image2 [label="image", shape="note"]; + + tag -> index [taillabel="latest", tailport=head, labeldistance=2.1, labelangle=108]; + tag2 -> image [taillabel="amd64", tailport=head, labeldistance=2.1, labelangle=108]; + tag3 -> image2 [taillabel="ppc64le", tailport=head, labeldistance=2.1, labelangle=252]; + index -> image; + index -> image2; +} diff --git a/images/dot/mutate.dot b/images/dot/mutate.dot new file mode 100644 index 0000000..228f8b6 --- /dev/null +++ b/images/dot/mutate.dot @@ -0,0 +1,59 @@ +digraph { + input [label="v1.Image", shape=box]; + output [label="v1.Image", shape=box]; + + ordering = "out"; + + subgraph cluster_source { + label = "Sources"; + "remotesource" [label="remote"]; + "tarballsource" [label="tarball"]; + "randomsource" [label="random"]; + "layoutsource" [label="layout"]; + "daemonsource" [label="daemon"]; + } + + subgraph cluster_mutate { + label = "mutate"; + "mutateconfig" [label="Config"]; + "mutatetime" [label="Time"]; + "mutatemediatype" [label="MediaType"]; + "mutateappend" [label="Append"]; + "mutaterebase" [label="Rebase"]; + } + + subgraph cluster_sinks { + label = "Sinks"; + labelloc = "b"; + + "remotesink" [label="remote"]; + "tarballsink" [label="tarball"]; + "legacy/tarballsink" [label="legacy/tarball"]; + "layoutsink" [label="layout"]; + "daemonsink" [label="daemon"]; + } + + "randomsource" -> input; + "layoutsource" -> input; + "daemonsource" -> input; + "tarballsource" -> input; + "remotesource" -> input; + + input -> "mutateconfig"; + input -> "mutatetime"; + input -> "mutatemediatype"; + input -> "mutateappend"; + input -> "mutaterebase"; + + "mutateconfig" -> output; + "mutatetime" -> output; + "mutatemediatype" -> output; + "mutateappend" -> output; + "mutaterebase" -> output; + + output -> "legacy/tarballsink"; + output -> "layoutsink"; + output -> "daemonsink"; + output -> "tarballsink"; + output -> "remotesink"; +} diff --git a/images/dot/remote.dot b/images/dot/remote.dot new file mode 100644 index 0000000..9b5e08c --- /dev/null +++ b/images/dot/remote.dot @@ -0,0 +1,66 @@ +digraph { + compound=true; + rankdir="LR"; + ordering = in; + + subgraph cluster_registry { + label = "registry"; + + subgraph cluster_tags { + label = "/v2/.../tags/list"; + + tag [label="tag", shape="rect"]; + tag2 [label="tag", shape="rect"]; + } + + subgraph cluster_manifests { + label = "/v2/.../manifests/"; + + subgraph cluster_manifest { + label = "manifest"; + + mconfig [label="config", shape="rect"]; + layers [label="layers", shape="rect"]; + } + + subgraph cluster_manifest2 { + label = "manifest"; + + mconfig2 [label="config", shape="rect"]; + layers2 [label="layers", shape="rect"]; + } + + subgraph cluster_index { + label = "index"; + + imanifest [label="manifests", shape="rect"]; + } + + imanifest -> mconfig [lhead=cluster_manifest]; + imanifest -> mconfig2 [lhead=cluster_manifest2]; + } + + subgraph cluster_blobs { + label = "/v2/.../blobs/"; + + bconfig [label="config", shape="hexagon"]; + bconfig2 [label="config", shape="hexagon"]; + + l1 [label="layer", shape="folder"]; + l2 [label="layer", shape="folder"]; + l3 [label="layer", shape="folder"]; + } + + layers -> l1; + layers -> l2; + + layers2 -> l2; + layers2 -> l3; + + mconfig -> bconfig; + mconfig2 -> bconfig2; + + tag -> mconfig [style="dashed", lhead=cluster_manifest]; + tag2 -> imanifest [style="dashed", lhead=cluster_index]; + } +} diff --git a/images/dot/stream.dot b/images/dot/stream.dot new file mode 100644 index 0000000..0987be7 --- /dev/null +++ b/images/dot/stream.dot @@ -0,0 +1,47 @@ +digraph G { + ordering=out; + + fs [label="input", shape="folder"]; + pr [label="io.PipeReader"]; + compressed [label="Compressed()", shape="rect"]; + rc2 [label="io.ReadCloser"]; + output [label="output", shape="cylinder"]; + + subgraph cluster_goroutine { + label = "goroutine"; + + rc [label="io.ReadCloser"]; + copy [label="io.Copy"]; + pw [label="io.PipeWriter"]; + mw [label="io.MultiWriter"]; + h1 [label="sha256.New"]; + gzip [label="gzip.Writer"]; + mw2 [label="io.MultiWriter"]; + h2 [label="sha256.New"]; + count [label="countWriter"]; + + size [label="Size()", shape="rect"]; + diffid [label="DiffID()", shape="rect"]; + digest [label="Digest()", shape="rect"]; + + + rc -> copy [style="bold"]; + copy -> mw [style="bold"]; + mw -> h1; + h1 -> diffid [style="dashed"]; + mw -> gzip [style="bold"]; + gzip -> mw2 [style="bold"]; + mw2 -> h2; + h2 -> digest [style="dashed"]; + mw2 -> count; + count -> size [style="dotted"]; + mw2 -> pw [style="bold"]; + }; + + fs -> rc [style="bold"]; + + pw -> pr [style="bold"]; + pr -> compressed [style="bold"]; + compressed -> rc2 [style="bold"]; + rc2 -> output [style="bold"]; +} diff --git a/images/dot/tarball.dot b/images/dot/tarball.dot new file mode 100644 index 0000000..595283f --- /dev/null +++ b/images/dot/tarball.dot @@ -0,0 +1,43 @@ +digraph { + compound=true; + rankdir="LR"; + ordering = out; + + subgraph cluster_tarball { + label = "image.tar"; + + subgraph cluster_manifest { + label = "manifest.json"; + + mconfig [label="Config", shape="rect"]; + layers [label="Layers", shape="rect"]; + sources [label="LayerSources", shape="rect"]; + tags [label="RepoTags", shape="rect"]; + } + + config [shape="note"]; + + mconfig -> config [label="image id"]; + + layers -> l1 [lhead=cluster_layer1, label="layer digest"]; + layers -> l2 [lhead=cluster_layer2, label="layer digest"]; + + config -> l1 [label="diffid"]; + config -> l2 [label="diffid"]; + + sources -> l1 [label="diffid"]; + sources -> l2 [label="diffid"]; + + subgraph cluster_layer1 { + label = "layer.tar.gz"; + margin = 20.0; + l1 [label="layer.tar", shape="folder"]; + } + + subgraph cluster_layer2 { + label = "layer.tar.gz"; + margin = 20.0; + l2 [label="layer.tar", shape="folder"]; + } + } +} diff --git a/images/dot/upload.dot b/images/dot/upload.dot new file mode 100644 index 0000000..2cb3e26 --- /dev/null +++ b/images/dot/upload.dot @@ -0,0 +1,67 @@ +digraph G { + ordering=out; + + fs [label="filesystem\nchangeset", shape=folder, href="https://github.com/opencontainers/image-spec/blob/master/layer.md"]; + configuration [label="image\nconfig", shape=hexagon, href="https://github.com/opencontainers/image-spec/blob/master/config.md#properties"]; + + tar [shape=rect]; + gzip [shape=rect]; + tee [shape=rect]; + tee2 [label=tee, shape=rect]; + tee3 [label=tee, shape=rect]; + sha256sum [shape=rect]; + sha256sum2 [label=sha256sum, shape=rect]; + sha256sum3 [label=sha256sum, shape=rect]; + curl [shape=rect]; + curl2 [label=curl, shape=rect]; + curl3 [label=curl, shape=rect]; + wc [label="wc -c", shape=rect]; + wc2 [label="wc -c", shape=rect]; + + config [label="config file", shape=note, href="https://github.com/opencontainers/image-spec/blob/master/config.md"]; + layer [shape=note, href="https://github.com/opencontainers/image-spec/blob/master/layer.md"]; + manifest [shape=note, href="https://github.com/opencontainers/image-spec/blob/master/manifest.md"]; + + registry [shape=cylinder, href="https://github.com/opencontainers/distribution-spec/blob/master/spec.md"]; + + config_size [label="config size"]; + layer_size [label="layer size"]; + config_digest [label="config digest\n(image id)", href="https://github.com/opencontainers/image-spec/blob/master/config.md#imageid"]; + layer_digest [label="layer digest"]; + + diffid [href="https://github.com/opencontainers/image-spec/blob/master/config.md#layer-diffid"]; + + configuration -> config; + fs -> tar; + + tar -> tee; + tee -> sha256sum; + sha256sum -> diffid [style=dashed]; + tee -> gzip; + gzip -> layer; + layer -> tee2; + tee2 -> sha256sum2; + sha256sum2 -> layer_digest [style=dashed]; + tee2 -> wc; + wc -> layer_size [style=dotted]; + layer_size -> manifest [style=dotted]; + tee2 -> curl; + + curl -> registry; + + diffid -> config [style=dashed]; + config -> tee3; + tee3 -> curl2; + curl2 -> registry; + + tee3 -> wc2; + tee3 -> sha256sum3; + wc2 -> config_size [style=dotted]; + sha256sum3 -> config_digest [style=dashed]; + + config_digest -> manifest [style=dashed]; + config_size -> manifest [style=dotted]; + layer_digest -> manifest [style=dashed]; + manifest -> curl3; + curl3 -> registry; +} diff --git a/images/gcrane.png b/images/gcrane.png new file mode 100644 index 0000000..461fbfe Binary files /dev/null and b/images/gcrane.png differ diff --git a/images/ggcr.dot.svg b/images/ggcr.dot.svg new file mode 100644 index 0000000..3dbeccd --- /dev/null +++ b/images/ggcr.dot.svg @@ -0,0 +1,874 @@ + + + + + + +godep + + + +bufio + + +bufio + + + + + +bytes + + +bytes + + + + + +context + + +context + + + + + +encoding/base64 + + +encoding/base64 + + + + + +encoding/json + + +encoding/json + + + + + +errors + + +errors + + + + + +fmt + + +fmt + + + + + +github.com/docker/cli/cli/config + + +github.com/docker/cli/cli/config + + + + + +github.com/docker/cli/cli/config->fmt + + + + + +github.com/docker/cli/cli/config/configfile + + +github.com/docker/cli/cli/config/configfile + + + + + +github.com/docker/cli/cli/config->github.com/docker/cli/cli/config/configfile + + + + + +github.com/docker/cli/cli/config/credentials + + +github.com/docker/cli/cli/config/credentials + + + + + +github.com/docker/cli/cli/config->github.com/docker/cli/cli/config/credentials + + + + + +github.com/docker/cli/cli/config/types + + +github.com/docker/cli/cli/config/types + + + + + +github.com/docker/cli/cli/config->github.com/docker/cli/cli/config/types + + + + + +github.com/docker/docker/pkg/homedir + + +github.com/docker/docker/pkg/homedir + + + + + +github.com/docker/cli/cli/config->github.com/docker/docker/pkg/homedir + + + + + +github.com/pkg/errors + + +github.com/pkg/errors + + + + + +github.com/docker/cli/cli/config->github.com/pkg/errors + + + + + +io + + +io + + + + + +github.com/docker/cli/cli/config->io + + + + + +os + + +os + + + + + +github.com/docker/cli/cli/config->os + + + + + +path/filepath + + +path/filepath + + + + + +github.com/docker/cli/cli/config->path/filepath + + + + + +strings + + +strings + + + + + +github.com/docker/cli/cli/config->strings + + + + + +github.com/docker/cli/cli/config/configfile->encoding/base64 + + + + + +github.com/docker/cli/cli/config/configfile->encoding/json + + + + + +github.com/docker/cli/cli/config/configfile->fmt + + + + + +github.com/docker/cli/cli/config/configfile->github.com/docker/cli/cli/config/credentials + + + + + +github.com/docker/cli/cli/config/configfile->github.com/docker/cli/cli/config/types + + + + + +github.com/docker/cli/cli/config/configfile->github.com/pkg/errors + + + + + +github.com/docker/cli/cli/config/configfile->io + + + + + +github.com/docker/cli/cli/config/configfile->os + + + + + +github.com/docker/cli/cli/config/configfile->path/filepath + + + + + +github.com/docker/cli/cli/config/configfile->strings + + + + + +io/ioutil + + +io/ioutil + + + + + +github.com/docker/cli/cli/config/configfile->io/ioutil + + + + + +github.com/docker/cli/cli/config/credentials->github.com/docker/cli/cli/config/types + + + + + +github.com/docker/cli/cli/config/credentials->strings + + + + + +github.com/docker/docker-credential-helpers/client + + +github.com/docker/docker-credential-helpers/client + + + + + +github.com/docker/cli/cli/config/credentials->github.com/docker/docker-credential-helpers/client + + + + + +github.com/docker/docker-credential-helpers/credentials + + +github.com/docker/docker-credential-helpers/credentials + + + + + +github.com/docker/cli/cli/config/credentials->github.com/docker/docker-credential-helpers/credentials + + + + + +os/exec + + +os/exec + + + + + +github.com/docker/cli/cli/config/credentials->os/exec + + + + + +github.com/docker/docker/pkg/homedir->errors + + + + + +github.com/docker/docker/pkg/homedir->os + + + + + +github.com/docker/docker/pkg/homedir->path/filepath + + + + + +github.com/docker/docker/pkg/homedir->strings + + + + + +os/user + + +os/user + + + + + +github.com/docker/docker/pkg/homedir->os/user + + + + + +github.com/pkg/errors->fmt + + + + + +github.com/pkg/errors->io + + + + + +github.com/pkg/errors->strings + + + + + +path + + +path + + + + + +github.com/pkg/errors->path + + + + + +runtime + + +runtime + + + + + +github.com/pkg/errors->runtime + + + + + +github.com/docker/docker-credential-helpers/client->bytes + + + + + +github.com/docker/docker-credential-helpers/client->encoding/json + + + + + +github.com/docker/docker-credential-helpers/client->fmt + + + + + +github.com/docker/docker-credential-helpers/client->io + + + + + +github.com/docker/docker-credential-helpers/client->os + + + + + +github.com/docker/docker-credential-helpers/client->strings + + + + + +github.com/docker/docker-credential-helpers/client->github.com/docker/docker-credential-helpers/credentials + + + + + +github.com/docker/docker-credential-helpers/client->os/exec + + + + + +github.com/docker/docker-credential-helpers/credentials->bufio + + + + + +github.com/docker/docker-credential-helpers/credentials->bytes + + + + + +github.com/docker/docker-credential-helpers/credentials->encoding/json + + + + + +github.com/docker/docker-credential-helpers/credentials->fmt + + + + + +github.com/docker/docker-credential-helpers/credentials->io + + + + + +github.com/docker/docker-credential-helpers/credentials->os + + + + + +github.com/docker/docker-credential-helpers/credentials->strings + + + + + +github.com/google/go-containerregistry/pkg/authn + + +github.com/google/go-containerregistry/pkg/authn + + + + + +github.com/google/go-containerregistry/pkg/authn->encoding/json + + + + + +github.com/google/go-containerregistry/pkg/authn->github.com/docker/cli/cli/config + + + + + +github.com/google/go-containerregistry/pkg/authn->github.com/docker/cli/cli/config/types + + + + + +github.com/google/go-containerregistry/pkg/authn->os + + + + + +github.com/google/go-containerregistry/pkg/logs + + +github.com/google/go-containerregistry/pkg/logs + + + + + +github.com/google/go-containerregistry/pkg/authn->github.com/google/go-containerregistry/pkg/logs + + + + + +github.com/google/go-containerregistry/pkg/name + + +github.com/google/go-containerregistry/pkg/name + + + + + +github.com/google/go-containerregistry/pkg/authn->github.com/google/go-containerregistry/pkg/name + + + + + +github.com/google/go-containerregistry/pkg/logs->io/ioutil + + + + + +log + + +log + + + + + +github.com/google/go-containerregistry/pkg/logs->log + + + + + +github.com/google/go-containerregistry/pkg/name->fmt + + + + + +github.com/google/go-containerregistry/pkg/name->strings + + + + + +net + + +net + + + + + +github.com/google/go-containerregistry/pkg/name->net + + + + + +net/url + + +net/url + + + + + +github.com/google/go-containerregistry/pkg/name->net/url + + + + + +regexp + + +regexp + + + + + +github.com/google/go-containerregistry/pkg/name->regexp + + + + + +unicode/utf8 + + +unicode/utf8 + + + + + +github.com/google/go-containerregistry/pkg/name->unicode/utf8 + + + + + +github.com/google/go-containerregistry/pkg/internal/retry + + +github.com/google/go-containerregistry/pkg/internal/retry + + + + + +github.com/google/go-containerregistry/pkg/internal/retry->context + + + + + +github.com/google/go-containerregistry/pkg/internal/retry->fmt + + + + + +github.com/google/go-containerregistry/pkg/internal/retry/wait + + +github.com/google/go-containerregistry/pkg/internal/retry/wait + + + + + +github.com/google/go-containerregistry/pkg/internal/retry->github.com/google/go-containerregistry/pkg/internal/retry/wait + + + + + +github.com/google/go-containerregistry/pkg/internal/retry/wait->errors + + + + + +math/rand + + +math/rand + + + + + +github.com/google/go-containerregistry/pkg/internal/retry/wait->math/rand + + + + + +time + + +time + + + + + +github.com/google/go-containerregistry/pkg/internal/retry/wait->time + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport + + +github.com/google/go-containerregistry/pkg/v1/remote/transport + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->encoding/base64 + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->encoding/json + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->fmt + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->strings + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->io/ioutil + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->github.com/google/go-containerregistry/pkg/authn + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->github.com/google/go-containerregistry/pkg/logs + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->github.com/google/go-containerregistry/pkg/name + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->github.com/google/go-containerregistry/pkg/internal/retry + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->time + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->net + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->net/url + + + + + +net/http + + +net/http + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->net/http + + + + + +net/http/httputil + + +net/http/httputil + + + + + +github.com/google/go-containerregistry/pkg/v1/remote/transport->net/http/httputil + + + + + diff --git a/images/image-anatomy.dot.svg b/images/image-anatomy.dot.svg new file mode 100644 index 0000000..d9fdaa5 --- /dev/null +++ b/images/image-anatomy.dot.svg @@ -0,0 +1,99 @@ + + + + + + +%3 + + +cluster_layer1 + +layer.tar.gz + + +cluster_layer2 + +layer.tar.gz + + + +tag + + + + +manifest + + + +manifest + + + +tag:head->manifest + + +digest +tag + + + +config + + + +config + + + +manifest->config + + +(image id) + + + +l1 + +layer.tar + + + +manifest->l1 + + +layer digest + + + +l2 + +layer.tar + + + +manifest->l2 + + +layer digest + + + +config->l1 + + +diffid + + + +config->l2 + + +diffid + + + diff --git a/images/index-anatomy-strange.dot.svg b/images/index-anatomy-strange.dot.svg new file mode 100644 index 0000000..f698139 --- /dev/null +++ b/images/index-anatomy-strange.dot.svg @@ -0,0 +1,125 @@ + + + + + + +%3 + + + +tag + + + + +index + + + +index + + + +tag:head->index + + +r124356 + + + +tag2 + + + + +index2 + + + +index + + + +tag2:head->index2 + + +stable-release + + + +tag3 + + + + +image + + + +image + + + +tag3:head->image + + +v1.0 + + + +index->index2 + + + + + +index->image + + + + + +xml + +xml + + + +index->xml + + + + + +image2 + + + +image + + + +index2->image2 + + + + + +image3 + + + +image + + + +index2->image3 + + + + + diff --git a/images/index-anatomy.dot.svg b/images/index-anatomy.dot.svg new file mode 100644 index 0000000..55e16a6 --- /dev/null +++ b/images/index-anatomy.dot.svg @@ -0,0 +1,85 @@ + + + + + + +%3 + + + +tag + + + + +index + + + +index + + + +tag:head->index + + +latest + + + +tag2 + + + + +image + + + +image + + + +tag2:head->image + + +amd64 + + + +tag3 + + + + +image2 + + + +image + + + +tag3:head->image2 + + +ppc64le + + + +index->image + + + + + +index->image2 + + + + + diff --git a/images/mutate.dot.svg b/images/mutate.dot.svg new file mode 100644 index 0000000..e493588 --- /dev/null +++ b/images/mutate.dot.svg @@ -0,0 +1,250 @@ + + + + + + +%3 + + +cluster_source + +Sources + + +cluster_mutate + +mutate + + +cluster_sinks + +Sinks + + + +input + +v1.Image + + + +mutateconfig + +Config + + + +input->mutateconfig + + + + + +mutatetime + +Time + + + +input->mutatetime + + + + + +mutatemediatype + +MediaType + + + +input->mutatemediatype + + + + + +mutateappend + +Append + + + +input->mutateappend + + + + + +mutaterebase + +Rebase + + + +input->mutaterebase + + + + + +output + +v1.Image + + + +remotesink + +remote + + + +output->remotesink + + + + + +tarballsink + +tarball + + + +output->tarballsink + + + + + +legacy/tarballsink + +legacy/tarball + + + +output->legacy/tarballsink + + + + + +layoutsink + +layout + + + +output->layoutsink + + + + + +daemonsink + +daemon + + + +output->daemonsink + + + + + +remotesource + +remote + + + +remotesource->input + + + + + +tarballsource + +tarball + + + +tarballsource->input + + + + + +randomsource + +random + + + +randomsource->input + + + + + +layoutsource + +layout + + + +layoutsource->input + + + + + +daemonsource + +daemon + + + +daemonsource->input + + + + + +mutateconfig->output + + + + + +mutatetime->output + + + + + +mutatemediatype->output + + + + + +mutateappend->output + + + + + +mutaterebase->output + + + + + diff --git a/images/ociimage.gv b/images/ociimage.gv new file mode 100644 index 0000000..5fbe947 --- /dev/null +++ b/images/ociimage.gv @@ -0,0 +1,97 @@ +digraph ociimage { + rankdir=LR; + node [shape=box]; + edge [splines=polyline]; + lrank [style=invisible][color=white]; + + "manifest A"[label=< + + + + + + + +
image manifest (platform A)
- schema version
- media type
- config : descriptor
- layers : array of descriptors
- (annotations)
>]; + + "image index"[label=< + + + + + + +
image index
- schema version
- media type
- manifests : array of descriptors
- (annotations)
>]; + + // references + edge [color=red][style=dashed]; + client [style=invisible][color=white]; + client -> "image index"[label="image reference"]; + client -> "manifest A"[label="image reference"]; + + // descriptors + edge [color=brown][style=solid]; + "image index" -> "manifest A"; + "image index" -> "image manifest (platform B)"; + "configuration"[label=< + + + + + +
configuration
- rootfs/diff_ids : array of layer ids
- container config
- history
>]; + "manifest A" -> "configuration"; + "layer 0"[label=< + + + +
layer
file system additions, overwrites, and deletions
>]; + "layer 1"[label=layer]; + "layer 2"[label=layer]; + "manifest A" -> "layer 0"[label=0]; + "manifest A" -> "layer 1"[label=1]; + "manifest A" -> "layer 2"[label=2]; + + // ids + edge [color=blue][style=dotted]; + "client" -> "configuration"[label="image id"]; + "configuration" -> "layer 0"[label=0]; + "configuration" -> "layer 1"[label=1]; + "configuration" -> "layer 2"[label=2]; + + // key + subgraph cluster { + k1 [label="Key:"][peripheries="0"]; + node [style=invisible][color=white]; + k2; + k3; + k4; + node [style=solid][color=black]; + k1 -> k2[color=red][style=dashed][label=< + + + + + + +
image reference
- hostname
- path
- (tag)
- (SHA-256 digest of compressed content)
>]; + k2 -> k3[color=brown][style=solid][label=< + + + + + + + + +
descriptor
targets content with the following properties:
- media type
- SHA-256 digest of compressed content
- size
- (urls)
- (annotations)
>]; + k3 -> k4[color=blue][style=dotted][label=< + + + +
id
- SHA-256 digest of uncompressed content
>]; + } + + { rank=same; lrank -> "layer 2" -> "layer 1" -> "layer 0" [style=invis] } + { rank=same; "manifest A", "image manifest (platform B)" } +} \ No newline at end of file diff --git a/images/ociimage.jpeg b/images/ociimage.jpeg new file mode 100644 index 0000000..b1e0ca5 Binary files /dev/null and b/images/ociimage.jpeg differ diff --git a/images/remote.dot.svg b/images/remote.dot.svg new file mode 100644 index 0000000..a4b5fae --- /dev/null +++ b/images/remote.dot.svg @@ -0,0 +1,180 @@ + + + + + + +%3 + + +cluster_registry + +registry + + +cluster_tags + +/v2/.../tags/list + + +cluster_manifests + +/v2/.../manifests/<ref> + + +cluster_manifest + +manifest + + +cluster_manifest2 + +manifest + + +cluster_index + +index + + +cluster_blobs + +/v2/.../blobs/<sha256> + + + +tag + +tag + + + +mconfig + +config + + + +tag->mconfig + + + + + +tag2 + +tag + + + +imanifest + +manifests + + + +tag2->imanifest + + + + + +bconfig + +config + + + +mconfig->bconfig + + + + + +layers + +layers + + + +l1 + +layer + + + +layers->l1 + + + + + +l2 + +layer + + + +layers->l2 + + + + + +mconfig2 + +config + + + +bconfig2 + +config + + + +mconfig2->bconfig2 + + + + + +layers2 + +layers + + + +layers2->l2 + + + + + +l3 + +layer + + + +layers2->l3 + + + + + +imanifest->mconfig + + + + + +imanifest->mconfig2 + + + + + diff --git a/images/stream.dot.svg b/images/stream.dot.svg new file mode 100644 index 0000000..3f3f04e --- /dev/null +++ b/images/stream.dot.svg @@ -0,0 +1,217 @@ + + + + + + +G + + +cluster_goroutine + +goroutine + + + +fs + +input + + + +rc + +io.ReadCloser + + + +fs->rc + + + + + +pr + +io.PipeReader + + + +compressed + +Compressed() + + + +pr->compressed + + + + + +rc2 + +io.ReadCloser + + + +compressed->rc2 + + + + + +output + + +output + + + +rc2->output + + + + + +copy + +io.Copy + + + +rc->copy + + + + + +mw + +io.MultiWriter + + + +copy->mw + + + + + +pw + +io.PipeWriter + + + +pw->pr + + + + + +h1 + +sha256.New + + + +mw->h1 + + + + + +gzip + +gzip.Writer + + + +mw->gzip + + + + + +diffid + +DiffID() + + + +h1->diffid + + + + + +mw2 + +io.MultiWriter + + + +gzip->mw2 + + + + + +mw2->pw + + + + + +h2 + +sha256.New + + + +mw2->h2 + + + + + +count + +countWriter + + + +mw2->count + + + + + +digest + +Digest() + + + +h2->digest + + + + + +size + +Size() + + + +count->size + + + + + diff --git a/images/tarball.dot.svg b/images/tarball.dot.svg new file mode 100644 index 0000000..4c6edc0 --- /dev/null +++ b/images/tarball.dot.svg @@ -0,0 +1,126 @@ + + + + + + +%3 + + +cluster_tarball + +image.tar + + +cluster_manifest + +manifest.json + + +cluster_layer1 + +layer.tar.gz + + +cluster_layer2 + +layer.tar.gz + + + +mconfig + +Config + + + +config + + + +config + + + +mconfig->config + + +image id + + + +layers + +Layers + + + +l1 + +layer.tar + + + +layers->l1 + + +layer digest + + + +l2 + +layer.tar + + + +layers->l2 + + +layer digest + + + +sources + +LayerSources + + + +sources->l1 + + +diffid + + + +sources->l2 + + +diffid + + + +tags + +RepoTags + + + +config->l1 + + +diffid + + + +config->l2 + + +diffid + + + diff --git a/images/upload.dot.svg b/images/upload.dot.svg new file mode 100644 index 0000000..16ba738 --- /dev/null +++ b/images/upload.dot.svg @@ -0,0 +1,359 @@ + + + + + + +G + + + +fs + + +filesystem +changeset + + + + + +tar + +tar + + + +fs->tar + + + + + +configuration + + +image +config + + + + + +config + + + + +config file + + + + + +configuration->config + + + + + +tee + +tee + + + +tar->tee + + + + + +gzip + +gzip + + + +layer + + + + +layer + + + + + +gzip->layer + + + + + +tee->gzip + + + + + +sha256sum + +sha256sum + + + +tee->sha256sum + + + + + +tee2 + +tee + + + +sha256sum2 + +sha256sum + + + +tee2->sha256sum2 + + + + + +curl + +curl + + + +tee2->curl + + + + + +wc + +wc -c + + + +tee2->wc + + + + + +tee3 + +tee + + + +sha256sum3 + +sha256sum + + + +tee3->sha256sum3 + + + + + +curl2 + +curl + + + +tee3->curl2 + + + + + +wc2 + +wc -c + + + +tee3->wc2 + + + + + +diffid + + +diffid + + + + + +sha256sum->diffid + + + + + +layer_digest + +layer digest + + + +sha256sum2->layer_digest + + + + + +config_digest + + +config digest +(image id) + + + + + +sha256sum3->config_digest + + + + + +registry + + + +registry + + + + + +curl->registry + + + + + +curl2->registry + + + + + +curl3 + +curl + + + +curl3->registry + + + + + +layer_size + +layer size + + + +wc->layer_size + + + + + +config_size + +config size + + + +wc2->config_size + + + + + +config->tee3 + + + + + +layer->tee2 + + + + + +manifest + + + + +manifest + + + + + +manifest->curl3 + + + + + +config_size->manifest + + + + + +layer_size->manifest + + + + + +config_digest->manifest + + + + + +layer_digest->manifest + + + + + +diffid->config + + + + + diff --git a/internal/and/and_closer.go b/internal/and/and_closer.go new file mode 100644 index 0000000..14a05ea --- /dev/null +++ b/internal/and/and_closer.go @@ -0,0 +1,48 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package and provides helpers for adding Close to io.{Reader|Writer}. +package and + +import ( + "io" +) + +// ReadCloser implements io.ReadCloser by reading from a particular io.Reader +// and then calling the provided "Close()" method. +type ReadCloser struct { + io.Reader + CloseFunc func() error +} + +var _ io.ReadCloser = (*ReadCloser)(nil) + +// Close implements io.ReadCloser +func (rac *ReadCloser) Close() error { + return rac.CloseFunc() +} + +// WriteCloser implements io.WriteCloser by reading from a particular io.Writer +// and then calling the provided "Close()" method. +type WriteCloser struct { + io.Writer + CloseFunc func() error +} + +var _ io.WriteCloser = (*WriteCloser)(nil) + +// Close implements io.WriteCloser +func (wac *WriteCloser) Close() error { + return wac.CloseFunc() +} diff --git a/internal/and/and_closer_test.go b/internal/and/and_closer_test.go new file mode 100644 index 0000000..947ceae --- /dev/null +++ b/internal/and/and_closer_test.go @@ -0,0 +1,85 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package and + +import ( + "bytes" + "io" + "testing" +) + +func TestRead(t *testing.T) { + want := "asdf" + r := bytes.NewBufferString(want) + called := false + + rac := &ReadCloser{ + Reader: r, + CloseFunc: func() error { + called = true + return nil + }, + } + + data, err := io.ReadAll(rac) + if err != nil { + t.Errorf("ReadAll(rac) = %v", err) + } + if got := string(data); got != want { + t.Errorf("ReadAll(rac); got %q, want %q", got, want) + } + + if called { + t.Error("called before Close(); got true, wanted false") + } + if err := rac.Close(); err != nil { + t.Errorf("Close() = %v", err) + } + if !called { + t.Error("called after Close(); got false, wanted true") + } +} + +func TestWrite(t *testing.T) { + w := bytes.NewBuffer([]byte{}) + called := false + + wac := &WriteCloser{ + Writer: w, + CloseFunc: func() error { + called = true + return nil + }, + } + + want := "asdf" + if _, err := wac.Write([]byte(want)); err != nil { + t.Errorf("Write(%q); = %v", want, err) + } + + if called { + t.Error("called before Close(); got true, wanted false") + } + if err := wac.Close(); err != nil { + t.Errorf("Close() = %v", err) + } + if !called { + t.Error("called after Close(); got false, wanted true") + } + + if got := w.String(); got != want { + t.Errorf("w.String(); got %q, want %q", got, want) + } +} diff --git a/internal/cmd/edit.go b/internal/cmd/edit.go new file mode 100644 index 0000000..707e80c --- /dev/null +++ b/internal/cmd/edit.go @@ -0,0 +1,485 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/internal/editor" + "github.com/google/go-containerregistry/internal/verify" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/spf13/cobra" +) + +// NewCmdEdit creates a new cobra.Command for the edit subcommand. +// +// This is currently hidden until we're happy with the interface and can test +// it on different operating systems and editors. +func NewCmdEdit(options *[]crane.Option) *cobra.Command { + cmd := &cobra.Command{ + Hidden: true, + Use: "edit", + Short: "Edit the contents of an image.", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, _ []string) { + cmd.Usage() + }, + } + cmd.AddCommand(NewCmdEditManifest(options), NewCmdEditConfig(options), NewCmdEditFs(options)) + + return cmd +} + +// NewCmdConfig creates a new cobra.Command for the config subcommand. +func NewCmdEditConfig(options *[]crane.Option) *cobra.Command { + var dst string + cmd := &cobra.Command{ + Use: "config", + Short: "Edit an image's config file.", + Example: ` # Edit ubuntu's config file + crane edit config ubuntu + + # Overwrite ubuntu's config file with '{}' + echo '{}' | crane edit config ubuntu`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ref, err := editConfig(cmd.InOrStdin(), cmd.OutOrStdout(), args[0], dst, *options...) + if err != nil { + return fmt.Errorf("editing config: %w", err) + } + fmt.Println(ref.String()) + return nil + }, + } + cmd.Flags().StringVarP(&dst, "tag", "t", "", "New tag reference to apply to mutated image. If not provided, uses original tag or pushes a new digest.") + + return cmd +} + +// NewCmdManifest creates a new cobra.Command for the manifest subcommand. +func NewCmdEditManifest(options *[]crane.Option) *cobra.Command { + var ( + dst string + mt string + ) + cmd := &cobra.Command{ + Use: "manifest", + Short: "Edit an image's manifest.", + Example: ` # Edit ubuntu's manifest + crane edit manifest ubuntu`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ref, err := editManifest(cmd.InOrStdin(), cmd.OutOrStdout(), args[0], dst, mt, *options...) + if err != nil { + return fmt.Errorf("editing manifest: %w", err) + } + fmt.Println(ref.String()) + return nil + }, + } + cmd.Flags().StringVarP(&dst, "tag", "t", "", "New tag reference to apply to mutated image. If not provided, uses original tag or pushes a new digest.") + cmd.Flags().StringVarP(&mt, "media-type", "m", "", "Override the mediaType used as the Content-Type for PUT") + + return cmd +} + +// NewCmdExport creates a new cobra.Command for the export subcommand. +func NewCmdEditFs(options *[]crane.Option) *cobra.Command { + var dst, name string + cmd := &cobra.Command{ + Use: "fs IMAGE", + Short: "Edit the contents of an image's filesystem.", + Example: ` # Edit motd-news using $EDITOR + crane edit fs ubuntu -f /etc/default/motd-news + + # Overwrite motd-news with 'ENABLED=0' + echo 'ENABLED=0' | crane edit fs ubuntu -f /etc/default/motd-news`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ref, err := editFile(cmd.InOrStdin(), cmd.OutOrStdout(), args[0], name, dst, *options...) + if err != nil { + return fmt.Errorf("editing file: %w", err) + } + fmt.Println(ref.String()) + return nil + }, + } + cmd.Flags().StringVarP(&name, "filename", "f", "", "Edit the given filename") + cmd.Flags().StringVarP(&dst, "tag", "t", "", "New tag reference to apply to mutated image. If not provided, uses original tag or pushes a new digest.") + cmd.MarkFlagRequired("filename") + + return cmd +} + +func interactive(in io.Reader, out io.Writer) bool { + return interactiveFile(in) && interactiveFile(out) +} + +func interactiveFile(i any) bool { + f, ok := i.(*os.File) + if !ok { + return false + } + stat, err := f.Stat() + if err != nil { + return false + } + return (stat.Mode() & os.ModeCharDevice) != 0 +} + +func editConfig(in io.Reader, out io.Writer, src, dst string, options ...crane.Option) (name.Reference, error) { + o := crane.GetOptions(options...) + + img, err := crane.Pull(src, options...) + if err != nil { + return nil, err + } + + mt, err := img.MediaType() + if err != nil { + return nil, err + } + + // We want to omit Layers in certain situations, so we don't use v1.Image.Manifest() here. + // Instead, we treat the manifest as a map[string]any and just manipulate the config desc. + mb, err := img.RawManifest() + if err != nil { + return nil, err + } + + jsonMap := map[string]any{} + if err := json.Unmarshal(mb, &jsonMap); err != nil { + return nil, err + } + + cv, ok := jsonMap["config"] + if !ok { + return nil, fmt.Errorf("config missing") + } + cb, err := json.Marshal(cv) + if err != nil { + return nil, fmt.Errorf("json.Marshal config: %w", err) + } + + config := v1.Descriptor{} + if err := json.Unmarshal(cb, &config); err != nil { + return nil, fmt.Errorf("json.Unmarshal config: %w", err) + } + + var edited []byte + if interactive(in, out) { + rcf, err := img.RawConfigFile() + if err != nil { + return nil, err + } + edited, err = editor.Edit(bytes.NewReader(rcf), ".json") + if err != nil { + return nil, err + } + } else { + b, err := io.ReadAll(in) + if err != nil { + return nil, err + } + edited = b + } + + // this has to happen before we modify the descriptor (so we can use verify.Descriptor to validate whether m.Config.Data matches m.Config.Digest/Size) + if config.Data != nil && verify.Descriptor(config) == nil { + // https://github.com/google/go-containerregistry/issues/1552#issuecomment-1452653875 + // "if data is non-empty and correct, we should update it" + config.Data = edited + } + + l := static.NewLayer(edited, config.MediaType) + layerDigest, err := l.Digest() + if err != nil { + return nil, err + } + + config.Digest = layerDigest + config.Size = int64(len(edited)) + + jsonMap["config"] = config + b, err := json.Marshal(jsonMap) + if err != nil { + return nil, err + } + rm := &rawManifest{ + body: b, + mediaType: mt, + } + + digest, _, _ := v1.SHA256(bytes.NewReader(b)) + + if dst == "" { + dst = src + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return nil, err + } + if _, ok := ref.(name.Digest); ok { + dst = ref.Context().Digest(digest.String()).String() + } + } + + dstRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + return nil, err + } + + if err := remote.WriteLayer(dstRef.Context(), l, o.Remote...); err != nil { + return nil, err + } + + if err := remote.Put(dstRef, rm, o.Remote...); err != nil { + return nil, err + } + + return dstRef, nil +} + +func editManifest(in io.Reader, out io.Writer, src string, dst string, mt string, options ...crane.Option) (name.Reference, error) { + o := crane.GetOptions(options...) + + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return nil, err + } + + desc, err := remote.Get(ref, o.Remote...) + if err != nil { + return nil, err + } + + var edited []byte + if interactive(in, out) { + edited, err = editor.Edit(bytes.NewReader(desc.Manifest), ".json") + if err != nil { + return nil, err + } + } else { + b, err := io.ReadAll(in) + if err != nil { + return nil, err + } + edited = b + } + + digest, _, err := v1.SHA256(bytes.NewReader(edited)) + if err != nil { + return nil, err + } + + if dst == "" { + dst = src + if _, ok := ref.(name.Digest); ok { + dst = ref.Context().Digest(digest.String()).String() + } + } + dstRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + return nil, err + } + + if mt == "" { + // If --media-type is unset, use Content-Type by default. + mt = string(desc.MediaType) + + // If document contains mediaType, default to that. + wmt := withMediaType{} + if err := json.Unmarshal(edited, &wmt); err == nil { + if wmt.MediaType != "" { + mt = wmt.MediaType + } + } + } + + rm := &rawManifest{ + body: edited, + mediaType: types.MediaType(mt), + } + + if err := remote.Put(dstRef, rm, o.Remote...); err != nil { + return nil, err + } + + return dstRef, nil +} + +func editFile(in io.Reader, out io.Writer, src, file, dst string, options ...crane.Option) (name.Reference, error) { + o := crane.GetOptions(options...) + + img, err := crane.Pull(src, options...) + if err != nil { + return nil, err + } + + // If stdin has content, read it in and use that for the file. + // Otherwise, scran through the image and open that file in an editor. + var ( + edited []byte + header *tar.Header + ) + if interactive(in, out) { + f, h, err := findFile(img, file) + if err != nil { + return nil, err + } + ext := filepath.Ext(h.Name) + if strings.Contains(ext, "..") { + return nil, fmt.Errorf("this is impossible but this check satisfies CWE-22 for file name %q", h.Name) + } + edited, err = editor.Edit(f, ext) + if err != nil { + return nil, err + } + header = h + } else { + b, err := io.ReadAll(in) + if err != nil { + return nil, err + } + edited = b + header = blankHeader(file) + } + + buf := bytes.NewBuffer(nil) + buf.Grow(len(edited)) + tw := tar.NewWriter(buf) + + header.Size = int64(len(edited)) + if err := tw.WriteHeader(header); err != nil { + return nil, err + } + if _, err := io.Copy(tw, bytes.NewReader(edited)); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + + fileBytes := buf.Bytes() + fileLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(fileBytes)), nil + }) + if err != nil { + return nil, err + } + img, err = mutate.Append(img, mutate.Addendum{ + Layer: fileLayer, + History: v1.History{ + Author: "crane", + CreatedBy: strings.Join(os.Args, " "), + }, + }) + if err != nil { + return nil, err + } + + digest, err := img.Digest() + if err != nil { + return nil, err + } + + if dst == "" { + dst = src + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return nil, err + } + if _, ok := ref.(name.Digest); ok { + dst = ref.Context().Digest(digest.String()).String() + } + } + + dstRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + return nil, err + } + + if err := crane.Push(img, dst, options...); err != nil { + return nil, err + } + + return dstRef, nil +} + +func findFile(img v1.Image, name string) (io.Reader, *tar.Header, error) { + name = normalize(name) + tr := tar.NewReader(mutate.Extract(img)) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, nil, fmt.Errorf("reading tar: %w", err) + } + if normalize(header.Name) == name { + return tr, header, nil + } + } + + // If we don't find the file, we should create a new one. + return bytes.NewBufferString(""), blankHeader(name), nil +} + +func blankHeader(name string) *tar.Header { + return &tar.Header{ + Name: name, + Typeflag: tar.TypeReg, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + } +} + +func normalize(name string) string { + return filepath.Clean("/" + name) +} + +type withMediaType struct { + MediaType string `json:"mediaType,omitempty"` +} + +type rawManifest struct { + body []byte + mediaType types.MediaType +} + +func (r *rawManifest) RawManifest() ([]byte, error) { + return r.body, nil +} + +func (r *rawManifest) MediaType() (types.MediaType, error) { + return r.mediaType, nil +} diff --git a/internal/cmd/edit_test.go b/internal/cmd/edit_test.go new file mode 100644 index 0000000..df18c45 --- /dev/null +++ b/internal/cmd/edit_test.go @@ -0,0 +1,174 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "io" + "net/http/httptest" + "net/url" + "path" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func mustRegistry(t *testing.T) (*httptest.Server, string) { + t.Helper() + s := httptest.NewServer(registry.New()) + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + return s, u.Host +} + +func TestEditConfig(t *testing.T) { + reg, host := mustRegistry(t) + defer reg.Close() + src := path.Join(host, "crane/edit/config") + + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + if err := crane.Push(img, src); err != nil { + t.Fatal(err) + } + + cmd := NewCmdEditConfig(&[]crane.Option{}) + cmd.SetArgs([]string{src}) + cmd.SetIn(strings.NewReader("{}")) + + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } +} + +func TestEditManifest(t *testing.T) { + reg, host := mustRegistry(t) + defer reg.Close() + src := path.Join(host, "crane/edit/manifest") + + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + if err := crane.Push(img, src); err != nil { + t.Fatal(err) + } + + cmd := NewCmdEditManifest(&[]crane.Option{}) + cmd.SetArgs([]string{src}) + cmd.SetIn(strings.NewReader("{}")) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } +} + +func TestEditFilesystem(t *testing.T) { + reg, host := mustRegistry(t) + defer reg.Close() + src := path.Join(host, "crane/edit/fs") + + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + if err := crane.Push(img, src); err != nil { + t.Fatal(err) + } + + cmd := NewCmdEditFs(&[]crane.Option{}) + cmd.SetArgs([]string{src}) + cmd.Flags().Set("filename", "/foo/bar") + cmd.SetIn(strings.NewReader("baz")) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + + img, err = crane.Pull(src) + if err != nil { + t.Fatal(err) + } + + r, _, err := findFile(img, "/foo/bar") + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(got, []byte("baz")) { + t.Fatalf("got: %s, want %s", got, "baz") + } + + // Edit the same file to make sure we can edit existing files. + cmd = NewCmdEditFs(&[]crane.Option{}) + cmd.SetArgs([]string{src}) + cmd.Flags().Set("filename", "/foo/bar") + cmd.SetIn(strings.NewReader("quux")) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + + img, err = crane.Pull(src) + if err != nil { + t.Fatal(err) + } + + r, _, err = findFile(img, "/foo/bar") + if err != nil { + t.Fatal(err) + } + + got, err = io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(got, []byte("quux")) { + t.Fatalf("got: %s, want %s", got, "quux") + } +} + +func TestFindFile(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + r, h, err := findFile(img, "/does-not-exist") + if err != nil { + t.Fatal(err) + } + + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if len(b) != 0 { + t.Errorf("expected empty reader, got: %s", string(b)) + } + + if h.Name != "/does-not-exist" { + t.Errorf("tar.Header has wrong name: %v", h) + } +} diff --git a/internal/compare/doc.go b/internal/compare/doc.go new file mode 100644 index 0000000..c8ca497 --- /dev/null +++ b/internal/compare/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compare provides methods for comparing images, indexes, and layers. +package compare diff --git a/internal/compare/image.go b/internal/compare/image.go new file mode 100644 index 0000000..fe5fd4c --- /dev/null +++ b/internal/compare/image.go @@ -0,0 +1,111 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "errors" + "fmt" + "reflect" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Images compares the given images to each other and returns an error if they +// differ. +func Images(a, b v1.Image) error { + digests := []v1.Hash{} + manifests := []*v1.Manifest{} + cns := []v1.Hash{} + sizes := []int64{} + mts := []types.MediaType{} + layerss := [][]v1.Layer{} + + errs := []string{} + + for _, img := range []v1.Image{a, b} { + layers, err := img.Layers() + if err != nil { + return err + } + layerss = append(layerss, layers) + + digest, err := img.Digest() + if err != nil { + return err + } + digests = append(digests, digest) + + manifest, err := img.Manifest() + if err != nil { + return err + } + manifests = append(manifests, manifest) + + cn, err := img.ConfigName() + if err != nil { + return err + } + cns = append(cns, cn) + + size, err := img.Size() + if err != nil { + return err + } + sizes = append(sizes, size) + + mt, err := img.MediaType() + if err != nil { + return err + } + mts = append(mts, mt) + } + + if want, got := digests[0], digests[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Digest() != b.Digest(); %s != %s", want, got)) + } + if want, got := cns[0], cns[1]; want != got { + errs = append(errs, fmt.Sprintf("a.ConfigName() != b.ConfigName(); %s != %s", want, got)) + } + if want, got := manifests[0], manifests[1]; !reflect.DeepEqual(want, got) { + errs = append(errs, fmt.Sprintf("a.Manifest() != b.Manifest(); %v != %v", want, got)) + } + if want, got := sizes[0], sizes[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Size() != b.Size(); %d != %d", want, got)) + } + if want, got := mts[0], mts[1]; want != got { + errs = append(errs, fmt.Sprintf("a.MediaType() != b.MediaType(); %s != %s", want, got)) + } + + if len(layerss[0]) != len(layerss[1]) { + // If we have fewer layers than the first image, abort with an error so we don't panic. + return errors.New("len(a.Layers()) != len(b.Layers())") + } + + // Compare each layer. + for i := 0; i < len(layerss[0]); i++ { + if err := Layers(layerss[0][i], layerss[1][i]); err != nil { + // Wrap the error in newlines to delineate layer errors. + errs = append(errs, fmt.Sprintf("Layers[%d]: %v\n", i, err)) + } + } + + if len(errs) != 0 { + return errors.New("Images differ:\n" + strings.Join(errs, "\n")) + } + + return nil +} diff --git a/internal/compare/image_test.go b/internal/compare/image_test.go new file mode 100644 index 0000000..fe011a4 --- /dev/null +++ b/internal/compare/image_test.go @@ -0,0 +1,66 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestDifferentImages(t *testing.T) { + a, err := random.Image(100, 3) + if err != nil { + t.Fatal(err) + } + b, err := random.Image(100, 3) + if err != nil { + t.Fatal(err) + } + + b = mutate.MediaType(b, types.OCIManifestSchema1) + + if err := Images(a, b); err == nil { + t.Errorf("got nil err, should have something") + } +} + +func TestMismatchedLayers(t *testing.T) { + a, err := random.Image(100, 3) + if err != nil { + t.Fatal(err) + } + b, err := random.Image(100, 2) + if err != nil { + t.Fatal(err) + } + + if err := Images(a, b); err == nil { + t.Errorf("got nil err, should have something") + } +} + +func TestEqualImages(t *testing.T) { + a, err := random.Image(100, 2) + if err != nil { + t.Fatal(err) + } + + if err := Images(a, a); err != nil { + t.Errorf("got err: %v", err) + } +} diff --git a/internal/compare/index.go b/internal/compare/index.go new file mode 100644 index 0000000..c19620b --- /dev/null +++ b/internal/compare/index.go @@ -0,0 +1,83 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "errors" + "fmt" + "reflect" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Indexes compares the given indexes to each other and returns an error if +// they differ. +func Indexes(a, b v1.ImageIndex) error { + digests := []v1.Hash{} + manifests := []*v1.IndexManifest{} + sizes := []int64{} + mts := []types.MediaType{} + + errs := []string{} + + for _, idx := range []v1.ImageIndex{a, b} { + digest, err := idx.Digest() + if err != nil { + return err + } + digests = append(digests, digest) + + manifest, err := idx.IndexManifest() + if err != nil { + return err + } + manifests = append(manifests, manifest) + + size, err := idx.Size() + if err != nil { + return err + } + sizes = append(sizes, size) + + mt, err := idx.MediaType() + if err != nil { + return err + } + mts = append(mts, mt) + } + + if want, got := digests[0], digests[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Digest() != b.Digest(); %s != %s", want, got)) + } + if want, got := manifests[0], manifests[1]; !reflect.DeepEqual(want, got) { + errs = append(errs, fmt.Sprintf("a.Manifest() != b.Manifest(); %v != %v", want, got)) + } + if want, got := sizes[0], sizes[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Size() != b.Size(); %d != %d", want, got)) + } + if want, got := mts[0], mts[1]; want != got { + errs = append(errs, fmt.Sprintf("a.MediaType() != b.MediaType(); %s != %s", want, got)) + } + + // TODO(jonjohnsonjr): Iterate over Manifest and compare Image and ImageIndex results. + + if len(errs) != 0 { + return errors.New("Indexes differ:\n" + strings.Join(errs, "\n")) + } + + return nil +} diff --git a/internal/compare/index_test.go b/internal/compare/index_test.go new file mode 100644 index 0000000..962bf22 --- /dev/null +++ b/internal/compare/index_test.go @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestDifferentIndexes(t *testing.T) { + a, err := random.Index(100, 3, 3) + if err != nil { + t.Fatal(err) + } + b, err := random.Index(100, 2, 2) + if err != nil { + t.Fatal(err) + } + + b = mutate.IndexMediaType(b, types.DockerManifestList) + + if err := Indexes(a, b); err == nil { + t.Errorf("got nil err, should have something") + } +} + +func TestEqualIndexes(t *testing.T) { + a, err := random.Index(100, 2, 2) + if err != nil { + t.Fatal(err) + } + + if err := Indexes(a, a); err != nil { + t.Errorf("got err: %v", err) + } +} diff --git a/internal/compare/layer.go b/internal/compare/layer.go new file mode 100644 index 0000000..01e63aa --- /dev/null +++ b/internal/compare/layer.go @@ -0,0 +1,80 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "errors" + "fmt" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Layers compares the given layers to each other and returns an error if they +// differ. Note that this does not compare the actual contents (by calling +// Compressed or Uncompressed). +func Layers(a, b v1.Layer) error { + digests := []v1.Hash{} + diffids := []v1.Hash{} + sizes := []int64{} + mts := []types.MediaType{} + errs := []string{} + + for _, layer := range []v1.Layer{a, b} { + digest, err := layer.Digest() + if err != nil { + return err + } + digests = append(digests, digest) + + diffid, err := layer.DiffID() + if err != nil { + return err + } + diffids = append(diffids, diffid) + + size, err := layer.Size() + if err != nil { + return err + } + sizes = append(sizes, size) + + mt, err := layer.MediaType() + if err != nil { + return err + } + mts = append(mts, mt) + } + + if want, got := digests[0], digests[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Digest() != b.Digest(); %s != %s", want, got)) + } + if want, got := diffids[0], diffids[1]; want != got { + errs = append(errs, fmt.Sprintf("a.DiffID() != b.DiffID(); %s != %s", want, got)) + } + if want, got := sizes[0], sizes[1]; want != got { + errs = append(errs, fmt.Sprintf("a.Size() != b.Size(); %d != %d", want, got)) + } + if want, got := mts[0], mts[1]; want != got { + errs = append(errs, fmt.Sprintf("a.MediaType() != b.MediaType(); %s != %s", want, got)) + } + + if len(errs) != 0 { + return errors.New("Layers differ:\n" + strings.Join(errs, "\n")) + } + + return nil +} diff --git a/internal/compare/layer_test.go b/internal/compare/layer_test.go new file mode 100644 index 0000000..6db9751 --- /dev/null +++ b/internal/compare/layer_test.go @@ -0,0 +1,48 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestDifferentLayers(t *testing.T) { + a, err := random.Layer(100, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + b, err := random.Layer(100, types.OCILayer) + if err != nil { + t.Fatal(err) + } + + if err := Layers(a, b); err == nil { + t.Errorf("got nil err, should have something") + } +} + +func TestEqualLayers(t *testing.T) { + a, err := random.Layer(100, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + + if err := Layers(a, a); err != nil { + t.Errorf("got err: %v", err) + } +} diff --git a/internal/compression/compression.go b/internal/compression/compression.go new file mode 100644 index 0000000..0124871 --- /dev/null +++ b/internal/compression/compression.go @@ -0,0 +1,97 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compression abstracts over gzip and zstd. +package compression + +import ( + "bufio" + "bytes" + "io" + + "github.com/google/go-containerregistry/internal/gzip" + "github.com/google/go-containerregistry/internal/zstd" + "github.com/google/go-containerregistry/pkg/compression" +) + +// Opener represents e.g. opening a file. +type Opener = func() (io.ReadCloser, error) + +// GetCompression detects whether an Opener is compressed and which algorithm is used. +func GetCompression(opener Opener) (compression.Compression, error) { + rc, err := opener() + if err != nil { + return compression.None, err + } + defer rc.Close() + + cp, _, err := PeekCompression(rc) + if err != nil { + return compression.None, err + } + + return cp, nil +} + +// PeekCompression detects whether the input stream is compressed and which algorithm is used. +// +// If r implements Peek, we will use that directly, otherwise a small number +// of bytes are buffered to Peek at the gzip/zstd header, and the returned +// PeekReader can be used as a replacement for the consumed input io.Reader. +func PeekCompression(r io.Reader) (compression.Compression, PeekReader, error) { + pr := intoPeekReader(r) + + if isGZip, _, err := checkHeader(pr, gzip.MagicHeader); err != nil { + return compression.None, pr, err + } else if isGZip { + return compression.GZip, pr, nil + } + + if isZStd, _, err := checkHeader(pr, zstd.MagicHeader); err != nil { + return compression.None, pr, err + } else if isZStd { + return compression.ZStd, pr, nil + } + + return compression.None, pr, nil +} + +// PeekReader is an io.Reader that also implements Peek a la bufio.Reader. +type PeekReader interface { + io.Reader + Peek(n int) ([]byte, error) +} + +// IntoPeekReader creates a PeekReader from an io.Reader. +// If the reader already has a Peek method, it will just return the passed reader. +func intoPeekReader(r io.Reader) PeekReader { + if p, ok := r.(PeekReader); ok { + return p + } + + return bufio.NewReader(r) +} + +// CheckHeader checks whether the first bytes from a PeekReader match an expected header +func checkHeader(pr PeekReader, expectedHeader []byte) (bool, PeekReader, error) { + header, err := pr.Peek(len(expectedHeader)) + if err != nil { + // https://github.com/google/go-containerregistry/issues/367 + if err == io.EOF { + return false, pr, nil + } + return false, pr, err + } + return bytes.Equal(header, expectedHeader), pr, nil +} diff --git a/internal/compression/compression_test.go b/internal/compression/compression_test.go new file mode 100644 index 0000000..5279dfe --- /dev/null +++ b/internal/compression/compression_test.go @@ -0,0 +1,78 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compression + +import ( + "bytes" + "io" + "testing" + + "github.com/google/go-containerregistry/internal/and" + "github.com/google/go-containerregistry/internal/gzip" + "github.com/google/go-containerregistry/internal/zstd" +) + +type Compressor = func(rc io.ReadCloser) io.ReadCloser +type Decompressor = func(rc io.ReadCloser) (io.ReadCloser, error) + +func testPeekCompression(t *testing.T, + compressionExpected string, + compress Compressor, + decompress Decompressor, +) { + content := "This is the input string." + contentBuf := bytes.NewBufferString(content) + + compressed := compress(io.NopCloser(contentBuf)) + compressionDetected, pr, err := PeekCompression(compressed) + if err != nil { + t.Error("PeekCompression() =", err) + } + + if got := string(compressionDetected); got != compressionExpected { + t.Errorf("PeekCompression(); got %q, content %q", got, compressionExpected) + } + + decompressed, err := decompress(withCloser(pr, compressed)) + if err != nil { + t.Fatal(err) + } + + b, err := io.ReadAll(decompressed) + if err != nil { + t.Error("ReadAll() =", err) + } + + if got := string(b); got != content { + t.Errorf("ReadAll(); got %q, content %q", got, content) + } +} + +func TestPeekCompression(t *testing.T) { + testPeekCompression(t, "gzip", gzip.ReadCloser, gzip.UnzipReadCloser) + testPeekCompression(t, "zstd", zstd.ReadCloser, zstd.UnzipReadCloser) + + nopCompress := func(rc io.ReadCloser) io.ReadCloser { return rc } + nopDecompress := func(rc io.ReadCloser) (io.ReadCloser, error) { return rc, nil } + + testPeekCompression(t, "none", nopCompress, nopDecompress) +} + +func withCloser(pr PeekReader, rc io.ReadCloser) io.ReadCloser { + return &and.ReadCloser{ + Reader: pr, + CloseFunc: rc.Close, + } +} diff --git a/internal/depcheck/depcheck.go b/internal/depcheck/depcheck.go new file mode 100644 index 0000000..ba24665 --- /dev/null +++ b/internal/depcheck/depcheck.go @@ -0,0 +1,186 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package depcheck defines a test utility for ensuring certain packages don't +// take on heavy dependencies. +// +// This is forked from https://pkg.go.dev/knative.dev/pkg/depcheck +package depcheck + +import ( + "fmt" + "sort" + "strings" + "testing" + + "golang.org/x/tools/go/packages" +) + +type node struct { + importpath string + consumers map[string]struct{} +} + +type graph map[string]node + +func (g graph) contains(name string) bool { + _, ok := g[name] + return ok +} + +func (g graph) order() []string { + order := make(sort.StringSlice, 0, len(g)) + for k := range g { + order = append(order, k) + } + order.Sort() + return order +} + +// path constructs an examplary path that looks something like: +// +// knative.dev/pkg/apis/duck +// knative.dev/pkg/apis # Also: [knative.dev/pkg/kmeta knative.dev/pkg/tracker] +// k8s.io/api/core/v1 +func (g graph) path(name string) []string { + n := g[name] + // Base case. + if len(n.consumers) == 0 { + return []string{name} + } + // Inductive step. + consumers := make(sort.StringSlice, 0, len(n.consumers)) + for k := range n.consumers { + consumers = append(consumers, k) + } + consumers.Sort() + base := g.path(consumers[0]) + if len(base) > 1 { // Don't decorate the first entry, which is always an entrypoint. + if len(consumers) > 1 { + // Attach other consumers to the last entry in base. + base = append(base[:len(base)-1], fmt.Sprintf("%s # Also: %v", consumers[0], consumers[1:])) + } + } + return append(base, name) +} + +func buildGraph(importpath string, buildFlags ...string) (graph, error) { + g := make(graph, 1) + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedDeps | packages.NeedModule, + BuildFlags: buildFlags, + }, importpath) + if err != nil { + return nil, err + } + packages.Visit(pkgs, func(pkg *packages.Package) bool { + g[pkg.PkgPath] = node{ + importpath: pkg.PkgPath, + consumers: make(map[string]struct{}), + } + return pkg.Module != nil + }, func(pkg *packages.Package) { + for _, imp := range pkg.Imports { + if _, ok := g[imp.PkgPath]; ok { + g[imp.PkgPath].consumers[pkg.PkgPath] = struct{}{} + } + } + }) + return g, nil +} + +// StdlibPackages returns the list of all standard library packages, including +// some golang.org/x/ dependencies. +func StdlibPackages() []string { + // pkg/registry is allowed to depend on any stdlib package, so collect + // all of those -- this also includes golang.org/x/ packages. + pkgs, err := packages.Load(nil, "std") + if err != nil { + panic(fmt.Sprintf("Loading stdlib packages: %v", err)) + } + pkgnames := make([]string, len(pkgs)) + for idx, p := range pkgs { + pkgnames[idx] = p.PkgPath + } + return pkgnames +} + +// CheckNoDependency checks that the given import paths (ip) does not +// depend (transitively) on certain banned imports. +func CheckNoDependency(ip string, banned []string, buildFlags ...string) error { + g, err := buildGraph(ip, buildFlags...) + if err != nil { + return fmt.Errorf("buildGraph(%q) = %w", ip, err) + } + for _, dip := range banned { + if g.contains(dip) { + return fmt.Errorf("%s depends on banned dependency %s\n%s", ip, dip, + strings.Join(g.path(dip), "\n")) + } + } + return nil +} + +// AssertNoDependency checks that the given import paths (the keys) do not +// depend (transitively) on certain banned imports (the values) +func AssertNoDependency(t *testing.T, banned map[string][]string, buildFlags ...string) { + t.Helper() + for ip, banned := range banned { + t.Run(ip, func(t *testing.T) { + if err := CheckNoDependency(ip, banned, buildFlags...); err != nil { + t.Error("CheckNoDependency() =", err) + } + }) + } +} + +// AssertOnlyDependencies checks that the given import paths (the keys) only +// depend (transitively) on certain allowed imports (the values). +// Note: while perhaps counterintuitive we allow the value to be a superset +// of the actual imports to that folks can use a constant that holds blessed +// import paths. +func AssertOnlyDependencies(t *testing.T, allowed map[string][]string, buildFlags ...string) { + t.Helper() + for ip, allow := range allowed { + // Always include our own package in the set of allowed dependencies. + allowed := make(map[string]struct{}, len(allow)+1) + for _, x := range append(allow, ip) { + allowed[x] = struct{}{} + } + t.Run(ip, func(t *testing.T) { + if err := CheckOnlyDependencies(ip, allowed, buildFlags...); err != nil { + t.Error("CheckOnlyDependencies() =", err) + } + }) + } +} + +// CheckOnlyDependencies checks that the given import path only +// depends (transitively) on certain allowed imports. +// Note: while perhaps counterintuitive we allow the value to be a superset +// of the actual imports to that folks can use a constant that holds blessed +// import paths. +func CheckOnlyDependencies(ip string, allowed map[string]struct{}, buildFlags ...string) error { + g, err := buildGraph(ip, buildFlags...) + if err != nil { + return fmt.Errorf("buildGraph(%q) = %w", ip, err) + } + for _, name := range g.order() { + if _, ok := allowed[name]; !ok { + return fmt.Errorf("dependency %s of %s is not explicitly allowed\n%s", name, ip, + strings.Join(g.path(name), "\n")) + } + } + return nil +} diff --git a/internal/editor/editor.go b/internal/editor/editor.go new file mode 100644 index 0000000..6a70fa0 --- /dev/null +++ b/internal/editor/editor.go @@ -0,0 +1,64 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package editor implements a simple interface for interactive file editing. +// It most likely does not work on windows. +package editor + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" +) + +// Edit opens a temporary file in the default editor (per $EDITOR, falling back +// to "vi") with the contents of the given io.Reader and a filename ending in +// the given extension (to give a hint to the editor for syntax highlighting). +// +// The contents of the edited file are returned, and the temporary file removed. +func Edit(input io.Reader, extension string) ([]byte, error) { + f, err := os.CreateTemp("", fmt.Sprintf("%s-edit.*.%s", filepath.Base(os.Args[0]), extension)) + if err != nil { + return nil, err + } + defer os.Remove(f.Name()) + + if _, err := io.Copy(f, input); err != nil { + return nil, err + } + f.Close() + + editor := "vi" + if env := os.Getenv("EDITOR"); env != "" { + editor = env + } + + path, err := exec.LookPath(editor) + if err != nil { + return nil, err + } + + cmd := exec.Command(path, f.Name()) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return nil, err + } + + return os.ReadFile(f.Name()) +} diff --git a/internal/estargz/estargz.go b/internal/estargz/estargz.go new file mode 100644 index 0000000..69021bc --- /dev/null +++ b/internal/estargz/estargz.go @@ -0,0 +1,54 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package estargz adapts the containerd estargz package to our abstractions. +package estargz + +import ( + "bytes" + "io" + + "github.com/containerd/stargz-snapshotter/estargz" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Assert that what we're returning is an io.ReadCloser +var _ io.ReadCloser = (*estargz.Blob)(nil) + +// ReadCloser reads uncompressed tarball input from the io.ReadCloser and +// returns: +// - An io.ReadCloser from which compressed data may be read, and +// - A v1.Hash with the hash of the estargz table of contents, or +// - An error if the estargz processing encountered a problem. +// +// Refer to estargz for the options: +// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option +func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) { + defer r.Close() + + // TODO(#876): Avoid buffering into memory. + bs, err := io.ReadAll(r) + if err != nil { + return nil, v1.Hash{}, err + } + br := bytes.NewReader(bs) + + rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...) + if err != nil { + return nil, v1.Hash{}, err + } + + h, err := v1.NewHash(rc.TOCDigest().String()) + return rc, h, err +} diff --git a/internal/estargz/estargz_test.go b/internal/estargz/estargz_test.go new file mode 100644 index 0000000..1eb3114 --- /dev/null +++ b/internal/estargz/estargz_test.go @@ -0,0 +1,108 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package estargz + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "strings" + "testing" + + "github.com/google/go-containerregistry/internal/gzip" +) + +func TestReader(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBuffer(nil) + tw := tar.NewWriter(buf) + + if err := tw.WriteHeader(&tar.Header{ + Name: "foo", + Size: int64(len(want)), + }); err != nil { + t.Fatal("WriteHeader() =", err) + } + if _, err := tw.Write([]byte(want)); err != nil { + t.Fatal("tw.Write() =", err) + } + tw.Close() + + zipped, _, err := ReadCloser(io.NopCloser(buf)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + unzipped, err := gzip.UnzipReadCloser(zipped) + if err != nil { + t.Error("gzip.UnzipReadCloser() =", err) + } + defer unzipped.Close() + + found := false + + r := tar.NewReader(unzipped) + for { + hdr, err := r.Next() + if err == io.EOF { + break + } else if err != nil { + t.Fatal("tar.Next() =", err) + } + + if hdr.Name != "foo" { + continue + } + found = true + + b, err := io.ReadAll(r) + if err != nil { + t.Error("ReadAll() =", err) + } + if got := string(b); got != want { + t.Errorf("ReadAll(); got %q, want %q", got, want) + } + if err := unzipped.Close(); err != nil { + t.Error("Close() =", err) + } + } + + if !found { + t.Error(`Did not find the expected file "foo"`) + } +} + +var ( + errRead = fmt.Errorf("Read failed") +) + +type failReader struct{} + +func (f failReader) Read(_ []byte) (int, error) { + return 0, errRead +} + +func TestReadErrors(t *testing.T) { + fr := failReader{} + + if _, _, err := ReadCloser(io.NopCloser(fr)); err != errRead { + t.Error("ReadCloser: expected errRead, got", err) + } + + buf := bytes.NewBufferString("not a tarball") + if _, _, err := ReadCloser(io.NopCloser(buf)); !strings.Contains(err.Error(), "failed to parse tar file") { + t.Error(`ReadCloser: expected "failed to parse tar file", got`, err) + } +} diff --git a/internal/gzip/zip.go b/internal/gzip/zip.go new file mode 100644 index 0000000..018c0f8 --- /dev/null +++ b/internal/gzip/zip.go @@ -0,0 +1,118 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gzip provides helper functions for interacting with gzipped streams. +package gzip + +import ( + "bufio" + "bytes" + "compress/gzip" + "io" + + "github.com/google/go-containerregistry/internal/and" +) + +// MagicHeader is the start of gzip files. +var MagicHeader = []byte{'\x1f', '\x8b'} + +// ReadCloser reads uncompressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which compressed data may be read. +// This uses gzip.BestSpeed for the compression level. +func ReadCloser(r io.ReadCloser) io.ReadCloser { + return ReadCloserLevel(r, gzip.BestSpeed) +} + +// ReadCloserLevel reads uncompressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which compressed data may be read. +// Refer to compress/gzip for the level: +// https://golang.org/pkg/compress/gzip/#pkg-constants +func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser { + pr, pw := io.Pipe() + + // For highly compressible layers, gzip.Writer will output a very small + // number of bytes per Write(). This is normally fine, but when pushing + // to a registry, we want to ensure that we're taking full advantage of + // the available bandwidth instead of sending tons of tiny writes over + // the wire. + // 64K ought to be small enough for anybody. + bw := bufio.NewWriterSize(pw, 2<<16) + + // Returns err so we can pw.CloseWithError(err) + go func() error { + // TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect. + // Context: https://golang.org/issue/24283 + gw, err := gzip.NewWriterLevel(bw, level) + if err != nil { + return pw.CloseWithError(err) + } + + if _, err := io.Copy(gw, r); err != nil { + defer r.Close() + defer gw.Close() + return pw.CloseWithError(err) + } + + // Close gzip writer to Flush it and write gzip trailers. + if err := gw.Close(); err != nil { + return pw.CloseWithError(err) + } + + // Flush bufio writer to ensure we write out everything. + if err := bw.Flush(); err != nil { + return pw.CloseWithError(err) + } + + // We don't really care if these fail. + defer pw.Close() + defer r.Close() + + return nil + }() + + return pr +} + +// UnzipReadCloser reads compressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which uncompressed data may be read. +func UnzipReadCloser(r io.ReadCloser) (io.ReadCloser, error) { + gr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + return &and.ReadCloser{ + Reader: gr, + CloseFunc: func() error { + // If the unzip fails, then this seems to return the same + // error as the read. We don't want this to interfere with + // us closing the main ReadCloser, since this could leave + // an open file descriptor (fails on Windows). + gr.Close() + return r.Close() + }, + }, nil +} + +// Is detects whether the input stream is compressed. +func Is(r io.Reader) (bool, error) { + magicHeader := make([]byte, 2) + n, err := r.Read(magicHeader) + if n == 0 && err == io.EOF { + return false, nil + } + if err != nil { + return false, err + } + return bytes.Equal(magicHeader, MagicHeader), nil +} diff --git a/internal/gzip/zip_test.go b/internal/gzip/zip_test.go new file mode 100644 index 0000000..d8c27f6 --- /dev/null +++ b/internal/gzip/zip_test.go @@ -0,0 +1,98 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gzip + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" +) + +func TestReader(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + zipped := ReadCloser(io.NopCloser(buf)) + unzipped, err := UnzipReadCloser(zipped) + if err != nil { + t.Error("UnzipReadCloser() =", err) + } + + b, err := io.ReadAll(unzipped) + if err != nil { + t.Error("ReadAll() =", err) + } + if got := string(b); got != want { + t.Errorf("ReadAll(); got %q, want %q", got, want) + } + if err := unzipped.Close(); err != nil { + t.Error("Close() =", err) + } +} + +func TestIs(t *testing.T) { + tests := []struct { + in []byte + out bool + err error + }{ + {[]byte{}, false, nil}, + {[]byte{'\x00', '\x00', '\x00'}, false, nil}, + {[]byte{'\x1f', '\x8b', '\x1b'}, true, nil}, + } + for _, test := range tests { + reader := bytes.NewReader(test.in) + got, err := Is(reader) + if got != test.out { + t.Errorf("Is; n: got %v, wanted %v\n", got, test.out) + } + if err != test.err { + t.Errorf("Is; err: got %v, wanted %v\n", err, test.err) + } + } +} + +var ( + errRead = fmt.Errorf("Read failed") +) + +type failReader struct{} + +func (f failReader) Read(_ []byte) (int, error) { + return 0, errRead +} + +func TestReadErrors(t *testing.T) { + fr := failReader{} + if _, err := Is(fr); err != errRead { + t.Error("Is: expected errRead, got", err) + } + + frc := io.NopCloser(fr) + if _, err := UnzipReadCloser(frc); err != errRead { + t.Error("UnzipReadCloser: expected errRead, got", err) + } + + zr := ReadCloser(io.NopCloser(fr)) + if _, err := zr.Read(nil); err != errRead { + t.Error("ReadCloser: expected errRead, got", err) + } + + zr = ReadCloserLevel(io.NopCloser(strings.NewReader("zip me")), -10) + if _, err := zr.Read(nil); err == nil { + t.Error("Expected invalid level error, got:", err) + } +} diff --git a/internal/httptest/httptest.go b/internal/httptest/httptest.go new file mode 100644 index 0000000..85b1719 --- /dev/null +++ b/internal/httptest/httptest.go @@ -0,0 +1,104 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httptest provides a method for testing a TLS server a la net/http/httptest. +package httptest + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "time" +) + +// NewTLSServer returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain. +// If you need a transport, Client().Transport is correctly configured. +func NewTLSServer(domain string, handler http.Handler) (*httptest.Server, error) { + s := httptest.NewUnstartedServer(handler) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.IPv6loopback, + }, + DNSNames: []string{domain}, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + return nil, err + } + + b, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + pc := &bytes.Buffer{} + if err := pem.Encode(pc, &pem.Block{Type: "CERTIFICATE", Bytes: b}); err != nil { + return nil, err + } + + ek, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return nil, err + } + + pk := &bytes.Buffer{} + if err := pem.Encode(pk, &pem.Block{Type: "EC PRIVATE KEY", Bytes: ek}); err != nil { + return nil, err + } + + c, err := tls.X509KeyPair(pc.Bytes(), pk.Bytes()) + if err != nil { + return nil, err + } + s.TLS = &tls.Config{ + Certificates: []tls.Certificate{c}, + } + s.StartTLS() + + certpool := x509.NewCertPool() + certpool.AddCert(s.Certificate()) + + t := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial(s.Listener.Addr().Network(), s.Listener.Addr().String()) + }, + } + s.Client().Transport = t + + return s, nil +} diff --git a/internal/legacy/copy.go b/internal/legacy/copy.go new file mode 100644 index 0000000..10467ba --- /dev/null +++ b/internal/legacy/copy.go @@ -0,0 +1,57 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package legacy provides methods for interacting with legacy image formats. +package legacy + +import ( + "bytes" + "encoding/json" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// CopySchema1 allows `[g]crane cp` to work with old images without adding +// full support for schema 1 images to this package. +func CopySchema1(desc *remote.Descriptor, srcRef, dstRef name.Reference, opts ...remote.Option) error { + m := schema1{} + if err := json.NewDecoder(bytes.NewReader(desc.Manifest)).Decode(&m); err != nil { + return err + } + + for _, layer := range m.FSLayers { + src := srcRef.Context().Digest(layer.BlobSum) + dst := dstRef.Context().Digest(layer.BlobSum) + + blob, err := remote.Layer(src, opts...) + if err != nil { + return err + } + + if err := remote.WriteLayer(dst.Context(), blob, opts...); err != nil { + return err + } + } + + return remote.Put(dstRef, desc, opts...) +} + +type fslayer struct { + BlobSum string `json:"blobSum"` +} + +type schema1 struct { + FSLayers []fslayer `json:"fsLayers"` +} diff --git a/internal/legacy/copy_test.go b/internal/legacy/copy_test.go new file mode 100644 index 0000000..b8ca799 --- /dev/null +++ b/internal/legacy/copy_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package legacy + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestCopySchema1(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + // We'll copy from src to dst. + src := fmt.Sprintf("%s/schema1/src", u.Host) + srcRef, err := name.ParseReference(src) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/schema1/dst", u.Host) + dstRef, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + // Create a random layer. + layer, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + digest, err := layer.Digest() + if err != nil { + t.Fatal(err) + } + layerRef, err := name.NewDigest(fmt.Sprintf("%s@%s", src, digest)) + if err != nil { + t.Fatal(err) + } + + // Populate the registry with a layer and a schema 1 manifest referencing it. + if err := remote.WriteLayer(layerRef.Context(), layer); err != nil { + t.Fatal(err) + } + manifest := schema1{ + FSLayers: []fslayer{{ + BlobSum: digest.String(), + }}, + } + b, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + desc := &remote.Descriptor{ + Manifest: b, + Descriptor: v1.Descriptor{ + MediaType: types.DockerManifestSchema1, + Digest: v1.Hash{Algorithm: "sha256", + Hex: strings.Repeat("a", 64), + }, + }, + } + if err := remote.Put(dstRef, desc); err != nil { + t.Fatal(err) + } + + if err := CopySchema1(desc, srcRef, dstRef); err != nil { + t.Errorf("failed to copy schema 1: %v", err) + } +} diff --git a/internal/redact/redact.go b/internal/redact/redact.go new file mode 100644 index 0000000..b2e3f18 --- /dev/null +++ b/internal/redact/redact.go @@ -0,0 +1,89 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redact contains a simple context signal for redacting requests. +package redact + +import ( + "context" + "errors" + "net/url" +) + +type contextKey string + +var redactKey = contextKey("redact") + +// NewContext creates a new ctx with the reason for redaction. +func NewContext(ctx context.Context, reason string) context.Context { + return context.WithValue(ctx, redactKey, reason) +} + +// FromContext returns the redaction reason, if any. +func FromContext(ctx context.Context) (bool, string) { + reason, ok := ctx.Value(redactKey).(string) + return ok, reason +} + +// Error redacts potentially sensitive query parameter values in the URL from the error's message. +// +// If the error is a *url.Error, this returns a *url.Error with the URL redacted. +// Any other error type, or nil, is returned unchanged. +func Error(err error) error { + // If the error is a url.Error, we can redact the URL. + // Otherwise (including if err is nil), we can't redact. + var uerr *url.Error + if ok := errors.As(err, &uerr); !ok { + return err + } + u, perr := url.Parse(uerr.URL) + if perr != nil { + return err // If the URL can't be parsed, just return the original error. + } + uerr.URL = URL(u).String() // Update the URL to the redacted URL. + return uerr +} + +// The set of query string keys that we expect to send as part of the registry +// protocol. Anything else is potentially dangerous to leak, as it's probably +// from a redirect. These redirects often included tokens or signed URLs. +var paramAllowlist = map[string]struct{}{ + // Token exchange + "scope": {}, + "service": {}, + // Cross-repo mounting + "mount": {}, + "from": {}, + // Layer PUT + "digest": {}, + // Listing tags and catalog + "n": {}, + "last": {}, +} + +// URL redacts potentially sensitive query parameter values from the URL's query string. +func URL(u *url.URL) *url.URL { + qs := u.Query() + for k, v := range qs { + for i := range v { + if _, ok := paramAllowlist[k]; !ok { + // key is not in the Allowlist + v[i] = "REDACTED" + } + } + } + r := *u + r.RawQuery = qs.Encode() + return &r +} diff --git a/internal/retry/retry.go b/internal/retry/retry.go new file mode 100644 index 0000000..c9e3564 --- /dev/null +++ b/internal/retry/retry.go @@ -0,0 +1,94 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package retry provides methods for retrying operations. It is a thin wrapper +// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier. +package retry + +import ( + "context" + "errors" + "fmt" + + "github.com/google/go-containerregistry/internal/retry/wait" +) + +// Backoff is an alias of our own wait.Backoff to avoid name conflicts with +// the kubernetes wait package. Typing retry.Backoff is aesier than fixing +// the wrong import every time you use wait.Backoff. +type Backoff = wait.Backoff + +// This is implemented by several errors in the net package as well as our +// transport.Error. +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if err implements Temporary() and it returns true. +func IsTemporary(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return false + } + if te, ok := err.(temporary); ok && te.Temporary() { + return true + } + return false +} + +// IsNotNil returns true if err is not nil. +func IsNotNil(err error) bool { + return err != nil +} + +// Predicate determines whether an error should be retried. +type Predicate func(error) (retry bool) + +// Retry retries a given function, f, until a predicate is satisfied, using +// exponential backoff. If the predicate is never satisfied, it will return the +// last error returned by f. +func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) { + if f == nil { + return fmt.Errorf("nil f passed to retry") + } + if p == nil { + return fmt.Errorf("nil p passed to retry") + } + + condition := func() (bool, error) { + err = f() + if p(err) { + return false, nil + } + return true, err + } + + wait.ExponentialBackoff(backoff, condition) + return +} + +type contextKey string + +var key = contextKey("never") + +// Never returns a context that signals something should not be retried. +// This is a hack and can be used to communicate across package boundaries +// to avoid retry amplification. +func Never(ctx context.Context) context.Context { + return context.WithValue(ctx, key, true) +} + +// Ever returns true if the context was wrapped by Never. +func Ever(ctx context.Context) bool { + return ctx.Value(key) == nil +} diff --git a/internal/retry/retry_test.go b/internal/retry/retry_test.go new file mode 100644 index 0000000..2091ca5 --- /dev/null +++ b/internal/retry/retry_test.go @@ -0,0 +1,100 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retry + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" +) + +type temp struct{} + +func (e temp) Error() string { + return "temporary error" +} + +func (e temp) Temporary() bool { + return true +} + +func TestRetry(t *testing.T) { + for i, test := range []struct { + predicate Predicate + err error + shouldRetry bool + }{{ + predicate: IsTemporary, + err: nil, + shouldRetry: false, + }, { + predicate: IsTemporary, + err: fmt.Errorf("not temporary"), + shouldRetry: false, + }, { + predicate: IsNotNil, + err: fmt.Errorf("not temporary"), + shouldRetry: true, + }, { + predicate: IsTemporary, + err: temp{}, + shouldRetry: true, + }, { + predicate: IsTemporary, + err: context.DeadlineExceeded, + shouldRetry: false, + }, { + predicate: IsTemporary, + err: &url.Error{ + Op: http.MethodPost, + URL: "http://127.0.0.1:56520/v2/example/blobs/uploads/", + Err: context.DeadlineExceeded, + }, + shouldRetry: false, + }} { + // Make sure we retry 5 times if we shouldRetry. + steps := 5 + backoff := Backoff{ + Steps: steps, + } + + // Count how many times this function is invoked. + count := 0 + f := func() error { + count++ + return test.err + } + + Retry(f, test.predicate, backoff) + + if test.shouldRetry && count != steps { + t.Errorf("expected %d to retry %v, did not", i, test.err) + } else if !test.shouldRetry && count == steps { + t.Errorf("expected %d not to retry %v, but did", i, test.err) + } + } +} + +// Make sure we don't panic. +func TestNil(t *testing.T) { + if err := Retry(nil, nil, Backoff{}); err == nil { + t.Errorf("got nil when passing in nil f") + } + if err := Retry(func() error { return nil }, nil, Backoff{}); err == nil { + t.Errorf("got nil when passing in nil p") + } +} diff --git a/internal/retry/wait/kubernetes_apimachinery_wait.go b/internal/retry/wait/kubernetes_apimachinery_wait.go new file mode 100644 index 0000000..ab06e5f --- /dev/null +++ b/internal/retry/wait/kubernetes_apimachinery_wait.go @@ -0,0 +1,123 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package wait is a subset of k8s.io/apimachinery to avoid conflicts +// in dependencies (specifically, logging). +package wait + +import ( + "errors" + "math/rand" + "time" +) + +// Jitter returns a time.Duration between duration and duration + maxFactor * +// duration. +// +// This allows clients to avoid converging on periodic behavior. If maxFactor +// is 0.0, a suggested default value will be chosen. +func Jitter(duration time.Duration, maxFactor float64) time.Duration { + if maxFactor <= 0.0 { + maxFactor = 1.0 + } + wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration)) + return wait +} + +// ErrWaitTimeout is returned when the condition exited without success. +var ErrWaitTimeout = errors.New("timed out waiting for the condition") + +// ConditionFunc returns true if the condition is satisfied, or an error +// if the loop should be aborted. +type ConditionFunc func() (done bool, err error) + +// Backoff holds parameters applied to a Backoff function. +type Backoff struct { + // The initial duration. + Duration time.Duration + // Duration is multiplied by factor each iteration, if factor is not zero + // and the limits imposed by Steps and Cap have not been reached. + // Should not be negative. + // The jitter does not contribute to the updates to the duration parameter. + Factor float64 + // The sleep at each iteration is the duration plus an additional + // amount chosen uniformly at random from the interval between + // zero and `jitter*duration`. + Jitter float64 + // The remaining number of iterations in which the duration + // parameter may change (but progress can be stopped earlier by + // hitting the cap). If not positive, the duration is not + // changed. Used for exponential backoff in combination with + // Factor and Cap. + Steps int + // A limit on revised values of the duration parameter. If a + // multiplication by the factor parameter would make the duration + // exceed the cap then the duration is set to the cap and the + // steps parameter is set to zero. + Cap time.Duration +} + +// Step (1) returns an amount of time to sleep determined by the +// original Duration and Jitter and (2) mutates the provided Backoff +// to update its Steps and Duration. +func (b *Backoff) Step() time.Duration { + if b.Steps < 1 { + if b.Jitter > 0 { + return Jitter(b.Duration, b.Jitter) + } + return b.Duration + } + b.Steps-- + + duration := b.Duration + + // calculate the next step + if b.Factor != 0 { + b.Duration = time.Duration(float64(b.Duration) * b.Factor) + if b.Cap > 0 && b.Duration > b.Cap { + b.Duration = b.Cap + b.Steps = 0 + } + } + + if b.Jitter > 0 { + duration = Jitter(duration, b.Jitter) + } + return duration +} + +// ExponentialBackoff repeats a condition check with exponential backoff. +// +// It repeatedly checks the condition and then sleeps, using `backoff.Step()` +// to determine the length of the sleep and adjust Duration and Steps. +// Stops and returns as soon as: +// 1. the condition check returns true or an error, +// 2. `backoff.Steps` checks of the condition have been done, or +// 3. a sleep truncated by the cap on duration has been completed. +// In case (1) the returned error is what the condition function returned. +// In all other cases, ErrWaitTimeout is returned. +func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error { + for backoff.Steps > 0 { + if ok, err := condition(); err != nil || ok { + return err + } + if backoff.Steps == 1 { + break + } + time.Sleep(backoff.Step()) + } + return ErrWaitTimeout +} diff --git a/internal/verify/verify.go b/internal/verify/verify.go new file mode 100644 index 0000000..463f7e4 --- /dev/null +++ b/internal/verify/verify.go @@ -0,0 +1,122 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package verify provides a ReadCloser that verifies content matches the +// expected hash values. +package verify + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + + "github.com/google/go-containerregistry/internal/and" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// SizeUnknown is a sentinel value to indicate that the expected size is not known. +const SizeUnknown = -1 + +type verifyReader struct { + inner io.Reader + hasher hash.Hash + expected v1.Hash + gotSize, wantSize int64 +} + +// Error provides information about the failed hash verification. +type Error struct { + got string + want v1.Hash + gotSize int64 +} + +func (v Error) Error() string { + return fmt.Sprintf("error verifying %s checksum after reading %d bytes; got %q, want %q", + v.want.Algorithm, v.gotSize, v.got, v.want) +} + +// Read implements io.Reader +func (vc *verifyReader) Read(b []byte) (int, error) { + n, err := vc.inner.Read(b) + vc.gotSize += int64(n) + if err == io.EOF { + if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize { + return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize) + } + got := hex.EncodeToString(vc.hasher.Sum(nil)) + if want := vc.expected.Hex; got != want { + return n, Error{ + got: vc.expected.Algorithm + ":" + got, + want: vc.expected, + gotSize: vc.gotSize, + } + } + } + return n, err +} + +// ReadCloser wraps the given io.ReadCloser to verify that its contents match +// the provided v1.Hash before io.EOF is returned. +// +// The reader will only be read up to size bytes, to prevent resource +// exhaustion. If EOF is returned before size bytes are read, an error is +// returned. +// +// A size of SizeUnknown (-1) indicates disables size verification when the size +// is unknown ahead of time. +func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) { + w, err := v1.Hasher(h.Algorithm) + if err != nil { + return nil, err + } + r2 := io.TeeReader(r, w) // pass all writes to the hasher. + if size != SizeUnknown { + r2 = io.LimitReader(r2, size) // if we know the size, limit to that size. + } + return &and.ReadCloser{ + Reader: &verifyReader{ + inner: r2, + hasher: w, + expected: h, + wantSize: size, + }, + CloseFunc: r.Close, + }, nil +} + +// Descriptor verifies that the embedded Data field matches the Size and Digest +// fields of the given v1.Descriptor, returning an error if the Data field is +// missing or if it contains incorrect data. +func Descriptor(d v1.Descriptor) error { + if d.Data == nil { + return errors.New("error verifying descriptor; Data == nil") + } + + h, sz, err := v1.SHA256(bytes.NewReader(d.Data)) + if err != nil { + return err + } + if h != d.Digest { + return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest) + } + if sz != d.Size { + return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size) + } + + return nil +} diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go new file mode 100644 index 0000000..a2cbbdc --- /dev/null +++ b/internal/verify/verify_test.go @@ -0,0 +1,147 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package verify + +import ( + "bytes" + "errors" + "fmt" + "io" + "strings" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func mustHash(s string, t *testing.T) v1.Hash { + h, _, err := v1.SHA256(strings.NewReader(s)) + if err != nil { + t.Fatalf("v1.SHA256(%s) = %v", s, err) + } + t.Logf("Hashed: %q -> %q", s, h) + return h +} + +func TestVerificationFailure(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + + verified, err := ReadCloser(io.NopCloser(buf), int64(len(want)), mustHash("not the same", t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if b, err := io.ReadAll(verified); err == nil { + t.Errorf("ReadAll() = %q; want verification error", string(b)) + } +} + +func TestVerification(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + + verified, err := ReadCloser(io.NopCloser(buf), int64(len(want)), mustHash(want, t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if _, err := io.ReadAll(verified); err != nil { + t.Error("ReadAll() =", err) + } +} + +func TestVerificationSizeUnknown(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + + verified, err := ReadCloser(io.NopCloser(buf), SizeUnknown, mustHash(want, t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if _, err := io.ReadAll(verified); err != nil { + t.Error("ReadAll() =", err) + } +} + +func TestBadHash(t *testing.T) { + h := v1.Hash{ + Algorithm: "fake256", + Hex: "whatever", + } + _, err := ReadCloser(io.NopCloser(strings.NewReader("hi")), 0, h) + if err == nil { + t.Errorf("ReadCloser() = %v, wanted err", err) + } +} + +func TestBadSize(t *testing.T) { + want := "This is the input string." + + // having too much content or expecting too much content returns an error. + for _, size := range []int64{3, 100} { + t.Run(fmt.Sprintf("expecting size %d", size), func(t *testing.T) { + buf := bytes.NewBufferString(want) + rc, err := ReadCloser(io.NopCloser(buf), size, mustHash(want, t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if b, err := io.ReadAll(rc); err == nil { + t.Errorf("ReadAll() = %q; want verification error", string(b)) + } + }) + } +} + +func TestDescriptor(t *testing.T) { + for _, tc := range []struct { + err error + desc v1.Descriptor + }{{ + err: errors.New("error verifying descriptor; Data == nil"), + }, { + err: errors.New(`error verifying Digest; got "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", want ":"`), + desc: v1.Descriptor{ + Data: []byte("abc"), + }, + }, { + err: errors.New("error verifying Size; got 3, want 0"), + desc: v1.Descriptor{ + Data: []byte("abc"), + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + }, + }, { + desc: v1.Descriptor{ + Data: []byte("abc"), + Size: 3, + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + }, + }} { + got, want := Descriptor(tc.desc), tc.err + + if got == nil { + if want != nil { + t.Errorf("Descriptor(): got nil, want %v", want) + } + } else if want == nil { + t.Errorf("Descriptor(): got %v, want nil", got) + } else if got, want := got.Error(), want.Error(); got != want { + t.Errorf("Descriptor(): got %q, want %q", got, want) + } + } +} diff --git a/internal/windows/windows.go b/internal/windows/windows.go new file mode 100644 index 0000000..62d04cf --- /dev/null +++ b/internal/windows/windows.go @@ -0,0 +1,114 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "path" + "strings" + + "github.com/google/go-containerregistry/internal/gzip" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// userOwnerAndGroupSID is a magic value needed to make the binary executable +// in a Windows container. +// +// owner: BUILTIN/Users group: BUILTIN/Users ($sddlValue="O:BUG:BU") +const userOwnerAndGroupSID = "AQAAgBQAAAAkAAAAAAAAAAAAAAABAgAAAAAABSAAAAAhAgAAAQIAAAAAAAUgAAAAIQIAAA==" + +// Windows returns a Layer that is converted to be pullable on Windows. +func Windows(layer v1.Layer) (v1.Layer, error) { + // TODO: do this lazily. + + layerReader, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("getting layer: %w", err) + } + defer layerReader.Close() + tarReader := tar.NewReader(layerReader) + w := new(bytes.Buffer) + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + for _, dir := range []string{"Files", "Hives"} { + if err := tarWriter.WriteHeader(&tar.Header{ + Name: dir, + Typeflag: tar.TypeDir, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + Format: tar.FormatPAX, + }); err != nil { + return nil, fmt.Errorf("writing %s directory: %w", dir, err) + } + } + + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("reading layer: %w", err) + } + + if strings.HasPrefix(header.Name, "Files/") { + return nil, fmt.Errorf("file path %q already suitable for Windows", header.Name) + } + + header.Name = path.Join("Files", header.Name) + header.Format = tar.FormatPAX + + // TODO: this seems to make the file executable on Windows; + // only do this if the file should be executable. + if header.PAXRecords == nil { + header.PAXRecords = map[string]string{} + } + header.PAXRecords["MSWINDOWS.rawsd"] = userOwnerAndGroupSID + + if err := tarWriter.WriteHeader(header); err != nil { + return nil, fmt.Errorf("writing tar header: %w", err) + } + + if header.Typeflag == tar.TypeReg { + if _, err = io.Copy(tarWriter, tarReader); err != nil { + return nil, fmt.Errorf("writing layer file: %w", err) + } + } + } + + if err := tarWriter.Close(); err != nil { + return nil, err + } + + b := w.Bytes() + // gzip the contents, then create the layer + opener := func() (io.ReadCloser, error) { + return gzip.ReadCloser(io.NopCloser(bytes.NewReader(b))), nil + } + layer, err = tarball.LayerFromOpener(opener) + if err != nil { + return nil, fmt.Errorf("creating layer: %w", err) + } + + return layer, nil +} diff --git a/internal/windows/windows_test.go b/internal/windows/windows_test.go new file mode 100644 index 0000000..3caf54c --- /dev/null +++ b/internal/windows/windows_test.go @@ -0,0 +1,81 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows + +import ( + "archive/tar" + "errors" + "io" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func TestWindows(t *testing.T) { + tarLayer, err := tarball.LayerFromFile("../../pkg/v1/tarball/testdata/content.tar") + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + win, err := Windows(tarLayer) + if err != nil { + t.Fatalf("Windows: %v", err) + } + if _, err := Windows(win); err == nil { + t.Error("expected an error double-Windowsifying a layer; got nil") + } + + rc, err := win.Uncompressed() + if err != nil { + t.Fatalf("Uncompressed: %v", err) + } + defer rc.Close() + tr := tar.NewReader(rc) + var sawHives, sawFiles bool + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if h.Name == "Hives" && h.Typeflag == tar.TypeDir { + sawHives = true + continue + } + if h.Name == "Files" && h.Typeflag == tar.TypeDir { + sawFiles = true + continue + } + if !strings.HasPrefix(h.Name, "Files/") { + t.Errorf("tar entry %q didn't have Files prefix", h.Name) + } + if h.Format != tar.FormatPAX { + t.Errorf("tar entry %q had unexpected Format; got %v, want %v", h.Name, h.Format, tar.FormatPAX) + } + want := map[string]string{ + "MSWINDOWS.rawsd": userOwnerAndGroupSID, + } + if !reflect.DeepEqual(h.PAXRecords, want) { + t.Errorf("tar entry %q: got %v, want %v", h.Name, h.PAXRecords, want) + } + } + if !sawHives { + t.Errorf("didn't see Hives/ directory") + } + if !sawFiles { + t.Errorf("didn't see Files/ directory") + } +} diff --git a/internal/zstd/zstd.go b/internal/zstd/zstd.go new file mode 100644 index 0000000..cccf54a --- /dev/null +++ b/internal/zstd/zstd.go @@ -0,0 +1,116 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package zstd provides helper functions for interacting with zstd streams. +package zstd + +import ( + "bufio" + "bytes" + "io" + + "github.com/google/go-containerregistry/internal/and" + "github.com/klauspost/compress/zstd" +) + +// MagicHeader is the start of zstd files. +var MagicHeader = []byte{'\x28', '\xb5', '\x2f', '\xfd'} + +// ReadCloser reads uncompressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which compressed data may be read. +// This uses zstd level 1 for the compression. +func ReadCloser(r io.ReadCloser) io.ReadCloser { + return ReadCloserLevel(r, 1) +} + +// ReadCloserLevel reads uncompressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which compressed data may be read. +func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser { + pr, pw := io.Pipe() + + // For highly compressible layers, zstd.Writer will output a very small + // number of bytes per Write(). This is normally fine, but when pushing + // to a registry, we want to ensure that we're taking full advantage of + // the available bandwidth instead of sending tons of tiny writes over + // the wire. + // 64K ought to be small enough for anybody. + bw := bufio.NewWriterSize(pw, 2<<16) + + // Returns err so we can pw.CloseWithError(err) + go func() error { + // TODO(go1.14): Just defer {pw,zw,r}.Close like you'd expect. + // Context: https://golang.org/issue/24283 + zw, err := zstd.NewWriter(bw, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + return pw.CloseWithError(err) + } + + if _, err := io.Copy(zw, r); err != nil { + defer r.Close() + defer zw.Close() + return pw.CloseWithError(err) + } + + // Close zstd writer to Flush it and write zstd trailers. + if err := zw.Close(); err != nil { + return pw.CloseWithError(err) + } + + // Flush bufio writer to ensure we write out everything. + if err := bw.Flush(); err != nil { + return pw.CloseWithError(err) + } + + // We don't really care if these fail. + defer pw.Close() + defer r.Close() + + return nil + }() + + return pr +} + +// UnzipReadCloser reads compressed input data from the io.ReadCloser and +// returns an io.ReadCloser from which uncompressed data may be read. +func UnzipReadCloser(r io.ReadCloser) (io.ReadCloser, error) { + gr, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + return &and.ReadCloser{ + Reader: gr, + CloseFunc: func() error { + // If the unzip fails, then this seems to return the same + // error as the read. We don't want this to interfere with + // us closing the main ReadCloser, since this could leave + // an open file descriptor (fails on Windows). + gr.Close() + return r.Close() + }, + }, nil +} + +// Is detects whether the input stream is compressed. +func Is(r io.Reader) (bool, error) { + magicHeader := make([]byte, 4) + n, err := r.Read(magicHeader) + if n == 0 && err == io.EOF { + return false, nil + } + if err != nil { + return false, err + } + return bytes.Equal(magicHeader, MagicHeader), nil +} diff --git a/internal/zstd/zstd_test.go b/internal/zstd/zstd_test.go new file mode 100644 index 0000000..c422e27 --- /dev/null +++ b/internal/zstd/zstd_test.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zstd + +import ( + "bytes" + "fmt" + "io" + "testing" +) + +func TestReader(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + zipped := ReadCloser(io.NopCloser(buf)) + unzipped, err := UnzipReadCloser(zipped) + if err != nil { + t.Error("UnzipReadCloser() =", err) + } + + b, err := io.ReadAll(unzipped) + if err != nil { + t.Error("ReadAll() =", err) + } + if got := string(b); got != want { + t.Errorf("ReadAll(); got %q, want %q", got, want) + } + if err := unzipped.Close(); err != nil { + t.Error("Close() =", err) + } +} + +func TestIs(t *testing.T) { + tests := []struct { + in []byte + out bool + err error + }{ + {[]byte{}, false, nil}, + {[]byte{'\x00', '\x00', '\x00', '\x00', '\x00'}, false, nil}, + {[]byte{'\x28', '\xb5', '\x2f', '\xfd', '\x1b'}, true, nil}, + } + for _, test := range tests { + reader := bytes.NewReader(test.in) + got, err := Is(reader) + if got != test.out { + t.Errorf("Is; n: got %v, wanted %v\n", got, test.out) + } + if err != test.err { + t.Errorf("Is; err: got %v, wanted %v\n", err, test.err) + } + } +} + +var ( + errRead = fmt.Errorf("read failed") +) + +type failReader struct{} + +func (f failReader) Read(_ []byte) (int, error) { + return 0, errRead +} + +func TestReadErrors(t *testing.T) { + fr := failReader{} + if _, err := Is(fr); err != errRead { + t.Error("Is: expected errRead, got", err) + } + + frc := io.NopCloser(fr) + if r, err := UnzipReadCloser(frc); err != errRead { + data := make([]byte, 100) + _, err := r.Read(data) + if err != errRead { + t.Error("UnzipReadCloser: expected errRead, got", err) + } + } + + zr := ReadCloser(io.NopCloser(fr)) + if _, err := zr.Read(nil); err != errRead { + t.Error("ReadCloser: expected errRead, got", err) + } +} diff --git a/pkg/authn/README.md b/pkg/authn/README.md new file mode 100644 index 0000000..042bdde --- /dev/null +++ b/pkg/authn/README.md @@ -0,0 +1,322 @@ +# `authn` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/authn?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/authn) + +This README outlines how we acquire and use credentials when interacting with a registry. + +As much as possible, we attempt to emulate `docker`'s authentication behavior and configuration so that this library "just works" if you've already configured credentials that work with `docker`; however, when things don't work, a basic understanding of what's going on can help with debugging. + +The official documentation for how authentication with `docker` works is (reasonably) scattered across several different sites and GitHub repositories, so we've tried to summarize the relevant bits here. + +## tl;dr for consumers of this package + +By default, [`pkg/v1/remote`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote) uses [`Anonymous`](https://godoc.org/github.com/google/go-containerregistry/pkg/authn#Anonymous) credentials (i.e. _none_), which for most registries will only allow read access to public images. + +To use the credentials found in your Docker config file, you can use the [`DefaultKeychain`](https://godoc.org/github.com/google/go-containerregistry/pkg/authn#DefaultKeychain), e.g.: + +```go +package main + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + ref, err := name.ParseReference("registry.example.com/private/repo") + if err != nil { + panic(err) + } + + // Fetch the manifest using default credentials. + img, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + panic(err) + } + + // Prints the digest of registry.example.com/private/repo + fmt.Println(img.Digest) +} +``` + +The `DefaultKeychain` will use credentials as described in your Docker config file -- usually `~/.docker/config.json`, or `%USERPROFILE%\.docker\config.json` on Windows -- or the location described by the `DOCKER_CONFIG` environment variable, if set. + +If those are not found, `DefaultKeychain` will look for credentials configured using [Podman's expectation](https://docs.podman.io/en/latest/markdown/podman-login.1.html) that these are found in `${XDG_RUNTIME_DIR}/containers/auth.json`. + +[See below](#docker-config-auth) for more information about what is configured in this file. + +## Emulating Cloud Provider Credential Helpers + +[`pkg/v1/google.Keychain`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/v1/google#Keychain) provides a `Keychain` implementation that emulates [`docker-credential-gcr`](https://github.com/GoogleCloudPlatform/docker-credential-gcr) to find credentials in the environment. +See [`google.NewEnvAuthenticator`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/v1/google#NewEnvAuthenticator) and [`google.NewGcloudAuthenticator`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/v1/google#NewGcloudAuthenticator) for more information. + +To emulate other credential helpers without requiring them to be available as executables, [`NewKeychainFromHelper`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/authn#NewKeychainFromHelper) provides an adapter that takes a Go implementation satisfying a subset of the [`credentials.Helper`](https://pkg.go.dev/github.com/docker/docker-credential-helpers/credentials#Helper) interface, and makes it available as a `Keychain`. + +This means that you can emulate, for example, [Amazon ECR's `docker-credential-ecr-login` credential helper](https://github.com/awslabs/amazon-ecr-credential-helper) using the same implementation: + +```go +import ( + ecr "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" + "github.com/awslabs/amazon-ecr-credential-helper/ecr-login/api" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + // ... + ecrHelper := ecr.ECRHelper{ClientFactory: api.DefaultClientFactory{}} + img, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.NewKeychainFromHelper(ecrHelper))) + if err != nil { + panic(err) + } + // ... +} +``` + +Likewise, you can emulate [Azure's ACR `docker-credential-acr-env` credential helper](https://github.com/chrismellard/docker-credential-acr-env): + +```go +import ( + "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + // ... + acrHelper := credhelper.NewACRCredentialsHelper() + img, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.NewKeychainFromHelper(acrHelper))) + if err != nil { + panic(err) + } + // ... +} +``` + + + +## Using Multiple `Keychain`s + +[`NewMultiKeychain`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/authn#NewMultiKeychain) allows you to specify multiple `Keychain` implementations, which will be checked in order when credentials are needed. + +For example: + +```go +kc := authn.NewMultiKeychain( + authn.DefaultKeychain, + google.Keychain, + authn.NewKeychainFromHelper(ecr.ECRHelper{ClientFactory: api.DefaultClientFactory{}}), + authn.NewKeychainFromHelper(acr.ACRCredHelper{}), +) +``` + +This multi-keychain will: + +- first check for credentials found in the Docker config file, as describe above, then +- check for GCP credentials available in the environment, as described above, then +- check for ECR credentials by emulating the ECR credential helper, then +- check for ACR credentials by emulating the ACR credential helper. + +If any keychain implementation is able to provide credentials for the request, they will be used, and further keychain implementations will not be consulted. + +If no implementations are able to provide credentials, `Anonymous` credentials will be used. + +## Docker Config Auth + +What follows attempts to gather useful information about Docker's config.json and make it available in one place. + +If you have questions, please [file an issue](https://github.com/google/go-containerregistry/issues/new). + +### Plaintext + +The config file is where your credentials are stored when you invoke `docker login`, e.g. the contents may look something like this: + +```json +{ + "auths": { + "registry.example.com": { + "auth": "QXp1cmVEaWFtb25kOmh1bnRlcjI=" + } + } +} +``` + +The `auths` map has an entry per registry, and the `auth` field contains your username and password encoded as [HTTP 'Basic' Auth](https://tools.ietf.org/html/rfc7617). + +**NOTE**: This means that your credentials are stored _in plaintext_: + +```bash +$ echo "QXp1cmVEaWFtb25kOmh1bnRlcjI=" | base64 -d +AzureDiamond:hunter2 +``` + +For what it's worth, this config file is equivalent to: + +```json +{ + "auths": { + "registry.example.com": { + "username": "AzureDiamond", + "password": "hunter2" + } + } +} +``` + +... which is useful to know if e.g. your CI system provides you a registry username and password via environment variables and you want to populate this file manually without invoking `docker login`. + +### Helpers + +If you log in like this, `docker` will warn you that you should use a [credential helper](https://docs.docker.com/engine/reference/commandline/login/#credentials-store), and you should! + +To configure a global credential helper: +```json +{ + "credsStore": "osxkeychain" +} +``` + +To configure a per-registry credential helper: +```json +{ + "credHelpers": { + "gcr.io": "gcr" + } +} +``` + +We use [`github.com/docker/cli/cli/config.Load`](https://godoc.org/github.com/docker/cli/cli/config#Load) to parse the config file and invoke any necessary credential helpers. This handles the logic of taking a [`ConfigFile`](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/configfile/file.go#L25-L54) + registry domain and producing an [`AuthConfig`](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/types/authconfig.go#L3-L22), which determines how we authenticate to the registry. + +## Credential Helpers + +The [credential helper protocol](https://github.com/docker/docker-credential-helpers) allows you to configure a binary that supplies credentials for the registry, rather than hard-coding them in the config file. + +The protocol has several verbs, but the one we most care about is `get`. + +For example, using the following config file: +```json +{ + "credHelpers": { + "gcr.io": "gcr", + "eu.gcr.io": "gcr" + } +} +``` + +To acquire credentials for `gcr.io`, we look in the `credHelpers` map to find +the credential helper for `gcr.io` is `gcr`. By appending that value to +`docker-credential-`, we can get the name of the binary we need to use. + +For this example, that's `docker-credential-gcr`, which must be on our `$PATH`. +We'll then invoke that binary to get credentials: + +```bash +$ echo "gcr.io" | docker-credential-gcr get +{"Username":"_token","Secret":""} +``` + +You can configure the same credential helper for multiple registries, which is +why we need to pass the domain in via STDIN, e.g. if we were trying to access +`eu.gcr.io`, we'd do this instead: + +```bash +$ echo "eu.gcr.io" | docker-credential-gcr get +{"Username":"_token","Secret":""} +``` + +### Debugging credential helpers + +If a credential helper is configured but doesn't seem to be working, it can be +challenging to debug. Implementing a fake credential helper lets you poke around +to make it easier to see where the failure is happening. + +This "implements" a credential helper with hard-coded values: +``` +#!/usr/bin/env bash +echo '{"Username":"","Secret":"hunter2"}' +``` + + +This implements a credential helper that prints the output of +`docker-credential-gcr` to both stderr and whatever called it, which allows you +to snoop on another credential helper: +``` +#!/usr/bin/env bash +docker-credential-gcr $@ | tee >(cat 1>&2) +``` + +Put those files somewhere on your path, naming them e.g. +`docker-credential-hardcoded` and `docker-credential-tee`, then modify the +config file to use them: + +```json +{ + "credHelpers": { + "gcr.io": "tee", + "eu.gcr.io": "hardcoded" + } +} +``` + +The `docker-credential-tee` trick works with both `crane` and `docker`: + +```bash +$ crane manifest gcr.io/google-containers/pause > /dev/null +{"ServerURL":"","Username":"_dcgcr_1_5_0_token","Secret":""} + +$ docker pull gcr.io/google-containers/pause +Using default tag: latest +{"ServerURL":"","Username":"_dcgcr_1_5_0_token","Secret":""} +latest: Pulling from google-containers/pause +a3ed95caeb02: Pull complete +4964c72cd024: Pull complete +Digest: sha256:a78c2d6208eff9b672de43f880093100050983047b7b0afe0217d3656e1b0d5f +Status: Downloaded newer image for gcr.io/google-containers/pause:latest +gcr.io/google-containers/pause:latest +``` + +## The Registry + +There are two methods for authenticating against a registry: +[token](https://docs.docker.com/registry/spec/auth/token/) and +[oauth2](https://docs.docker.com/registry/spec/auth/oauth/). + +Both methods are used to acquire an opaque `Bearer` token (or +[RegistryToken](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/types/authconfig.go#L21)) +to use in the `Authorization` header. The registry will return a `401 +Unauthorized` during the [version +check](https://github.com/opencontainers/distribution-spec/blob/2c3975d1f03b67c9a0203199038adea0413f0573/spec.md#api-version-check) +(or during normal operations) with +[Www-Authenticate](https://tools.ietf.org/html/rfc7235#section-4.1) challenge +indicating how to proceed. + +### Token + +If we get back an `AuthConfig` containing a [`Username/Password`](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/types/authconfig.go#L5-L6) +or +[`Auth`](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/types/authconfig.go#L7), +we'll use the token method for authentication: + +![basic](../../images/credhelper-basic.svg) + +### OAuth 2 + +If we get back an `AuthConfig` containing an [`IdentityToken`](https://github.com/docker/cli/blob/ba63a92655c0bea4857b8d6cc4991498858b3c60/cli/config/types/authconfig.go#L18) +we'll use the oauth2 method for authentication: + +![oauth](../../images/credhelper-oauth.svg) + +This happens when a credential helper returns a response with the +[`Username`](https://github.com/docker/docker-credential-helpers/blob/f78081d1f7fef6ad74ad6b79368de6348386e591/credentials/credentials.go#L16) +set to `` (no, that's not a placeholder, the literal string `""`). +It is unclear why: [moby/moby#36926](https://github.com/moby/moby/issues/36926). + +We only support the oauth2 `grant_type` for `refresh_token` ([#629](https://github.com/google/go-containerregistry/issues/629)), +since it's impossible to determine from the registry response whether we should +use oauth, and the token method for authentication is widely implemented by +registries. diff --git a/pkg/authn/anon.go b/pkg/authn/anon.go new file mode 100644 index 0000000..8321495 --- /dev/null +++ b/pkg/authn/anon.go @@ -0,0 +1,26 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +// anonymous implements Authenticator for anonymous authentication. +type anonymous struct{} + +// Authorization implements Authenticator. +func (a *anonymous) Authorization() (*AuthConfig, error) { + return &AuthConfig{}, nil +} + +// Anonymous is a singleton Authenticator for providing anonymous auth. +var Anonymous Authenticator = &anonymous{} diff --git a/pkg/authn/anon_test.go b/pkg/authn/anon_test.go new file mode 100644 index 0000000..83c8214 --- /dev/null +++ b/pkg/authn/anon_test.go @@ -0,0 +1,31 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "reflect" + "testing" +) + +func TestAnonymous(t *testing.T) { + cfg, err := Anonymous.Authorization() + if err != nil { + t.Fatalf("Authorization() = %v", err) + } + want := &AuthConfig{} + if !reflect.DeepEqual(cfg, want) { + t.Errorf("Authorization(); got %v, wanted {}", cfg) + } +} diff --git a/pkg/authn/auth.go b/pkg/authn/auth.go new file mode 100644 index 0000000..0111f1a --- /dev/null +++ b/pkg/authn/auth.go @@ -0,0 +1,30 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +// auth is an Authenticator that simply returns the wrapped AuthConfig. +type auth struct { + config AuthConfig +} + +// FromConfig returns an Authenticator that just returns the given AuthConfig. +func FromConfig(cfg AuthConfig) Authenticator { + return &auth{cfg} +} + +// Authorization implements Authenticator. +func (a *auth) Authorization() (*AuthConfig, error) { + return &a.config, nil +} diff --git a/pkg/authn/authn.go b/pkg/authn/authn.go new file mode 100644 index 0000000..172d218 --- /dev/null +++ b/pkg/authn/authn.go @@ -0,0 +1,115 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" +) + +// Authenticator is used to authenticate Docker transports. +type Authenticator interface { + // Authorization returns the value to use in an http transport's Authorization header. + Authorization() (*AuthConfig, error) +} + +// AuthConfig contains authorization information for connecting to a Registry +// Inlined what we use from github.com/docker/cli/cli/config/types +type AuthConfig struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth,omitempty"` + + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `json:"identitytoken,omitempty"` + + // RegistryToken is a bearer token to be sent to a registry + RegistryToken string `json:"registrytoken,omitempty"` +} + +// This is effectively a copy of the type AuthConfig. This simplifies +// JSON unmarshalling since AuthConfig methods are not inherited +type authConfig AuthConfig + +// UnmarshalJSON implements json.Unmarshaler +func (a *AuthConfig) UnmarshalJSON(data []byte) error { + var shadow authConfig + err := json.Unmarshal(data, &shadow) + if err != nil { + return err + } + + *a = (AuthConfig)(shadow) + + if len(shadow.Auth) != 0 { + var derr error + a.Username, a.Password, derr = decodeDockerConfigFieldAuth(shadow.Auth) + if derr != nil { + err = fmt.Errorf("unable to decode auth field: %w", derr) + } + } else if len(a.Username) != 0 && len(a.Password) != 0 { + a.Auth = encodeDockerConfigFieldAuth(shadow.Username, shadow.Password) + } + + return err +} + +// MarshalJSON implements json.Marshaler +func (a AuthConfig) MarshalJSON() ([]byte, error) { + shadow := (authConfig)(a) + shadow.Auth = encodeDockerConfigFieldAuth(shadow.Username, shadow.Password) + return json.Marshal(shadow) +} + +// decodeDockerConfigFieldAuth deserializes the "auth" field from dockercfg into a +// username and a password. The format of the auth field is base64(:). +// +// From https://github.com/kubernetes/kubernetes/blob/75e49ec824b183288e1dbaccfd7dbe77d89db381/pkg/credentialprovider/config.go +// Copyright 2014 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 +func decodeDockerConfigFieldAuth(field string) (username, password string, err error) { + var decoded []byte + // StdEncoding can only decode padded string + // RawStdEncoding can only decode unpadded string + if strings.HasSuffix(strings.TrimSpace(field), "=") { + // decode padded data + decoded, err = base64.StdEncoding.DecodeString(field) + } else { + // decode unpadded data + decoded, err = base64.RawStdEncoding.DecodeString(field) + } + + if err != nil { + return + } + + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + err = fmt.Errorf("must be formatted as base64(username:password)") + return + } + + username = parts[0] + password = parts[1] + + return +} + +func encodeDockerConfigFieldAuth(username, password string) string { + return base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) +} diff --git a/pkg/authn/authn_test.go b/pkg/authn/authn_test.go new file mode 100644 index 0000000..f191acd --- /dev/null +++ b/pkg/authn/authn_test.go @@ -0,0 +1,148 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestAuthConfigMarshalJSON(t *testing.T) { + cases := []struct { + name string + config AuthConfig + json string + }{{ + name: "auth field is calculated", + config: AuthConfig{ + Username: "user", + Password: "pass", + IdentityToken: "id", + RegistryToken: "reg", + }, + json: `{"username":"user","password":"pass","auth":"dXNlcjpwYXNz","identitytoken":"id","registrytoken":"reg"}`, + }, { + name: "auth field replaced", + config: AuthConfig{ + Username: "user", + Password: "pass", + Auth: "blah", + IdentityToken: "id", + RegistryToken: "reg", + }, + json: `{"username":"user","password":"pass","auth":"dXNlcjpwYXNz","identitytoken":"id","registrytoken":"reg"}`, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bytes, err := json.Marshal(&tc.config) + + if err != nil { + t.Fatal("Marshal() =", err) + } + + if diff := cmp.Diff(tc.json, string(bytes)); diff != "" { + t.Error("json output diff (-want, +got): ", diff) + } + }) + } +} + +func TestAuthConfigUnmarshalJSON(t *testing.T) { + cases := []struct { + name string + json string + err string + want AuthConfig + }{{ + name: "valid config no auth", + json: `{ + "username": "user", + "password": "pass", + "identitytoken": "id", + "registrytoken": "reg" + }`, + want: AuthConfig{ + // Auth value is set based on username and password + Auth: "dXNlcjpwYXNz", + Username: "user", + Password: "pass", + IdentityToken: "id", + RegistryToken: "reg", + }, + }, { + name: "bad json input", + json: `{"username":true}`, + err: "json: cannot unmarshal", + }, { + name: "auth is base64", + json: `{ "auth": "dXNlcjpwYXNz" }`, // user:pass + want: AuthConfig{ + Username: "user", + Password: "pass", + Auth: "dXNlcjpwYXNz", + }, + }, { + name: "auth field overrides others", + json: `{ "auth": "dXNlcjpwYXNz", "username":"foo", "password":"bar" }`, // user:pass + want: AuthConfig{ + Username: "user", + Password: "pass", + Auth: "dXNlcjpwYXNz", + }, + }, { + name: "auth is base64 padded", + json: `{ "auth": "dXNlcjpwYXNzd29yZA==" }`, // user:password + want: AuthConfig{ + Username: "user", + Password: "password", + Auth: "dXNlcjpwYXNzd29yZA==", + }, + }, { + name: "auth is not base64", + json: `{ "auth": "bad-auth-bad" }`, + err: "unable to decode auth field", + }, { + name: "decoded auth is not valid", + json: `{ "auth": "Zm9vYmFy" }`, + err: "unable to decode auth field: must be formatted as base64(username:password)", + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got AuthConfig + err := json.Unmarshal([]byte(tc.json), &got) + if tc.err != "" && err == nil { + t.Fatal("no error occurred expected:", tc.err) + } else if tc.err != "" && err != nil { + if !strings.HasPrefix(err.Error(), tc.err) { + t.Fatalf("expected err %q to have prefix %q", err, tc.err) + } + return + } + + if err != nil { + t.Fatal("Unmarshal()=", err) + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Fatal("unexpected diff (-want, +got)\n", diff) + } + }) + } +} diff --git a/pkg/authn/basic.go b/pkg/authn/basic.go new file mode 100644 index 0000000..500cb66 --- /dev/null +++ b/pkg/authn/basic.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +// Basic implements Authenticator for basic authentication. +type Basic struct { + Username string + Password string +} + +// Authorization implements Authenticator. +func (b *Basic) Authorization() (*AuthConfig, error) { + return &AuthConfig{ + Username: b.Username, + Password: b.Password, + }, nil +} diff --git a/pkg/authn/basic_test.go b/pkg/authn/basic_test.go new file mode 100644 index 0000000..aecbe15 --- /dev/null +++ b/pkg/authn/basic_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "reflect" + "testing" +) + +func TestBasic(t *testing.T) { + basic := &Basic{Username: "foo", Password: "bar"} + + got, err := basic.Authorization() + if err != nil { + t.Fatalf("Authorization() = %v", err) + } + want := &AuthConfig{Username: "foo", Password: "bar"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Authorization(); got %v, want %v", got, want) + } +} diff --git a/pkg/authn/bearer.go b/pkg/authn/bearer.go new file mode 100644 index 0000000..4cf86df --- /dev/null +++ b/pkg/authn/bearer.go @@ -0,0 +1,27 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +// Bearer implements Authenticator for bearer authentication. +type Bearer struct { + Token string `json:"token"` +} + +// Authorization implements Authenticator. +func (b *Bearer) Authorization() (*AuthConfig, error) { + return &AuthConfig{ + RegistryToken: b.Token, + }, nil +} diff --git a/pkg/authn/bearer_test.go b/pkg/authn/bearer_test.go new file mode 100644 index 0000000..7d6b26b --- /dev/null +++ b/pkg/authn/bearer_test.go @@ -0,0 +1,31 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "testing" +) + +func TestBearer(t *testing.T) { + anon := &Bearer{Token: "bazinga"} + + auth, err := anon.Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + if got, want := auth.RegistryToken, "bazinga"; got != want { + t.Errorf("Authorization(); got %v, want %v", got, want) + } +} diff --git a/pkg/authn/doc.go b/pkg/authn/doc.go new file mode 100644 index 0000000..c2a5fc0 --- /dev/null +++ b/pkg/authn/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package authn defines different methods of authentication for +// talking to a container registry. +package authn diff --git a/pkg/authn/github/keychain.go b/pkg/authn/github/keychain.go new file mode 100644 index 0000000..97ad34e --- /dev/null +++ b/pkg/authn/github/keychain.go @@ -0,0 +1,59 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package github provides a keychain for the GitHub Container Registry. +package github + +import ( + "net/url" + "os" + + "github.com/google/go-containerregistry/pkg/authn" +) + +const ghcrHostname = "ghcr.io" + +// Keychain exports an instance of the GitHub Keychain. +// +// This keychain matches on requests for ghcr.io and provides the value of the +// environment variable $GITHUB_TOKEN, if it's set. +var Keychain authn.Keychain = githubKeychain{} + +type githubKeychain struct{} + +func (githubKeychain) Resolve(r authn.Resource) (authn.Authenticator, error) { + serverURL, err := url.Parse("https://" + r.String()) + if err != nil { + return authn.Anonymous, nil + } + if serverURL.Hostname() == ghcrHostname { + username := os.Getenv("GITHUB_ACTOR") + if username == "" { + username = "unset" + } + if tok := os.Getenv("GITHUB_TOKEN"); tok != "" { + return githubAuthenticator{username, tok}, nil + } + } + return authn.Anonymous, nil +} + +type githubAuthenticator struct{ username, password string } + +func (g githubAuthenticator) Authorization() (*authn.AuthConfig, error) { + return &authn.AuthConfig{ + Username: g.username, + Password: g.password, + }, nil +} diff --git a/pkg/authn/github/keychain_test.go b/pkg/authn/github/keychain_test.go new file mode 100644 index 0000000..c3e858a --- /dev/null +++ b/pkg/authn/github/keychain_test.go @@ -0,0 +1,112 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "os" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// TestKeychain checks that the keychain resolves when $GITHUB_TOKEN is set and +// the request is for GHCR. +func TestKeychain(t *testing.T) { + username, tok := "octocat", "my-token" + os.Setenv("GITHUB_ACTOR", username) + os.Setenv("GITHUB_TOKEN", tok) + got, err := Keychain.Resolve(resource("ghcr.io/my/repo")) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if got == authn.Anonymous { + t.Fatalf("Got anonymous, wanted authenticator") + } + + auth, err := got.Authorization() + if err != nil { + t.Fatalf("Authorization: %v", err) + } + if auth.Username != username { + t.Errorf("Got username %q, want %q", auth.Username, username) + } + if auth.Password != tok { + t.Errorf("Got password %q, want %q", auth.Password, tok) + } +} + +// TestKeychainUsernameUnset checks that the keychain resolves an "unset" +// username when $GITHUB_ACTOR is not set. +func TestKeychainUsernameUnset(t *testing.T) { + tok := "my-token" + os.Unsetenv("GITHUB_ACTOR") + os.Setenv("GITHUB_TOKEN", tok) + got, err := Keychain.Resolve(resource("ghcr.io/my/repo")) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if got == authn.Anonymous { + t.Fatalf("Got anonymous, wanted authenticator") + } + + auth, err := got.Authorization() + if err != nil { + t.Fatalf("Authorization: %v", err) + } + if auth.Username != "unset" { + t.Errorf("Got username %q, want unset", auth.Username) + } + if auth.Password != tok { + t.Errorf("Got password %q, want %q", auth.Password, tok) + } +} + +// TestKeychainUnset checks that the keychain doesn't resolve when the +// environment variable is unset. +func TestKeychainUnset(t *testing.T) { + os.Unsetenv("GITHUB_TOKEN") + + got, err := Keychain.Resolve(resource("ghcr.io/my/repo")) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if got != authn.Anonymous { + t.Errorf("Resolve(ghcr.io) got %v, want Anonymous", got) + } +} + +// TestNoMatch checks that the keychain doesn't resolve for non-GHCR registries. +func TestNoMatch(t *testing.T) { + os.Setenv("GITHUB_TOKEN", "my-token") + for _, s := range []string{ + "gcr.io", + "example.com", + "ghcr.io.example.com", + "invalid-domain-name -- %U)(@*)(%*)@(*#%@", + } { + got, err := Keychain.Resolve(resource(s)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if got != authn.Anonymous { + t.Errorf("Resolve(%q) got %v, want Anonymous", s, got) + } + } +} + +type resource string + +func (r resource) String() string { return string(r) } +func (r resource) RegistryStr() string { return string(r) } diff --git a/pkg/authn/k8schain/README.md b/pkg/authn/k8schain/README.md new file mode 100644 index 0000000..0bf4371 --- /dev/null +++ b/pkg/authn/k8schain/README.md @@ -0,0 +1,49 @@ +# `k8schain` + +This is an implementation of the [`authn.Keychain`](https://godoc.org/github.com/google/go-containerregistry/authn#Keychain) interface loosely based on the authentication semantics used by the Kubelet when performing the pull of a Pod's images. + +This keychain supports passing a Kubernetes Service Account and some ImagePullSecrets which may represent registry credentials. + +In addition to those, the keychain also includes cloud-specific credential helpers for Google Container Registry (and Artifact Registry), Azure Container Registry, and Amazon AWS Elasic Container Registry. +This means that if the keychain is used from within Kubernetes services on those clouds (GKE, AKS, EKS), any available service credentials will be discovered and used. + +In general this keychain should be used when the code is expected to run in a Kubernetes cluster, and especially when it will run in one of those clouds. +To get a cloud-agnostic keychain, use [`pkg/authn/kubernetes`](../kubernetes) instead. + +To get only cloud-aware keychains, use [`google.Keychain`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/google#Keychain), or [`pkg/authn.NewKeychainFromHelper`](https://godoc.org/github.com/google/go-containerregistry/pkg/authn#NewKeychainFromHelper) with a cloud credential helper implementation -- see the implementation of `k8schain.NewNoClient` for more details. + +## Usage + +### Creating a keychain + +A `k8schain` keychain can be built via one of: + +```go +// client is a kubernetes.Interface +kc, err := k8schain.New(ctx, client, k8schain.Options{}) +... + +// This method is suitable for use by controllers or other in-cluster processes. +kc, err := k8schain.NewInCluster(ctx, k8schain.Options{}) +... +``` + +### Using the keychain + +The `k8schain` keychain can be used directly as an `authn.Keychain`, e.g. + +```go +auth, err := kc.Resolve(registry) +if err != nil { + ... +} +``` + +Or, with the [`remote.WithAuthFromKeychain`](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/v1/remote#WithAuthFromKeychain) option: + +```go +img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) +if err != nil { + ... +} +``` diff --git a/pkg/authn/k8schain/doc.go b/pkg/authn/k8schain/doc.go new file mode 100644 index 0000000..c9ae7f1 --- /dev/null +++ b/pkg/authn/k8schain/doc.go @@ -0,0 +1,18 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package k8schain exposes an implementation of the authn.Keychain interface +// based on the semantics the Kubelet follows when pulling the images for a +// Pod in Kubernetes. +package k8schain diff --git a/pkg/authn/k8schain/go.mod b/pkg/authn/k8schain/go.mod new file mode 100644 index 0000000..b49beac --- /dev/null +++ b/pkg/authn/k8schain/go.mod @@ -0,0 +1,96 @@ +module github.com/google/go-containerregistry/pkg/authn/k8schain + +go 1.18 + +replace ( + github.com/google/go-containerregistry => ../../../ + github.com/google/go-containerregistry/pkg/authn/kubernetes => ../kubernetes/ +) + +require ( + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1 + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 + github.com/google/go-containerregistry v0.13.0 + github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20230309011546-ff810c186c77 + k8s.io/api v0.26.2 + k8s.io/client-go v0.26.2 +) + +require ( + cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/docker/cli v23.0.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v23.0.1+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic v0.6.9 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.29.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.26.2 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect + k8s.io/utils v0.0.0-20230308161112-d77c459e9343 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/pkg/authn/k8schain/go.sum b/pkg/authn/k8schain/go.sum new file mode 100644 index 0000000..e5cb660 --- /dev/null +++ b/pkg/authn/k8schain/go.sum @@ -0,0 +1,364 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= +github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15 h1:0rZQIi6deJFjOEgHI9HI2eZcLPPEGQPictX66oRFLL8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 h1:Kbiv9PGnQfG/imNI4L/heyUXvzKmcWSBeDvkrQz5pFc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5 h1:tGA4ZoAsrYhGBypKAo2jwoX/Z5ponBZOTEUMNN/rHP4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.18.5/go.mod h1:cDZh+PHP8Adt9E0zfZT9cK4qadbtIuU/czLpEJtm4wc= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4 h1:6OBVD6KE4gLReaNfG7CSXFvNIVqKIqrywRcG1kUKr4M= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.4/go.mod h1:gUxgbzXs+gHsj/6al9dzzoByeSrEl03Oj4iJBu/m/Rk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 h1:QoOybhwRfciWUBbZ0gp9S7XaDnCuSTeK/fySB99V1ls= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 h1:qJdM48OOLl1FBSzI7ZrA1ZfLwOyCYqkXV5lko1hYDBw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 h1:YRkWXQveFb0tFC0TLktmmhGsOcCgLwvq88MC2al47AA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 h1:L1600eLr0YvTT7gNh3Ni24yGI7NSHkq9Gp62vijPRCs= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1 h1:uQhxQriOPUu/knXSPM7D/VyS3GMz+4wsE43eB8f9ojg= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230228174139-39c3d18f0af1/go.mod h1:/JmJjW2NJpzRSI3pOxQPC6eOD/tR8SfOA9X1FurmzXI= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ= +k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= +k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ= +k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI= +k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/utils v0.0.0-20230308161112-d77c459e9343 h1:m7tbIjXGcGIAtpmQr7/NAi7RsWoW3E7Zcm4jI1HicTc= +k8s.io/utils v0.0.0-20230308161112-d77c459e9343/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/authn/k8schain/k8schain.go b/pkg/authn/k8schain/k8schain.go new file mode 100644 index 0000000..8ecbd5f --- /dev/null +++ b/pkg/authn/k8schain/k8schain.go @@ -0,0 +1,105 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8schain + +import ( + "context" + "io" + + ecr "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" + "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper" + "github.com/google/go-containerregistry/pkg/authn" + kauth "github.com/google/go-containerregistry/pkg/authn/kubernetes" + "github.com/google/go-containerregistry/pkg/v1/google" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + amazonKeychain authn.Keychain = authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard))) + azureKeychain authn.Keychain = authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper()) +) + +// Options holds configuration data for guiding credential resolution. +type Options = kauth.Options + +// New returns a new authn.Keychain suitable for resolving image references as +// scoped by the provided Options. It speaks to Kubernetes through the provided +// client interface. +func New(ctx context.Context, client kubernetes.Interface, opt Options) (authn.Keychain, error) { + k8s, err := kauth.New(ctx, client, kauth.Options(opt)) + if err != nil { + return nil, err + } + + return authn.NewMultiKeychain( + k8s, + authn.DefaultKeychain, + google.Keychain, + amazonKeychain, + azureKeychain, + ), nil +} + +// NewInCluster returns a new authn.Keychain suitable for resolving image references as +// scoped by the provided Options, constructing a kubernetes.Interface based on in-cluster +// authentication. +func NewInCluster(ctx context.Context, opt Options) (authn.Keychain, error) { + clusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + client, err := kubernetes.NewForConfig(clusterConfig) + if err != nil { + return nil, err + } + return New(ctx, client, opt) +} + +// NewNoClient returns a new authn.Keychain that supports the portions of the K8s keychain +// that don't read ImagePullSecrets. This limits it to roughly the Node-identity-based +// authentication schemes in Kubernetes pkg/credentialprovider. This version of the +// k8schain drops the requirement that we run as a K8s serviceaccount with access to all +// of the on-cluster secrets. This drop in fidelity also diminishes its value as a stand-in +// for Kubernetes authentication, but this actually targets a different use-case. What +// remains is an interesting sweet spot: this variant can serve as a credential provider +// for all of the major public clouds, but in library form (vs. an executable you exec). +func NewNoClient(ctx context.Context) (authn.Keychain, error) { + return authn.NewMultiKeychain( + authn.DefaultKeychain, + google.Keychain, + amazonKeychain, + azureKeychain, + ), nil +} + +// NewFromPullSecrets returns a new authn.Keychain suitable for resolving image references as +// scoped by the pull secrets. +func NewFromPullSecrets(ctx context.Context, pullSecrets []corev1.Secret) (authn.Keychain, error) { + k8s, err := kauth.NewFromPullSecrets(ctx, pullSecrets) + if err != nil { + return nil, err + } + + return authn.NewMultiKeychain( + k8s, + authn.DefaultKeychain, + google.Keychain, + amazonKeychain, + azureKeychain, + ), nil +} diff --git a/pkg/authn/k8schain/tests/explicit/main.go b/pkg/authn/k8schain/tests/explicit/main.go new file mode 100644 index 0000000..744320e --- /dev/null +++ b/pkg/authn/k8schain/tests/explicit/main.go @@ -0,0 +1,52 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + ref, err := name.NewTag("gcr.io/build-crd-testing/secret-sauce:latest") + if err != nil { + log.Fatalf("NewTag() = %v", err) + } + + kc, err := k8schain.NewInCluster(context.Background(), k8schain.Options{ + Namespace: "explicit-namespace", + ImagePullSecrets: []string{ + "explicit-secret", + }, + }) + if err != nil { + log.Fatalf("k8schain.New() = %v", err) + } + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + if err != nil { + log.Fatalf("remote.Image() = %v", err) + } + + digest, err := img.Digest() + if err != nil { + log.Fatalf("Digest() = %v", err) + } + log.Printf("got digest: %v", digest) +} diff --git a/pkg/authn/k8schain/tests/explicit/test.yaml b/pkg/authn/k8schain/tests/explicit/test.yaml new file mode 100644 index 0000000..10cdfba --- /dev/null +++ b/pkg/authn/k8schain/tests/explicit/test.yaml @@ -0,0 +1,59 @@ +# Copyright 2018 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Namespace +metadata: + name: explicit-namespace +--- +apiVersion: v1 +kind: Secret +metadata: + name: explicit-secret + namespace: explicit-namespace +type: kubernetes.io/dockercfg +data: + # This service account is JUST a storage reader on gcr.io/build-crd-testing + .dockercfg: eyJodHRwczovL2djci5pbyI6eyJ1c2VybmFtZSI6Il9qc29uX2tleSIsInBhc3N3b3JkIjoie1xuICBcInR5cGVcIjogXCJzZXJ2aWNlX2FjY291bnRcIixcbiAgXCJwcm9qZWN0X2lkXCI6IFwiYnVpbGQtY3JkLXRlc3RpbmdcIixcbiAgXCJwcml2YXRlX2tleV9pZFwiOiBcIjA1MDJhNDFhODEyZmI2NGNlNTZhNjhlYzU4MzJhYjBiYTExYzExZTZcIixcbiAgXCJwcml2YXRlX2tleVwiOiBcIi0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLVxcbk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzlYNEVZT0FSYnhRTThcXG5EMnhYY2FaVGsrZ1k4ZWp1OTh0THFDUXFUckdNVzlSZVQyeE9ZNUF5Z2FsUFArcDd5WEVja3dCRC9IaE0wZ2xJXFxuN01UTGRlZUtXcityQTFMd0haeVdGVzdIME9uZjd3bllIRUhMV1VtYzNCQ09SRUR0SFJaN1pyUEJmMUhUQUEvM1xcbk1uVzVsWkhTTjlvanpTU0Z3NkFWdTZqNmF4YkJJSUo3NTRMcmdLZUFZdXJ3ZklRMlJMVHUyMDFrMklxTFliaGJcXG4zbVNWRzVSK3RiS3oxQ3ZNNTNuSENiN0NmdVZlV3NyQThrazd4SHJyTFFLTW1JOXYyc2dSdWd5TUF6d3ovNnpOXFxuaDUvaU14eGdlcTVXOHhrVngzSjJuWThKSmRIYWYvVDZBR3NPTkVvNDNweGVpUVZqblJmL0tuMTBUQ2MyRXNJWVxcblM0OVVzWjdCQWdNQkFBRUNnZ0VBQXVwbGR1a0NRUXVENVUvZ2FtSHQ3R2dXM0FNVjE4ZXFuSG5DYTJqbGFoK1NcXG5BZVVHbmhnSmpOdkUrcE1GbFN2NXVmMnAySzRlZC9veEQ2K0NwOVpYRFJqZ3ZmdEl5cWpsemJ3dkZjZ3p3TnVEXFxueWdVa3VwN0hlY0RzRDhUdGVBb2JUL1Zwd3E2ektNckJ3Q3ZOa3Z5NmJWbG9FajV4M2JYc2F4ZTk1RE8veXB1NlxcbncwVzk3enh3d0RKWTZLUWNJV01qaHJHeHZ3WDduaVVDZU00bGVXQkR5R3R3MXplSm40aEVjNk4zYWpRYWNYS2NcXG4rNFFseGNpYW1ZcVFXYlBudHhXUWhoUXpjSFdMaTJsOWNGYlpENyt1SkxGNGlONnk4bVZOVTNLM0sxYlJZclNEXFxuUlVwM2FVVkJYbUZnK1ovMnB1VkwrbVUzajNMTFdZeUJPa2V2dU9tZGdRS0JnUURlM0dJUWt5V0lTMTRUZE1PU1xcbkJpS0JDRHk4aDk2ZWhMMEhrRGJ5T2tTdFBLZEY5cHVFeFp4aHk3b2pIQ0lNNUZWcnBSTjI1cDRzRXp3RmFjK3ZcXG5KSUZnRXZxN21YZm1YaVhJTmllUG9FUWFDbm54RHhXZ21yMEhVS0VtUzlvTWRnTGNHVStrQ1ZHTnN6N0FPdW0wXFxuS3FZM3MyMlE5bFE2N0ZPeXFpdThXRlE3UVFLQmdRRFppRmhURVprUEVjcVpqbndKcFRCNTZaV1A5S1RzbFpQN1xcbndVNGJ6aTZ5K21leWYzTUorNEwyU3lIYzNjcFNNYmp0Tk9aQ3Q0N2I5MDhGVW1MWFVHTmhjd3VaakVReEZleTBcXG5tNDFjUzVlNFA0OWI5bjZ5TEJqQnJCb3FzMldCYWwyZWdkaE5KU3NDV29pWlA4L1pUOGVnWHZoN2I5MWp6b0syXFxucTJQVW1BNERnUUtCZ0FXTDJJanZFSTBPeXgyUzExY24vZTNXSmFUUGdOUFRHOTAzVXBhK3FuemhPSXgrTWFxaFxcblBGNFdzdUF5MEFvZ0dKd2dOSmJOOEh2S1VzRVR2QTV3eXlOMzlYTjd3MGNoYXJGTDM3b3NVK1dPQXpEam5qY3NcXG5BcTVPN0dQR21YdWI2RUJRQlBKaEpQMXd5NHYvSzFmSGcvRjQ3cTRmNDBMQUpPa2FZUkpENUh6QkFvR0JBTlVoXFxubklCUEpxcTRJTXZRNmNDOWc4QisxeFlEZWE5L1lrMXcrU21QR3Z3ckVYeTNHS3g0SzdsS3BiUHo3bTRYMzNzeFxcbnNFVS8rWTJWUW13UmExeFFtLzUzcks3VjJsNUpmL0Q0MDBqUm02WmZTQU92Z0RUcnRablVHSk1yejlFN3VOdzdcXG5sZ1VIM0pyaXZ5Ri9meE1JOHFzelFid1hQMCt4bnlxQXhFQWdkdUtCQW9HQUlNK1BTTllXQ1pYeERwU0hJMThkXFxuaktrb0FidzJNb3l3UUlsa2V1QW4xZFhGYWQxenNYUUdkVHJtWHl2N05QUCs4R1hCa25CTGkzY3Z4VGlsSklTeVxcbnVjTnJDTWlxTkFTbi9kcTdjV0RGVUFCZ2pYMTZKSDJETkZaL2wvVVZGM05EQUpqWENzMVg3eUlKeVhCNm94L3pcXG5hU2xxbElNVjM1REJEN3F4Unl1S3Nnaz1cXG4tLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tXFxuXCIsXG4gIFwiY2xpZW50X2VtYWlsXCI6IFwicHVsbC1zZWNyZXQtdGVzdGluZ0BidWlsZC1jcmQtdGVzdGluZy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbVwiLFxuICBcImNsaWVudF9pZFwiOiBcIjEwNzkzNTg2MjAzMzAyNTI1MTM1MlwiLFxuICBcImF1dGhfdXJpXCI6IFwiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tL28vb2F1dGgyL2F1dGhcIixcbiAgXCJ0b2tlbl91cmlcIjogXCJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20vby9vYXV0aDIvdG9rZW5cIixcbiAgXCJhdXRoX3Byb3ZpZGVyX3g1MDlfY2VydF91cmxcIjogXCJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHNcIixcbiAgXCJjbGllbnRfeDUwOV9jZXJ0X3VybFwiOiBcImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvcHVsbC1zZWNyZXQtdGVzdGluZyU0MGJ1aWxkLWNyZC10ZXN0aW5nLmlhbS5nc2VydmljZWFjY291bnQuY29tXCJcbn0iLCJlbWFpbCI6Im5vcmVwbHlAZ29vZ2xlLmNvbSIsImF1dGgiOiJYMnB6YjI1ZmEyVjVPbnNLSUNBaWRIbHdaU0k2SUNKelpYSjJhV05sWDJGalkyOTFiblFpTEFvZ0lDSndjbTlxWldOMFgybGtJam9nSW1KMWFXeGtMV055WkMxMFpYTjBhVzVuSWl3S0lDQWljSEpwZG1GMFpWOXJaWGxmYVdRaU9pQWlNRFV3TW1FME1XRTRNVEptWWpZMFkyVTFObUUyT0dWak5UZ3pNbUZpTUdKaE1URmpNVEZsTmlJc0NpQWdJbkJ5YVhaaGRHVmZhMlY1SWpvZ0lpMHRMUzB0UWtWSFNVNGdVRkpKVmtGVVJTQkxSVmt0TFMwdExWeHVUVWxKUlhaUlNVSkJSRUZPUW1kcmNXaHJhVWM1ZHpCQ1FWRkZSa0ZCVTBOQ1MyTjNaMmRUYWtGblJVRkJiMGxDUVZGRE9WZzBSVmxQUVZKaWVGRk5PRnh1UkRKNFdHTmhXbFJySzJkWk9HVnFkVGs0ZEV4eFExRnhWSEpIVFZjNVVtVlVNbmhQV1RWQmVXZGhiRkJRSzNBM2VWaEZZMnQzUWtRdlNHaE5NR2RzU1Z4dU4wMVVUR1JsWlV0WGNpdHlRVEZNZDBoYWVWZEdWemRJTUU5dVpqZDNibGxJUlVoTVYxVnRZek5DUTA5U1JVUjBTRkphTjFweVVFSm1NVWhVUVVFdk0xeHVUVzVYTld4YVNGTk9PVzlxZWxOVFJuYzJRVloxTm1vMllYaGlRa2xKU2pjMU5FeHlaMHRsUVZsMWNuZG1TVkV5VWt4VWRUSXdNV3N5U1hGTVdXSm9ZbHh1TTIxVFZrYzFVaXQwWWt0Nk1VTjJUVFV6YmtoRFlqZERablZXWlZkemNrRTRhMnMzZUVoeWNreFJTMDF0U1RsMk1uTm5VblZuZVUxQmVuZDZMelo2VGx4dWFEVXZhVTE0ZUdkbGNUVlhPSGhyVm5nelNqSnVXVGhLU21SSVlXWXZWRFpCUjNOUFRrVnZORE53ZUdWcFVWWnFibEptTDB0dU1UQlVRMk15UlhOSldWeHVVelE1VlhOYU4wSkJaMDFDUVVGRlEyZG5SVUZCZFhCc1pIVnJRMUZSZFVRMVZTOW5ZVzFJZERkSFoxY3pRVTFXTVRobGNXNUlia05oTW1wc1lXZ3JVMXh1UVdWVlIyNW9aMHBxVG5aRkszQk5SbXhUZGpWMVpqSndNa3MwWldRdmIzaEVOaXREY0RsYVdFUlNhbWQyWm5SSmVYRnFiSHBpZDNaR1kyZDZkMDUxUkZ4dWVXZFZhM1Z3TjBobFkwUnpSRGhVZEdWQmIySlVMMVp3ZDNFMmVrdE5ja0ozUTNaT2EzWjVObUpXYkc5RmFqVjRNMkpZYzJGNFpUazFSRTh2ZVhCMU5seHVkekJYT1RkNmVIZDNSRXBaTmt0UlkwbFhUV3BvY2tkNGRuZFlOMjVwVlVObFRUUnNaVmRDUkhsSGRIY3hlbVZLYmpSb1JXTTJUak5oYWxGaFkxaExZMXh1S3pSUmJIaGphV0Z0V1hGUlYySlFiblI0VjFGb2FGRjZZMGhYVEdreWJEbGpSbUphUkRjcmRVcE1SalJwVGpaNU9HMVdUbFV6U3pOTE1XSlNXWEpUUkZ4dVVsVndNMkZWVmtKWWJVWm5LMW92TW5CMVZrd3JiVlV6YWpOTVRGZFplVUpQYTJWMmRVOXRaR2RSUzBKblVVUmxNMGRKVVd0NVYwbFRNVFJVWkUxUFUxeHVRbWxMUWtORWVUaG9PVFpsYUV3d1NHdEVZbmxQYTFOMFVFdGtSamx3ZFVWNFduaG9lVGR2YWtoRFNVMDFSbFp5Y0ZKT01qVndOSE5GZW5kR1lXTXJkbHh1U2tsR1owVjJjVGR0V0dadFdHbFlTVTVwWlZCdlJWRmhRMjV1ZUVSNFYyZHRjakJJVlV0RmJWTTViMDFrWjB4alIxVXJhME5XUjA1emVqZEJUM1Z0TUZ4dVMzRlpNM015TWxFNWJGRTJOMFpQZVhGcGRUaFhSbEUzVVZGTFFtZFJSRnBwUm1oVVJWcHJVRVZqY1ZwcWJuZEtjRlJDTlRaYVYxQTVTMVJ6YkZwUU4xeHVkMVUwWW5wcE5ua3JiV1Y1WmpOTlNpczBUREpUZVVoak0yTndVMDFpYW5ST1QxcERkRFEzWWprd09FWlZiVXhZVlVkT2FHTjNkVnBxUlZGNFJtVjVNRnh1YlRReFkxTTFaVFJRTkRsaU9XNDJlVXhDYWtKeVFtOXhjekpYUW1Gc01tVm5aR2hPU2xOelExZHZhVnBRT0M5YVZEaGxaMWgyYURkaU9URnFlbTlMTWx4dWNUSlFWVzFCTkVSblVVdENaMEZYVERKSmFuWkZTVEJQZVhneVV6RXhZMjR2WlROWFNtRlVVR2RPVUZSSE9UQXpWWEJoSzNGdWVtaFBTWGdyVFdGeGFGeHVVRVkwVjNOMVFYa3dRVzluUjBwM1owNUtZazQ0U0haTFZYTkZWSFpCTlhkNWVVNHpPVmhPTjNjd1kyaGhja1pNTXpkdmMxVXJWMDlCZWtScWJtcGpjMXh1UVhFMVR6ZEhVRWR0V0hWaU5rVkNVVUpRU21oS1VERjNlVFIyTDBzeFpraG5MMFkwTjNFMFpqUXdURUZLVDJ0aFdWSktSRFZJZWtKQmIwZENRVTVWYUZ4dWJrbENVRXB4Y1RSSlRYWlJObU5ET1djNFFpc3hlRmxFWldFNUwxbHJNWGNyVTIxUVIzWjNja1ZZZVROSFMzZzBTemRzUzNCaVVIbzNiVFJZTXpOemVGeHVjMFZWTHl0Wk1sWlJiWGRTWVRGNFVXMHZOVE55U3pkV01tdzFTbVl2UkRRd01HcFNiVFphWmxOQlQzWm5SRlJ5ZEZwdVZVZEtUWEo2T1VVM2RVNTNOMXh1YkdkVlNETktjbWwyZVVZdlpuaE5TVGh4YzNwUlluZFlVREFyZUc1NWNVRjRSVUZuWkhWTFFrRnZSMEZKVFN0UVUwNVpWME5hV0hoRWNGTklTVEU0WkZ4dWFrdHJiMEZpZHpKTmIzbDNVVWxzYTJWMVFXNHhaRmhHWVdReGVuTllVVWRrVkhKdFdIbDJOMDVRVUNzNFIxaENhMjVDVEdrelkzWjRWR2xzU2tsVGVWeHVkV05PY2tOTmFYRk9RVk51TDJSeE4yTlhSRVpWUVVKbmFsZ3hOa3BJTWtST1Jsb3ZiQzlWVmtZelRrUkJTbXBZUTNNeFdEZDVTVXA1V0VJMmIzZ3ZlbHh1WVZOc2NXeEpUVll6TlVSQ1JEZHhlRko1ZFV0eloyczlYRzR0TFMwdExVVk9SQ0JRVWtsV1FWUkZJRXRGV1MwdExTMHRYRzRpTEFvZ0lDSmpiR2xsYm5SZlpXMWhhV3dpT2lBaWNIVnNiQzF6WldOeVpYUXRkR1Z6ZEdsdVowQmlkV2xzWkMxamNtUXRkR1Z6ZEdsdVp5NXBZVzB1WjNObGNuWnBZMlZoWTJOdmRXNTBMbU52YlNJc0NpQWdJbU5zYVdWdWRGOXBaQ0k2SUNJeE1EYzVNelU0TmpJd016TXdNalV5TlRFek5USWlMQW9nSUNKaGRYUm9YM1Z5YVNJNklDSm9kSFJ3Y3pvdkwyRmpZMjkxYm5SekxtZHZiMmRzWlM1amIyMHZieTl2WVhWMGFESXZZWFYwYUNJc0NpQWdJblJ2YTJWdVgzVnlhU0k2SUNKb2RIUndjem92TDJGalkyOTFiblJ6TG1kdmIyZHNaUzVqYjIwdmJ5OXZZWFYwYURJdmRHOXJaVzRpTEFvZ0lDSmhkWFJvWDNCeWIzWnBaR1Z5WDNnMU1EbGZZMlZ5ZEY5MWNtd2lPaUFpYUhSMGNITTZMeTkzZDNjdVoyOXZaMnhsWVhCcGN5NWpiMjB2YjJGMWRHZ3lMM1l4TDJObGNuUnpJaXdLSUNBaVkyeHBaVzUwWDNnMU1EbGZZMlZ5ZEY5MWNtd2lPaUFpYUhSMGNITTZMeTkzZDNjdVoyOXZaMnhsWVhCcGN5NWpiMjB2Y205aWIzUXZkakV2YldWMFlXUmhkR0V2ZURVd09TOXdkV3hzTFhObFkzSmxkQzEwWlhOMGFXNW5KVFF3WW5WcGJHUXRZM0prTFhSbGMzUnBibWN1YVdGdExtZHpaWEoyYVdObFlXTmpiM1Z1ZEM1amIyMGlDbjA9In19 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: explicit + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: explicit +subjects: + - kind: ServiceAccount + name: explicit + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Pod +metadata: + name: explicit + annotations: + sidecar.istio.io/inject: "false" +spec: + serviceAccountName: explicit + containers: + - name: explicit + image: github.com/google/go-containerregistry/pkg/authn/k8schain/tests/explicit + restartPolicy: Never diff --git a/pkg/authn/k8schain/tests/implicit/main.go b/pkg/authn/k8schain/tests/implicit/main.go new file mode 100644 index 0000000..369240f --- /dev/null +++ b/pkg/authn/k8schain/tests/implicit/main.go @@ -0,0 +1,52 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "os" + + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + if len(os.Args) != 2 { + log.Fatalf("expected usage: , got: %v", os.Args) + } + + kc, err := k8schain.NewInCluster(context.Background(), k8schain.Options{}) + if err != nil { + log.Fatalf("k8schain.New() = %v", err) + } + + ref, err := name.NewDigest(os.Args[1]) + if err != nil { + log.Fatalf("NewDigest() = %v", err) + } + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + if err != nil { + log.Fatalf("remote.Image() = %v", err) + } + + digest, err := img.Digest() + if err != nil { + log.Fatalf("Digest() = %v", err) + } + log.Printf("got digest: %v", digest) +} diff --git a/pkg/authn/k8schain/tests/implicit/test.yaml b/pkg/authn/k8schain/tests/implicit/test.yaml new file mode 100644 index 0000000..fba7e33 --- /dev/null +++ b/pkg/authn/k8schain/tests/implicit/test.yaml @@ -0,0 +1,47 @@ +# Copyright 2018 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: implicit + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: implicit +subjects: + - kind: ServiceAccount + name: implicit + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Pod +metadata: + name: implicit + annotations: + sidecar.istio.io/inject: "false" +spec: + serviceAccountName: implicit + containers: + - name: implicit + image: github.com/google/go-containerregistry/pkg/authn/k8schain/tests/implicit + args: + # This test assumes that the KO_DOCKER_REPO is private. + - github.com/google/go-containerregistry/pkg/authn/k8schain/tests/implicit + restartPolicy: Never diff --git a/pkg/authn/k8schain/tests/noauth/main.go b/pkg/authn/k8schain/tests/noauth/main.go new file mode 100644 index 0000000..cff69fa --- /dev/null +++ b/pkg/authn/k8schain/tests/noauth/main.go @@ -0,0 +1,47 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + kc, err := k8schain.NewInCluster(context.Background(), k8schain.Options{}) + if err != nil { + log.Fatalf("k8schain.New() = %v", err) + } + + ref, err := name.ParseReference("ubuntu:latest") + if err != nil { + log.Fatalf("ParseReference() = %v", err) + } + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + if err != nil { + log.Fatalf("remote.Image() = %v", err) + } + + digest, err := img.Digest() + if err != nil { + log.Fatalf("Digest() = %v", err) + } + log.Printf("got digest: %v", digest) +} diff --git a/pkg/authn/k8schain/tests/noauth/test.yaml b/pkg/authn/k8schain/tests/noauth/test.yaml new file mode 100644 index 0000000..a02b302 --- /dev/null +++ b/pkg/authn/k8schain/tests/noauth/test.yaml @@ -0,0 +1,44 @@ +# Copyright 2018 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: noauth + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: noauth +subjects: + - kind: ServiceAccount + name: noauth + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Pod +metadata: + name: noauth + annotations: + sidecar.istio.io/inject: "false" +spec: + serviceAccountName: noauth + containers: + - name: noauth + image: github.com/google/go-containerregistry/pkg/authn/k8schain/tests/noauth + restartPolicy: Never diff --git a/pkg/authn/k8schain/tests/serviceaccount/main.go b/pkg/authn/k8schain/tests/serviceaccount/main.go new file mode 100644 index 0000000..a895dc6 --- /dev/null +++ b/pkg/authn/k8schain/tests/serviceaccount/main.go @@ -0,0 +1,54 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + ref, err := name.NewTag("gcr.io/build-crd-testing/secret-sauce:latest") + if err != nil { + log.Fatalf("NewTag() = %v", err) + } + + kc, err := k8schain.NewInCluster(context.Background(), k8schain.Options{ + Namespace: "serviceaccount-namespace", + ServiceAccountName: "serviceaccount", + // This is the name of the imagePullSecrets attached to this service account. + // ImagePullSecrets: []string{ + // "serviceaccount-secret", + // }, + }) + if err != nil { + log.Fatalf("k8schain.New() = %v", err) + } + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + if err != nil { + log.Fatalf("remote.Image() = %v", err) + } + + digest, err := img.Digest() + if err != nil { + log.Fatalf("Digest() = %v", err) + } + log.Printf("got digest: %v", digest) +} diff --git a/pkg/authn/k8schain/tests/serviceaccount/test.yaml b/pkg/authn/k8schain/tests/serviceaccount/test.yaml new file mode 100644 index 0000000..f8fa089 --- /dev/null +++ b/pkg/authn/k8schain/tests/serviceaccount/test.yaml @@ -0,0 +1,67 @@ +# Copyright 2018 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Namespace +metadata: + name: serviceaccount-namespace +--- +apiVersion: v1 +kind: Secret +metadata: + name: serviceaccount-secret + namespace: serviceaccount-namespace +type: kubernetes.io/dockercfg +data: + # This service account is JUST a storage reader on gcr.io/build-crd-testing + .dockercfg: eyJodHRwczovL2djci5pbyI6eyJ1c2VybmFtZSI6Il9qc29uX2tleSIsInBhc3N3b3JkIjoie1xuICBcInR5cGVcIjogXCJzZXJ2aWNlX2FjY291bnRcIixcbiAgXCJwcm9qZWN0X2lkXCI6IFwiYnVpbGQtY3JkLXRlc3RpbmdcIixcbiAgXCJwcml2YXRlX2tleV9pZFwiOiBcIjA1MDJhNDFhODEyZmI2NGNlNTZhNjhlYzU4MzJhYjBiYTExYzExZTZcIixcbiAgXCJwcml2YXRlX2tleVwiOiBcIi0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLVxcbk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzlYNEVZT0FSYnhRTThcXG5EMnhYY2FaVGsrZ1k4ZWp1OTh0THFDUXFUckdNVzlSZVQyeE9ZNUF5Z2FsUFArcDd5WEVja3dCRC9IaE0wZ2xJXFxuN01UTGRlZUtXcityQTFMd0haeVdGVzdIME9uZjd3bllIRUhMV1VtYzNCQ09SRUR0SFJaN1pyUEJmMUhUQUEvM1xcbk1uVzVsWkhTTjlvanpTU0Z3NkFWdTZqNmF4YkJJSUo3NTRMcmdLZUFZdXJ3ZklRMlJMVHUyMDFrMklxTFliaGJcXG4zbVNWRzVSK3RiS3oxQ3ZNNTNuSENiN0NmdVZlV3NyQThrazd4SHJyTFFLTW1JOXYyc2dSdWd5TUF6d3ovNnpOXFxuaDUvaU14eGdlcTVXOHhrVngzSjJuWThKSmRIYWYvVDZBR3NPTkVvNDNweGVpUVZqblJmL0tuMTBUQ2MyRXNJWVxcblM0OVVzWjdCQWdNQkFBRUNnZ0VBQXVwbGR1a0NRUXVENVUvZ2FtSHQ3R2dXM0FNVjE4ZXFuSG5DYTJqbGFoK1NcXG5BZVVHbmhnSmpOdkUrcE1GbFN2NXVmMnAySzRlZC9veEQ2K0NwOVpYRFJqZ3ZmdEl5cWpsemJ3dkZjZ3p3TnVEXFxueWdVa3VwN0hlY0RzRDhUdGVBb2JUL1Zwd3E2ektNckJ3Q3ZOa3Z5NmJWbG9FajV4M2JYc2F4ZTk1RE8veXB1NlxcbncwVzk3enh3d0RKWTZLUWNJV01qaHJHeHZ3WDduaVVDZU00bGVXQkR5R3R3MXplSm40aEVjNk4zYWpRYWNYS2NcXG4rNFFseGNpYW1ZcVFXYlBudHhXUWhoUXpjSFdMaTJsOWNGYlpENyt1SkxGNGlONnk4bVZOVTNLM0sxYlJZclNEXFxuUlVwM2FVVkJYbUZnK1ovMnB1VkwrbVUzajNMTFdZeUJPa2V2dU9tZGdRS0JnUURlM0dJUWt5V0lTMTRUZE1PU1xcbkJpS0JDRHk4aDk2ZWhMMEhrRGJ5T2tTdFBLZEY5cHVFeFp4aHk3b2pIQ0lNNUZWcnBSTjI1cDRzRXp3RmFjK3ZcXG5KSUZnRXZxN21YZm1YaVhJTmllUG9FUWFDbm54RHhXZ21yMEhVS0VtUzlvTWRnTGNHVStrQ1ZHTnN6N0FPdW0wXFxuS3FZM3MyMlE5bFE2N0ZPeXFpdThXRlE3UVFLQmdRRFppRmhURVprUEVjcVpqbndKcFRCNTZaV1A5S1RzbFpQN1xcbndVNGJ6aTZ5K21leWYzTUorNEwyU3lIYzNjcFNNYmp0Tk9aQ3Q0N2I5MDhGVW1MWFVHTmhjd3VaakVReEZleTBcXG5tNDFjUzVlNFA0OWI5bjZ5TEJqQnJCb3FzMldCYWwyZWdkaE5KU3NDV29pWlA4L1pUOGVnWHZoN2I5MWp6b0syXFxucTJQVW1BNERnUUtCZ0FXTDJJanZFSTBPeXgyUzExY24vZTNXSmFUUGdOUFRHOTAzVXBhK3FuemhPSXgrTWFxaFxcblBGNFdzdUF5MEFvZ0dKd2dOSmJOOEh2S1VzRVR2QTV3eXlOMzlYTjd3MGNoYXJGTDM3b3NVK1dPQXpEam5qY3NcXG5BcTVPN0dQR21YdWI2RUJRQlBKaEpQMXd5NHYvSzFmSGcvRjQ3cTRmNDBMQUpPa2FZUkpENUh6QkFvR0JBTlVoXFxubklCUEpxcTRJTXZRNmNDOWc4QisxeFlEZWE5L1lrMXcrU21QR3Z3ckVYeTNHS3g0SzdsS3BiUHo3bTRYMzNzeFxcbnNFVS8rWTJWUW13UmExeFFtLzUzcks3VjJsNUpmL0Q0MDBqUm02WmZTQU92Z0RUcnRablVHSk1yejlFN3VOdzdcXG5sZ1VIM0pyaXZ5Ri9meE1JOHFzelFid1hQMCt4bnlxQXhFQWdkdUtCQW9HQUlNK1BTTllXQ1pYeERwU0hJMThkXFxuaktrb0FidzJNb3l3UUlsa2V1QW4xZFhGYWQxenNYUUdkVHJtWHl2N05QUCs4R1hCa25CTGkzY3Z4VGlsSklTeVxcbnVjTnJDTWlxTkFTbi9kcTdjV0RGVUFCZ2pYMTZKSDJETkZaL2wvVVZGM05EQUpqWENzMVg3eUlKeVhCNm94L3pcXG5hU2xxbElNVjM1REJEN3F4Unl1S3Nnaz1cXG4tLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tXFxuXCIsXG4gIFwiY2xpZW50X2VtYWlsXCI6IFwicHVsbC1zZWNyZXQtdGVzdGluZ0BidWlsZC1jcmQtdGVzdGluZy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbVwiLFxuICBcImNsaWVudF9pZFwiOiBcIjEwNzkzNTg2MjAzMzAyNTI1MTM1MlwiLFxuICBcImF1dGhfdXJpXCI6IFwiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tL28vb2F1dGgyL2F1dGhcIixcbiAgXCJ0b2tlbl91cmlcIjogXCJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20vby9vYXV0aDIvdG9rZW5cIixcbiAgXCJhdXRoX3Byb3ZpZGVyX3g1MDlfY2VydF91cmxcIjogXCJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHNcIixcbiAgXCJjbGllbnRfeDUwOV9jZXJ0X3VybFwiOiBcImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvcHVsbC1zZWNyZXQtdGVzdGluZyU0MGJ1aWxkLWNyZC10ZXN0aW5nLmlhbS5nc2VydmljZWFjY291bnQuY29tXCJcbn0iLCJlbWFpbCI6Im5vcmVwbHlAZ29vZ2xlLmNvbSIsImF1dGgiOiJYMnB6YjI1ZmEyVjVPbnNLSUNBaWRIbHdaU0k2SUNKelpYSjJhV05sWDJGalkyOTFiblFpTEFvZ0lDSndjbTlxWldOMFgybGtJam9nSW1KMWFXeGtMV055WkMxMFpYTjBhVzVuSWl3S0lDQWljSEpwZG1GMFpWOXJaWGxmYVdRaU9pQWlNRFV3TW1FME1XRTRNVEptWWpZMFkyVTFObUUyT0dWak5UZ3pNbUZpTUdKaE1URmpNVEZsTmlJc0NpQWdJbkJ5YVhaaGRHVmZhMlY1SWpvZ0lpMHRMUzB0UWtWSFNVNGdVRkpKVmtGVVJTQkxSVmt0TFMwdExWeHVUVWxKUlhaUlNVSkJSRUZPUW1kcmNXaHJhVWM1ZHpCQ1FWRkZSa0ZCVTBOQ1MyTjNaMmRUYWtGblJVRkJiMGxDUVZGRE9WZzBSVmxQUVZKaWVGRk5PRnh1UkRKNFdHTmhXbFJySzJkWk9HVnFkVGs0ZEV4eFExRnhWSEpIVFZjNVVtVlVNbmhQV1RWQmVXZGhiRkJRSzNBM2VWaEZZMnQzUWtRdlNHaE5NR2RzU1Z4dU4wMVVUR1JsWlV0WGNpdHlRVEZNZDBoYWVWZEdWemRJTUU5dVpqZDNibGxJUlVoTVYxVnRZek5DUTA5U1JVUjBTRkphTjFweVVFSm1NVWhVUVVFdk0xeHVUVzVYTld4YVNGTk9PVzlxZWxOVFJuYzJRVloxTm1vMllYaGlRa2xKU2pjMU5FeHlaMHRsUVZsMWNuZG1TVkV5VWt4VWRUSXdNV3N5U1hGTVdXSm9ZbHh1TTIxVFZrYzFVaXQwWWt0Nk1VTjJUVFV6YmtoRFlqZERablZXWlZkemNrRTRhMnMzZUVoeWNreFJTMDF0U1RsMk1uTm5VblZuZVUxQmVuZDZMelo2VGx4dWFEVXZhVTE0ZUdkbGNUVlhPSGhyVm5nelNqSnVXVGhLU21SSVlXWXZWRFpCUjNOUFRrVnZORE53ZUdWcFVWWnFibEptTDB0dU1UQlVRMk15UlhOSldWeHVVelE1VlhOYU4wSkJaMDFDUVVGRlEyZG5SVUZCZFhCc1pIVnJRMUZSZFVRMVZTOW5ZVzFJZERkSFoxY3pRVTFXTVRobGNXNUlia05oTW1wc1lXZ3JVMXh1UVdWVlIyNW9aMHBxVG5aRkszQk5SbXhUZGpWMVpqSndNa3MwWldRdmIzaEVOaXREY0RsYVdFUlNhbWQyWm5SSmVYRnFiSHBpZDNaR1kyZDZkMDUxUkZ4dWVXZFZhM1Z3TjBobFkwUnpSRGhVZEdWQmIySlVMMVp3ZDNFMmVrdE5ja0ozUTNaT2EzWjVObUpXYkc5RmFqVjRNMkpZYzJGNFpUazFSRTh2ZVhCMU5seHVkekJYT1RkNmVIZDNSRXBaTmt0UlkwbFhUV3BvY2tkNGRuZFlOMjVwVlVObFRUUnNaVmRDUkhsSGRIY3hlbVZLYmpSb1JXTTJUak5oYWxGaFkxaExZMXh1S3pSUmJIaGphV0Z0V1hGUlYySlFiblI0VjFGb2FGRjZZMGhYVEdreWJEbGpSbUphUkRjcmRVcE1SalJwVGpaNU9HMVdUbFV6U3pOTE1XSlNXWEpUUkZ4dVVsVndNMkZWVmtKWWJVWm5LMW92TW5CMVZrd3JiVlV6YWpOTVRGZFplVUpQYTJWMmRVOXRaR2RSUzBKblVVUmxNMGRKVVd0NVYwbFRNVFJVWkUxUFUxeHVRbWxMUWtORWVUaG9PVFpsYUV3d1NHdEVZbmxQYTFOMFVFdGtSamx3ZFVWNFduaG9lVGR2YWtoRFNVMDFSbFp5Y0ZKT01qVndOSE5GZW5kR1lXTXJkbHh1U2tsR1owVjJjVGR0V0dadFdHbFlTVTVwWlZCdlJWRmhRMjV1ZUVSNFYyZHRjakJJVlV0RmJWTTViMDFrWjB4alIxVXJhME5XUjA1emVqZEJUM1Z0TUZ4dVMzRlpNM015TWxFNWJGRTJOMFpQZVhGcGRUaFhSbEUzVVZGTFFtZFJSRnBwUm1oVVJWcHJVRVZqY1ZwcWJuZEtjRlJDTlRaYVYxQTVTMVJ6YkZwUU4xeHVkMVUwWW5wcE5ua3JiV1Y1WmpOTlNpczBUREpUZVVoak0yTndVMDFpYW5ST1QxcERkRFEzWWprd09FWlZiVXhZVlVkT2FHTjNkVnBxUlZGNFJtVjVNRnh1YlRReFkxTTFaVFJRTkRsaU9XNDJlVXhDYWtKeVFtOXhjekpYUW1Gc01tVm5aR2hPU2xOelExZHZhVnBRT0M5YVZEaGxaMWgyYURkaU9URnFlbTlMTWx4dWNUSlFWVzFCTkVSblVVdENaMEZYVERKSmFuWkZTVEJQZVhneVV6RXhZMjR2WlROWFNtRlVVR2RPVUZSSE9UQXpWWEJoSzNGdWVtaFBTWGdyVFdGeGFGeHVVRVkwVjNOMVFYa3dRVzluUjBwM1owNUtZazQ0U0haTFZYTkZWSFpCTlhkNWVVNHpPVmhPTjNjd1kyaGhja1pNTXpkdmMxVXJWMDlCZWtScWJtcGpjMXh1UVhFMVR6ZEhVRWR0V0hWaU5rVkNVVUpRU21oS1VERjNlVFIyTDBzeFpraG5MMFkwTjNFMFpqUXdURUZLVDJ0aFdWSktSRFZJZWtKQmIwZENRVTVWYUZ4dWJrbENVRXB4Y1RSSlRYWlJObU5ET1djNFFpc3hlRmxFWldFNUwxbHJNWGNyVTIxUVIzWjNja1ZZZVROSFMzZzBTemRzUzNCaVVIbzNiVFJZTXpOemVGeHVjMFZWTHl0Wk1sWlJiWGRTWVRGNFVXMHZOVE55U3pkV01tdzFTbVl2UkRRd01HcFNiVFphWmxOQlQzWm5SRlJ5ZEZwdVZVZEtUWEo2T1VVM2RVNTNOMXh1YkdkVlNETktjbWwyZVVZdlpuaE5TVGh4YzNwUlluZFlVREFyZUc1NWNVRjRSVUZuWkhWTFFrRnZSMEZKVFN0UVUwNVpWME5hV0hoRWNGTklTVEU0WkZ4dWFrdHJiMEZpZHpKTmIzbDNVVWxzYTJWMVFXNHhaRmhHWVdReGVuTllVVWRrVkhKdFdIbDJOMDVRVUNzNFIxaENhMjVDVEdrelkzWjRWR2xzU2tsVGVWeHVkV05PY2tOTmFYRk9RVk51TDJSeE4yTlhSRVpWUVVKbmFsZ3hOa3BJTWtST1Jsb3ZiQzlWVmtZelRrUkJTbXBZUTNNeFdEZDVTVXA1V0VJMmIzZ3ZlbHh1WVZOc2NXeEpUVll6TlVSQ1JEZHhlRko1ZFV0eloyczlYRzR0TFMwdExVVk9SQ0JRVWtsV1FWUkZJRXRGV1MwdExTMHRYRzRpTEFvZ0lDSmpiR2xsYm5SZlpXMWhhV3dpT2lBaWNIVnNiQzF6WldOeVpYUXRkR1Z6ZEdsdVowQmlkV2xzWkMxamNtUXRkR1Z6ZEdsdVp5NXBZVzB1WjNObGNuWnBZMlZoWTJOdmRXNTBMbU52YlNJc0NpQWdJbU5zYVdWdWRGOXBaQ0k2SUNJeE1EYzVNelU0TmpJd016TXdNalV5TlRFek5USWlMQW9nSUNKaGRYUm9YM1Z5YVNJNklDSm9kSFJ3Y3pvdkwyRmpZMjkxYm5SekxtZHZiMmRzWlM1amIyMHZieTl2WVhWMGFESXZZWFYwYUNJc0NpQWdJblJ2YTJWdVgzVnlhU0k2SUNKb2RIUndjem92TDJGalkyOTFiblJ6TG1kdmIyZHNaUzVqYjIwdmJ5OXZZWFYwYURJdmRHOXJaVzRpTEFvZ0lDSmhkWFJvWDNCeWIzWnBaR1Z5WDNnMU1EbGZZMlZ5ZEY5MWNtd2lPaUFpYUhSMGNITTZMeTkzZDNjdVoyOXZaMnhsWVhCcGN5NWpiMjB2YjJGMWRHZ3lMM1l4TDJObGNuUnpJaXdLSUNBaVkyeHBaVzUwWDNnMU1EbGZZMlZ5ZEY5MWNtd2lPaUFpYUhSMGNITTZMeTkzZDNjdVoyOXZaMnhsWVhCcGN5NWpiMjB2Y205aWIzUXZkakV2YldWMFlXUmhkR0V2ZURVd09TOXdkV3hzTFhObFkzSmxkQzEwWlhOMGFXNW5KVFF3WW5WcGJHUXRZM0prTFhSbGMzUnBibWN1YVdGdExtZHpaWEoyYVdObFlXTmpiM1Z1ZEM1amIyMGlDbjA9In19 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: serviceaccount + namespace: serviceaccount-namespace +imagePullSecrets: +- name: serviceaccount-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: serviceaccount + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: serviceaccount +subjects: + - kind: ServiceAccount + name: serviceaccount + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Pod +metadata: + name: serviceaccount + annotations: + sidecar.istio.io/inject: "false" +spec: + serviceAccountName: serviceaccount + containers: + - name: serviceaccount + image: github.com/google/go-containerregistry/pkg/authn/k8schain/tests/serviceaccount + restartPolicy: Never diff --git a/pkg/authn/keychain.go b/pkg/authn/keychain.go new file mode 100644 index 0000000..a4a88b3 --- /dev/null +++ b/pkg/authn/keychain.go @@ -0,0 +1,180 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "os" + "path/filepath" + "sync" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/config/types" + "github.com/google/go-containerregistry/pkg/name" + "github.com/mitchellh/go-homedir" +) + +// Resource represents a registry or repository that can be authenticated against. +type Resource interface { + // String returns the full string representation of the target, e.g. + // gcr.io/my-project or just gcr.io. + String() string + + // RegistryStr returns just the registry portion of the target, e.g. for + // gcr.io/my-project, this should just return gcr.io. This is needed to + // pull out an appropriate hostname. + RegistryStr() string +} + +// Keychain is an interface for resolving an image reference to a credential. +type Keychain interface { + // Resolve looks up the most appropriate credential for the specified target. + Resolve(Resource) (Authenticator, error) +} + +// defaultKeychain implements Keychain with the semantics of the standard Docker +// credential keychain. +type defaultKeychain struct { + mu sync.Mutex +} + +var ( + // DefaultKeychain implements Keychain by interpreting the docker config file. + DefaultKeychain Keychain = &defaultKeychain{} +) + +const ( + // DefaultAuthKey is the key used for dockerhub in config files, which + // is hardcoded for historical reasons. + DefaultAuthKey = "https://" + name.DefaultRegistry + "/v1/" +) + +// Resolve implements Keychain. +func (dk *defaultKeychain) Resolve(target Resource) (Authenticator, error) { + dk.mu.Lock() + defer dk.mu.Unlock() + + // Podman users may have their container registry auth configured in a + // different location, that Docker packages aren't aware of. + // If the Docker config file isn't found, we'll fallback to look where + // Podman configures it, and parse that as a Docker auth config instead. + + // First, check $HOME/.docker/config.json + foundDockerConfig := false + home, err := homedir.Dir() + if err == nil { + foundDockerConfig = fileExists(filepath.Join(home, ".docker/config.json")) + } + // If $HOME/.docker/config.json isn't found, check $DOCKER_CONFIG (if set) + if !foundDockerConfig && os.Getenv("DOCKER_CONFIG") != "" { + foundDockerConfig = fileExists(filepath.Join(os.Getenv("DOCKER_CONFIG"), "config.json")) + } + // If either of those locations are found, load it using Docker's + // config.Load, which may fail if the config can't be parsed. + // + // If neither was found, look for Podman's auth at + // $XDG_RUNTIME_DIR/containers/auth.json and attempt to load it as a + // Docker config. + // + // If neither are found, fallback to Anonymous. + var cf *configfile.ConfigFile + if foundDockerConfig { + cf, err = config.Load(os.Getenv("DOCKER_CONFIG")) + if err != nil { + return nil, err + } + } else { + f, err := os.Open(filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "containers/auth.json")) + if err != nil { + return Anonymous, nil + } + defer f.Close() + cf, err = config.LoadFromReader(f) + if err != nil { + return nil, err + } + } + + // See: + // https://github.com/google/ko/issues/90 + // https://github.com/moby/moby/blob/fc01c2b481097a6057bec3cd1ab2d7b4488c50c4/registry/config.go#L397-L404 + var cfg, empty types.AuthConfig + for _, key := range []string{ + target.String(), + target.RegistryStr(), + } { + if key == name.DefaultRegistry { + key = DefaultAuthKey + } + + cfg, err = cf.GetAuthConfig(key) + if err != nil { + return nil, err + } + // cf.GetAuthConfig automatically sets the ServerAddress attribute. Since + // we don't make use of it, clear the value for a proper "is-empty" test. + // See: https://github.com/google/go-containerregistry/issues/1510 + cfg.ServerAddress = "" + if cfg != empty { + break + } + } + if cfg == empty { + return Anonymous, nil + } + + return FromConfig(AuthConfig{ + Username: cfg.Username, + Password: cfg.Password, + Auth: cfg.Auth, + IdentityToken: cfg.IdentityToken, + RegistryToken: cfg.RegistryToken, + }), nil +} + +// fileExists returns true if the given path exists and is not a directory. +func fileExists(path string) bool { + fi, err := os.Stat(path) + return err == nil && !fi.IsDir() +} + +// Helper is a subset of the Docker credential helper credentials.Helper +// interface used by NewKeychainFromHelper. +// +// See: +// https://pkg.go.dev/github.com/docker/docker-credential-helpers/credentials#Helper +type Helper interface { + Get(serverURL string) (string, string, error) +} + +// NewKeychainFromHelper returns a Keychain based on a Docker credential helper +// implementation that can Get username and password credentials for a given +// server URL. +func NewKeychainFromHelper(h Helper) Keychain { return wrapper{h} } + +type wrapper struct{ h Helper } + +func (w wrapper) Resolve(r Resource) (Authenticator, error) { + u, p, err := w.h.Get(r.RegistryStr()) + if err != nil { + return Anonymous, nil + } + // If the secret being stored is an identity token, the Username should be set to + // ref: https://docs.docker.com/engine/reference/commandline/login/#credential-helper-protocol + if u == "" { + return FromConfig(AuthConfig{Username: u, IdentityToken: p}), nil + } + return FromConfig(AuthConfig{Username: u, Password: p}), nil +} diff --git a/pkg/authn/keychain_test.go b/pkg/authn/keychain_test.go new file mode 100644 index 0000000..9dfbad1 --- /dev/null +++ b/pkg/authn/keychain_test.go @@ -0,0 +1,392 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "encoding/base64" + "errors" + "fmt" + "log" + "os" + "path" + "path/filepath" + "reflect" + "testing" + + "github.com/google/go-containerregistry/pkg/name" +) + +var ( + fresh = 0 + testRegistry, _ = name.NewRegistry("test.io", name.WeakValidation) + testRepo, _ = name.NewRepository("test.io/my-repo", name.WeakValidation) + defaultRegistry, _ = name.NewRegistry(name.DefaultRegistry, name.WeakValidation) +) + +func TestMain(m *testing.M) { + // Set $HOME to a temp empty dir, to ensure $HOME/.docker/config.json + // isn't unexpectedly found. + tmp, err := os.MkdirTemp("", "keychain_test_home") + if err != nil { + log.Fatal(err) + } + os.Setenv("HOME", tmp) + os.Exit(func() int { + defer os.RemoveAll(tmp) + return m.Run() + }()) +} + +// setupConfigDir sets up an isolated configDir() for this test. +func setupConfigDir(t *testing.T) string { + tmpdir := os.Getenv("TEST_TMPDIR") + if tmpdir == "" { + tmpdir = t.TempDir() + } + + fresh++ + p := filepath.Join(tmpdir, fmt.Sprintf("%d", fresh)) + t.Logf("DOCKER_CONFIG=%s", p) + t.Setenv("DOCKER_CONFIG", p) + if err := os.Mkdir(p, 0777); err != nil { + t.Fatalf("mkdir %q: %v", p, err) + } + return p +} + +func setupConfigFile(t *testing.T, content string) string { + cd := setupConfigDir(t) + p := filepath.Join(cd, "config.json") + if err := os.WriteFile(p, []byte(content), 0600); err != nil { + t.Fatalf("write %q: %v", p, err) + } + + // return the config dir so we can clean up + return cd +} + +func TestNoConfig(t *testing.T) { + cd := setupConfigDir(t) + defer os.RemoveAll(filepath.Dir(cd)) + + auth, err := DefaultKeychain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + + if auth != Anonymous { + t.Errorf("expected Anonymous, got %v", auth) + } +} + +func TestPodmanConfig(t *testing.T) { + tmpdir := os.Getenv("TEST_TMPDIR") + if tmpdir == "" { + tmpdir = t.TempDir() + } + fresh++ + p := filepath.Join(tmpdir, fmt.Sprintf("%d", fresh)) + t.Setenv("XDG_RUNTIME_DIR", p) + os.Unsetenv("DOCKER_CONFIG") + if err := os.MkdirAll(filepath.Join(p, "containers"), 0777); err != nil { + t.Fatalf("mkdir %s/containers: %v", p, err) + } + cfg := filepath.Join(p, "containers/auth.json") + content := fmt.Sprintf(`{"auths": {"test.io": {"auth": %q}}}`, encode("foo", "bar")) + if err := os.WriteFile(cfg, []byte(content), 0600); err != nil { + t.Fatalf("write %q: %v", cfg, err) + } + + // At first, $DOCKER_CONFIG is unset and $HOME/.docker/config.json isn't + // found, but Podman auth is configured. This should return Podman's + // auth. + auth, err := DefaultKeychain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + got, err := auth.Authorization() + if err != nil { + t.Fatal(err) + } + want := &AuthConfig{ + Username: "foo", + Password: "bar", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } + + // Now, configure $HOME/.docker/config.json, which should override + // Podman auth and be used. + if err := os.MkdirAll(filepath.Join(os.Getenv("HOME"), ".docker"), 0777); err != nil { + t.Fatalf("mkdir $HOME/.docker: %v", err) + } + cfg = filepath.Join(os.Getenv("HOME"), ".docker/config.json") + content = fmt.Sprintf(`{"auths": {"test.io": {"auth": %q}}}`, encode("home-foo", "home-bar")) + if err := os.WriteFile(cfg, []byte(content), 0600); err != nil { + t.Fatalf("write %q: %v", cfg, err) + } + defer func() { os.Remove(cfg) }() + auth, err = DefaultKeychain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + got, err = auth.Authorization() + if err != nil { + t.Fatal(err) + } + want = &AuthConfig{ + Username: "home-foo", + Password: "home-bar", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } + + // Then, configure DOCKER_CONFIG with a valid config file with different + // auth configured. + // This demonstrates that DOCKER_CONFIG is preferred over Podman auth + // and $HOME/.docker/config.json. + content = fmt.Sprintf(`{"auths": {"test.io": {"auth": %q}}}`, encode("another-foo", "another-bar")) + cd := setupConfigFile(t, content) + defer os.RemoveAll(filepath.Dir(cd)) + + auth, err = DefaultKeychain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + got, err = auth.Authorization() + if err != nil { + t.Fatal(err) + } + want = &AuthConfig{ + Username: "another-foo", + Password: "another-bar", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } +} + +func encode(user, pass string) string { + delimited := fmt.Sprintf("%s:%s", user, pass) + return base64.StdEncoding.EncodeToString([]byte(delimited)) +} + +func TestVariousPaths(t *testing.T) { + tests := []struct { + desc string + content string + wantErr bool + target Resource + cfg *AuthConfig + anonymous bool + }{{ + desc: "invalid config file", + target: testRegistry, + content: `}{`, + wantErr: true, + }, { + desc: "creds store does not exist", + target: testRegistry, + content: `{"credsStore":"#definitely-does-not-exist"}`, + wantErr: true, + }, { + desc: "valid config file", + target: testRegistry, + content: fmt.Sprintf(`{"auths": {"test.io": {"auth": %q}}}`, encode("foo", "bar")), + cfg: &AuthConfig{ + Username: "foo", + Password: "bar", + }, + }, { + desc: "valid config file; default registry", + target: defaultRegistry, + content: fmt.Sprintf(`{"auths": {"%s": {"auth": %q}}}`, DefaultAuthKey, encode("foo", "bar")), + cfg: &AuthConfig{ + Username: "foo", + Password: "bar", + }, + }, { + desc: "valid config file; matches registry w/ v1", + target: testRegistry, + content: fmt.Sprintf(`{ + "auths": { + "http://test.io/v1/": {"auth": %q} + } + }`, encode("baz", "quux")), + cfg: &AuthConfig{ + Username: "baz", + Password: "quux", + }, + }, { + desc: "valid config file; matches registry w/ v2", + target: testRegistry, + content: fmt.Sprintf(`{ + "auths": { + "http://test.io/v2/": {"auth": %q} + } + }`, encode("baz", "quux")), + cfg: &AuthConfig{ + Username: "baz", + Password: "quux", + }, + }, { + desc: "valid config file; matches repo", + target: testRepo, + content: fmt.Sprintf(`{ + "auths": { + "test.io/my-repo": {"auth": %q}, + "test.io/another-repo": {"auth": %q}, + "test.io": {"auth": %q} + } +}`, encode("foo", "bar"), encode("bar", "baz"), encode("baz", "quux")), + cfg: &AuthConfig{ + Username: "foo", + Password: "bar", + }, + }, { + desc: "ignore unrelated repo", + target: testRepo, + content: fmt.Sprintf(`{ + "auths": { + "test.io/another-repo": {"auth": %q}, + "test.io": {} + } +}`, encode("bar", "baz")), + cfg: &AuthConfig{}, + anonymous: true, + }} + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + cd := setupConfigFile(t, test.content) + // For some reason, these tempdirs don't get cleaned up. + defer os.RemoveAll(filepath.Dir(cd)) + + auth, err := DefaultKeychain.Resolve(test.target) + if test.wantErr { + if err == nil { + t.Fatal("wanted err, got nil") + } else if err != nil { + // success + return + } + } + if err != nil { + t.Fatalf("wanted nil, got err: %v", err) + } + cfg, err := auth.Authorization() + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(cfg, test.cfg) { + t.Errorf("got %+v, want %+v", cfg, test.cfg) + } + + if test.anonymous != (auth == Anonymous) { + t.Fatalf("unexpected anonymous authenticator") + } + }) + } +} + +type helper struct { + u, p string + err error +} + +func (h helper) Get(serverURL string) (string, string, error) { + if serverURL != "example.com" { + return "", "", fmt.Errorf("unexpected serverURL: %s", serverURL) + } + return h.u, h.p, h.err +} + +func TestNewKeychainFromHelper(t *testing.T) { + var repo = name.MustParseReference("example.com/my/repo").Context() + + t.Run("success", func(t *testing.T) { + kc := NewKeychainFromHelper(helper{"username", "password", nil}) + auth, err := kc.Resolve(repo) + if err != nil { + t.Fatalf("Resolve(%q): %v", repo, err) + } + cfg, err := auth.Authorization() + if err != nil { + t.Fatalf("Authorization: %v", err) + } + if got, want := cfg.Username, "username"; got != want { + t.Errorf("Username: got %q, want %q", got, want) + } + if got, want := cfg.IdentityToken, ""; got != want { + t.Errorf("IdentityToken: got %q, want %q", got, want) + } + if got, want := cfg.Password, "password"; got != want { + t.Errorf("Password: got %q, want %q", got, want) + } + }) + + t.Run("success; identity token", func(t *testing.T) { + kc := NewKeychainFromHelper(helper{"", "idtoken", nil}) + auth, err := kc.Resolve(repo) + if err != nil { + t.Fatalf("Resolve(%q): %v", repo, err) + } + cfg, err := auth.Authorization() + if err != nil { + t.Fatalf("Authorization: %v", err) + } + if got, want := cfg.Username, ""; got != want { + t.Errorf("Username: got %q, want %q", got, want) + } + if got, want := cfg.IdentityToken, "idtoken"; got != want { + t.Errorf("IdentityToken: got %q, want %q", got, want) + } + if got, want := cfg.Password, ""; got != want { + t.Errorf("Password: got %q, want %q", got, want) + } + }) + + t.Run("failure", func(t *testing.T) { + kc := NewKeychainFromHelper(helper{"", "", errors.New("oh no bad")}) + auth, err := kc.Resolve(repo) + if err != nil { + t.Fatalf("Resolve(%q): %v", repo, err) + } + if auth != Anonymous { + t.Errorf("Resolve: got %v, want %v", auth, Anonymous) + } + }) +} + +func TestConfigFileIsADir(t *testing.T) { + tmpdir := setupConfigDir(t) + // Create "config.json" as a directory, not a file to simulate optional + // secrets in Kubernetes. + err := os.Mkdir(path.Join(tmpdir, "config.json"), 0777) + if err != nil { + t.Fatal(err) + } + + auth, err := DefaultKeychain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + if auth != Anonymous { + t.Errorf("expected Anonymous, got %v", auth) + } +} diff --git a/pkg/authn/kubernetes/go.mod b/pkg/authn/kubernetes/go.mod new file mode 100644 index 0000000..ba2f45c --- /dev/null +++ b/pkg/authn/kubernetes/go.mod @@ -0,0 +1,59 @@ +module github.com/google/go-containerregistry/pkg/authn/kubernetes + +go 1.18 + +replace github.com/google/go-containerregistry => ../../../ + +require ( + github.com/google/go-cmp v0.5.9 + github.com/google/go-containerregistry v0.13.0 + k8s.io/api v0.26.2 + k8s.io/apimachinery v0.26.2 + k8s.io/client-go v0.26.2 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v23.0.1+incompatible // indirect + github.com/docker/docker v23.0.1+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic v0.6.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.29.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.1.0 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect + k8s.io/utils v0.0.0-20230308161112-d77c459e9343 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/pkg/authn/kubernetes/go.sum b/pkg/authn/kubernetes/go.sum new file mode 100644 index 0000000..df8143e --- /dev/null +++ b/pkg/authn/kubernetes/go.sum @@ -0,0 +1,276 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= +gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ= +k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= +k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ= +k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI= +k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/utils v0.0.0-20230308161112-d77c459e9343 h1:m7tbIjXGcGIAtpmQr7/NAi7RsWoW3E7Zcm4jI1HicTc= +k8s.io/utils v0.0.0-20230308161112-d77c459e9343/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/authn/kubernetes/keychain.go b/pkg/authn/kubernetes/keychain.go new file mode 100644 index 0000000..368d829 --- /dev/null +++ b/pkg/authn/kubernetes/keychain.go @@ -0,0 +1,331 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/url" + "path/filepath" + "sort" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + // NoServiceAccount is a constant that can be passed via ServiceAccountName + // to tell the keychain that looking up the service account is unnecessary. + // This value cannot collide with an actual service account name because + // service accounts do not allow spaces. + NoServiceAccount = "no service account" +) + +// Options holds configuration data for guiding credential resolution. +type Options struct { + // Namespace holds the namespace inside of which we are resolving service + // account and pull secret references to access the image. + // If empty, "default" is assumed. + Namespace string + + // ServiceAccountName holds the serviceaccount (within Namespace) as which a + // Pod might access the image. Service accounts may have image pull secrets + // attached, so we lookup the service account to complete the keychain. + // If empty, "default" is assumed. To avoid a service account lookup, pass + // NoServiceAccount explicitly. + ServiceAccountName string + + // ImagePullSecrets holds the names of the Kubernetes secrets (scoped to + // Namespace) containing credential data to use for the image pull. + ImagePullSecrets []string + + // UseMountSecrets determines whether or not mount secrets in the ServiceAccount + // should be considered. Mount secrets are those listed under the `.secrets` + // attribute of the ServiceAccount resource. Ignored if ServiceAccountName is set + // to NoServiceAccount. + UseMountSecrets bool +} + +// New returns a new authn.Keychain suitable for resolving image references as +// scoped by the provided Options. It speaks to Kubernetes through the provided +// client interface. +func New(ctx context.Context, client kubernetes.Interface, opt Options) (authn.Keychain, error) { + if opt.Namespace == "" { + opt.Namespace = "default" + } + if opt.ServiceAccountName == "" { + opt.ServiceAccountName = "default" + } + + // Implement a Kubernetes-style authentication keychain. + // This needs to support roughly the following kinds of authentication: + // 1) The explicit authentication from imagePullSecrets on Pod + // 2) The semi-implicit authentication where imagePullSecrets are on the + // Pod's service account. + + // First, fetch all of the explicitly declared pull secrets + var pullSecrets []corev1.Secret + for _, name := range opt.ImagePullSecrets { + ps, err := client.CoreV1().Secrets(opt.Namespace).Get(ctx, name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + logs.Warn.Printf("secret %s/%s not found; ignoring", opt.Namespace, name) + continue + } else if err != nil { + return nil, err + } + pullSecrets = append(pullSecrets, *ps) + } + + // Second, fetch all of the pull secrets attached to our service account, + // unless the user has explicitly specified that no service account lookup + // is desired. + if opt.ServiceAccountName != NoServiceAccount { + sa, err := client.CoreV1().ServiceAccounts(opt.Namespace).Get(ctx, opt.ServiceAccountName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + logs.Warn.Printf("serviceaccount %s/%s not found; ignoring", opt.Namespace, opt.ServiceAccountName) + } else if err != nil { + return nil, err + } + if sa != nil { + for _, localObj := range sa.ImagePullSecrets { + ps, err := client.CoreV1().Secrets(opt.Namespace).Get(ctx, localObj.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + logs.Warn.Printf("secret %s/%s not found; ignoring", opt.Namespace, localObj.Name) + continue + } else if err != nil { + return nil, err + } + pullSecrets = append(pullSecrets, *ps) + } + + if opt.UseMountSecrets { + for _, obj := range sa.Secrets { + s, err := client.CoreV1().Secrets(opt.Namespace).Get(ctx, obj.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + logs.Warn.Printf("secret %s/%s not found; ignoring", opt.Namespace, obj.Name) + continue + } else if err != nil { + return nil, err + } + pullSecrets = append(pullSecrets, *s) + } + } + } + } + + return NewFromPullSecrets(ctx, pullSecrets) +} + +// NewInCluster returns a new authn.Keychain suitable for resolving image references as +// scoped by the provided Options, constructing a kubernetes.Interface based on in-cluster +// authentication. +func NewInCluster(ctx context.Context, opt Options) (authn.Keychain, error) { + clusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + client, err := kubernetes.NewForConfig(clusterConfig) + if err != nil { + return nil, err + } + return New(ctx, client, opt) +} + +type dockerConfigJSON struct { + Auths map[string]authn.AuthConfig +} + +// NewFromPullSecrets returns a new authn.Keychain suitable for resolving image references as +// scoped by the pull secrets. +func NewFromPullSecrets(ctx context.Context, secrets []corev1.Secret) (authn.Keychain, error) { + keyring := &keyring{ + index: make([]string, 0), + creds: make(map[string][]authn.AuthConfig), + } + + var cfg dockerConfigJSON + + // From: https://github.com/kubernetes/kubernetes/blob/0dcafb1f37ee522be3c045753623138e5b907001/pkg/credentialprovider/keyring.go + for _, secret := range secrets { + if b, exists := secret.Data[corev1.DockerConfigJsonKey]; secret.Type == corev1.SecretTypeDockerConfigJson && exists && len(b) > 0 { + if err := json.Unmarshal(b, &cfg); err != nil { + return nil, err + } + } + if b, exists := secret.Data[corev1.DockerConfigKey]; secret.Type == corev1.SecretTypeDockercfg && exists && len(b) > 0 { + if err := json.Unmarshal(b, &cfg.Auths); err != nil { + return nil, err + } + } + + for registry, v := range cfg.Auths { + value := registry + if !strings.HasPrefix(value, "https://") && !strings.HasPrefix(value, "http://") { + value = "https://" + value + } + parsed, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("Entry %q in dockercfg invalid (%w)", value, err) + } + + // The docker client allows exact matches: + // foo.bar.com/namespace + // Or hostname matches: + // foo.bar.com + // It also considers /v2/ and /v1/ equivalent to the hostname + // See ResolveAuthConfig in docker/registry/auth.go. + effectivePath := parsed.Path + if strings.HasPrefix(effectivePath, "/v2/") || strings.HasPrefix(effectivePath, "/v1/") { + effectivePath = effectivePath[3:] + } + var key string + if (len(effectivePath) > 0) && (effectivePath != "/") { + key = parsed.Host + effectivePath + } else { + key = parsed.Host + } + + if _, ok := keyring.creds[key]; !ok { + keyring.index = append(keyring.index, key) + } + + keyring.creds[key] = append(keyring.creds[key], v) + + } + + // We reverse sort in to give more specific (aka longer) keys priority + // when matching for creds + sort.Sort(sort.Reverse(sort.StringSlice(keyring.index))) + } + return keyring, nil +} + +type keyring struct { + index []string + creds map[string][]authn.AuthConfig +} + +func (keyring *keyring) Resolve(target authn.Resource) (authn.Authenticator, error) { + image := target.String() + auths := []authn.AuthConfig{} + + for _, k := range keyring.index { + // both k and image are schemeless URLs because even though schemes are allowed + // in the credential configurations, we remove them when constructing the keyring + if matched, _ := urlsMatchStr(k, image); matched { + auths = append(auths, keyring.creds[k]...) + } + } + + if len(auths) == 0 { + return authn.Anonymous, nil + } + + return toAuthenticator(auths) +} + +// urlsMatchStr is wrapper for URLsMatch, operating on strings instead of URLs. +func urlsMatchStr(glob string, target string) (bool, error) { + globURL, err := parseSchemelessURL(glob) + if err != nil { + return false, err + } + targetURL, err := parseSchemelessURL(target) + if err != nil { + return false, err + } + return urlsMatch(globURL, targetURL) +} + +// parseSchemelessURL parses a schemeless url and returns a url.URL +// url.Parse require a scheme, but ours don't have schemes. Adding a +// scheme to make url.Parse happy, then clear out the resulting scheme. +func parseSchemelessURL(schemelessURL string) (*url.URL, error) { + parsed, err := url.Parse("https://" + schemelessURL) + if err != nil { + return nil, err + } + // clear out the resulting scheme + parsed.Scheme = "" + return parsed, nil +} + +// splitURL splits the host name into parts, as well as the port +func splitURL(url *url.URL) (parts []string, port string) { + host, port, err := net.SplitHostPort(url.Host) + if err != nil { + // could not parse port + host, port = url.Host, "" + } + return strings.Split(host, "."), port +} + +// urlsMatch checks whether the given target url matches the glob url, which may have +// glob wild cards in the host name. +// +// Examples: +// +// globURL=*.docker.io, targetURL=blah.docker.io => match +// globURL=*.docker.io, targetURL=not.right.io => no match +// +// Note that we don't support wildcards in ports and paths yet. +func urlsMatch(globURL *url.URL, targetURL *url.URL) (bool, error) { + globURLParts, globPort := splitURL(globURL) + targetURLParts, targetPort := splitURL(targetURL) + if globPort != targetPort { + // port doesn't match + return false, nil + } + if len(globURLParts) != len(targetURLParts) { + // host name does not have the same number of parts + return false, nil + } + if !strings.HasPrefix(targetURL.Path, globURL.Path) { + // the path of the credential must be a prefix + return false, nil + } + for k, globURLPart := range globURLParts { + targetURLPart := targetURLParts[k] + matched, err := filepath.Match(globURLPart, targetURLPart) + if err != nil { + return false, err + } + if !matched { + // glob mismatch for some part + return false, nil + } + } + // everything matches + return true, nil +} + +func toAuthenticator(configs []authn.AuthConfig) (authn.Authenticator, error) { + cfg := configs[0] + + if cfg.Auth != "" { + cfg.Auth = "" + } + + return authn.FromConfig(cfg), nil +} diff --git a/pkg/authn/kubernetes/keychain_test.go b/pkg/authn/kubernetes/keychain_test.go new file mode 100644 index 0000000..e015771 --- /dev/null +++ b/pkg/authn/kubernetes/keychain_test.go @@ -0,0 +1,586 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + "crypto/md5" + "encoding/base64" + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + fakeclient "k8s.io/client-go/kubernetes/fake" +) + +var dockerSecretTypes = []secretType{ + dockerConfigJSONSecretType, + dockerCfgSecretType, +} + +type secretType struct { + name corev1.SecretType + key string + marshal func(t *testing.T, registry string, auth authn.AuthConfig) []byte +} + +func (s *secretType) Create(t *testing.T, namespace, name string, registry string, auth authn.AuthConfig) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: s.name, + Data: map[string][]byte{ + s.key: s.marshal(t, registry, auth), + }, + } +} + +var dockerConfigJSONSecretType = secretType{ + name: corev1.SecretTypeDockerConfigJson, + key: corev1.DockerConfigJsonKey, + marshal: func(t *testing.T, target string, auth authn.AuthConfig) []byte { + return toJSON(t, dockerConfigJSON{ + Auths: map[string]authn.AuthConfig{target: auth}, + }) + }, +} + +var dockerCfgSecretType = secretType{ + name: corev1.SecretTypeDockercfg, + key: corev1.DockerConfigKey, + marshal: func(t *testing.T, target string, auth authn.AuthConfig) []byte { + return toJSON(t, map[string]authn.AuthConfig{target: auth}) + }, +} + +func TestAnonymousFallback(t *testing.T) { + client := fakeclient.NewSimpleClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + }) + + kc, err := New(context.Background(), client, Options{}) + if err != nil { + t.Errorf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.Anonymous) +} + +func TestAnonymousFallbackNoServiceAccount(t *testing.T) { + kc, err := New(context.Background(), nil, Options{ + ServiceAccountName: NoServiceAccount, + }) + if err != nil { + t.Errorf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.Anonymous) +} + +func TestSecretNotFound(t *testing.T) { + client := fakeclient.NewSimpleClientset() + + kc, err := New(context.Background(), client, Options{ + ServiceAccountName: NoServiceAccount, + ImagePullSecrets: []string{"not-found"}, + }) + if err != nil { + t.Errorf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.Anonymous) +} + +func TestServiceAccountNotFound(t *testing.T) { + client := fakeclient.NewSimpleClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + }) + kc, err := New(context.Background(), client, Options{ + ServiceAccountName: "not-found", + }) + if err != nil { + t.Errorf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.Anonymous) +} + +func TestImagePullSecretAttachedServiceAccount(t *testing.T) { + username, password := "foo", "bar" + client := fakeclient.NewSimpleClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svcacct", + Namespace: "ns", + }, + ImagePullSecrets: []corev1.LocalObjectReference{{ + Name: "secret", + }}, + }, + dockerCfgSecretType.Create(t, "ns", "secret", "fake.registry.io", authn.AuthConfig{ + Username: username, + Password: password, + }), + ) + + kc, err := New(context.Background(), client, Options{ + Namespace: "ns", + ServiceAccountName: "svcacct", + }) + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), + &authn.Basic{Username: username, Password: password}) +} + +func TestSecretAttachedServiceAccount(t *testing.T) { + username, password := "foo", "bar" + + cases := []struct { + name string + createSecret bool + useMountSecrets bool + expected authn.Authenticator + }{ + { + name: "resolved successfully", + createSecret: true, + useMountSecrets: true, + expected: &authn.Basic{Username: username, Password: password}, + }, + { + name: "missing secret skipped", + createSecret: false, + useMountSecrets: true, + expected: &authn.Basic{}, + }, + { + name: "skip option", + createSecret: true, + useMountSecrets: false, + expected: &authn.Basic{}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + objs := []runtime.Object{ + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svcacct", + Namespace: "ns", + }, + Secrets: []corev1.ObjectReference{{ + Name: "secret", + }}, + }, + } + if c.createSecret { + objs = append(objs, dockerCfgSecretType.Create( + t, "ns", "secret", "fake.registry.io", authn.AuthConfig{ + Username: username, + Password: password, + })) + } + client := fakeclient.NewSimpleClientset(objs...) + + kc, err := New(context.Background(), client, Options{ + Namespace: "ns", + ServiceAccountName: "svcacct", + UseMountSecrets: c.useMountSecrets, + }) + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), c.expected) + }) + } + +} + +// Prioritze picking the first secret +func TestSecretPriority(t *testing.T) { + secrets := []corev1.Secret{ + *dockerCfgSecretType.Create(t, "ns", "secret", "fake.registry.io", authn.AuthConfig{ + Username: "user", Password: "pass", + }), + *dockerCfgSecretType.Create(t, "ns", "secret-2", "fake.registry.io", authn.AuthConfig{ + Username: "anotherUser", Password: "anotherPass", + }), + } + + kc, err := NewFromPullSecrets(context.Background(), secrets) + if err != nil { + t.Fatalf("NewFromPullSecrets() = %v", err) + } + + expectedAuth := &authn.Basic{Username: "user", Password: "pass"} + testResolve(t, kc, registry(t, "fake.registry.io"), expectedAuth) +} + +func TestResolveTargets(t *testing.T) { + // Iterate over target types + targetTypes := []authn.Resource{ + registry(t, "fake.registry.io"), + repo(t, "fake.registry.io/repo"), + } + + for _, secretType := range dockerSecretTypes { + for _, target := range targetTypes { + // Drop the . + testName := secretType.key[1:] + "_" + target.String() + + t.Run(testName, func(t *testing.T) { + auth := authn.AuthConfig{ + Password: fmt.Sprintf("%x", md5.Sum([]byte(t.Name()))), + Username: "user" + fmt.Sprintf("%x", md5.Sum([]byte(t.Name()))), + } + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *secretType.Create(t, "ns", "secret", target.String(), auth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + authenticator := &authn.Basic{Username: auth.Username, Password: auth.Password} + testResolve(t, kc, target, authenticator) + }) + } + } +} + +func TestAuthWithScheme(t *testing.T) { + auth := authn.AuthConfig{ + Password: "password", + Username: "username", + } + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret", "https://fake.registry.io", auth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + authenticator := &authn.Basic{Username: auth.Username, Password: auth.Password} + testResolve(t, kc, registry(t, "fake.registry.io"), authenticator) + testResolve(t, kc, repo(t, "fake.registry.io/repo"), authenticator) +} + +func TestAuthWithPorts(t *testing.T) { + auth := authn.AuthConfig{ + Password: "password", + Username: "username", + } + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret", "fake.registry.io:5000", auth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + authenticator := &authn.Basic{Username: auth.Username, Password: auth.Password} + testResolve(t, kc, registry(t, "fake.registry.io:5000"), authenticator) + testResolve(t, kc, repo(t, "fake.registry.io:5000/repo"), authenticator) + + // Non-matching ports should return Anonymous + testResolve(t, kc, registry(t, "fake.registry.io:1000"), authn.Anonymous) + testResolve(t, kc, repo(t, "fake.registry.io:1000/repo"), authn.Anonymous) +} + +func TestAuthPathMatching(t *testing.T) { + rootAuth := authn.AuthConfig{Username: "root", Password: "root"} + nestedAuth := authn.AuthConfig{Username: "nested", Password: "nested"} + leafAuth := authn.AuthConfig{Username: "leaf", Password: "leaf"} + partialAuth := authn.AuthConfig{Username: "partial", Password: "partial"} + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret-1", "fake.registry.io", rootAuth), + *dockerConfigJSONSecretType.Create(t, "ns", "secret-2", "fake.registry.io/nested", nestedAuth), + *dockerConfigJSONSecretType.Create(t, "ns", "secret-3", "fake.registry.io/nested/repo", leafAuth), + *dockerConfigJSONSecretType.Create(t, "ns", "secret-4", "fake.registry.io/par", partialAuth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + testResolve(t, kc, registry(t, "fake.registry.io"), authn.FromConfig(rootAuth)) + testResolve(t, kc, repo(t, "fake.registry.io/nested"), authn.FromConfig(nestedAuth)) + testResolve(t, kc, repo(t, "fake.registry.io/nested/repo"), authn.FromConfig(leafAuth)) + testResolve(t, kc, repo(t, "fake.registry.io/nested/repo/dirt"), authn.FromConfig(leafAuth)) + testResolve(t, kc, repo(t, "fake.registry.io/partial"), authn.FromConfig(partialAuth)) +} + +func TestAuthHostNameVariations(t *testing.T) { + rootAuth := authn.AuthConfig{Username: "root", Password: "root"} + subdomainAuth := authn.AuthConfig{Username: "sub", Password: "sub"} + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret-1", "fake.registry.io", rootAuth), + *dockerConfigJSONSecretType.Create(t, "ns", "secret-2", "1.fake.registry.io", subdomainAuth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.FromConfig(rootAuth)) + testResolve(t, kc, registry(t, "1.fake.registry.io"), authn.FromConfig(subdomainAuth)) + + // Unrecognized subdomain uses Anonymous + testResolve(t, kc, registry(t, "2.fake.registry.io"), authn.Anonymous) +} + +func TestAuthSpecialPathsIgnored(t *testing.T) { + auth := authn.AuthConfig{Username: "root", Password: "root"} + auth2 := authn.AuthConfig{Username: "root2", Password: "root2"} + + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + // Note the paths need a trailing '/' + *dockerConfigJSONSecretType.Create(t, "ns", "secret-1", "https://fake.registry.io/v1/", auth), + *dockerConfigJSONSecretType.Create(t, "ns", "secret-2", "https://fake2.registry.io/v2/", auth2), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.FromConfig(auth)) + testResolve(t, kc, repo(t, "fake.registry.io/repo"), authn.FromConfig(auth)) + testResolve(t, kc, registry(t, "fake2.registry.io"), authn.FromConfig(auth2)) + testResolve(t, kc, repo(t, "fake2.registry.io/repo"), authn.FromConfig(auth2)) +} + +func TestAuthDockerRegistry(t *testing.T) { + auth := authn.AuthConfig{Username: "root", Password: "root"} + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret", "index.docker.io", auth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, repo(t, "ubuntu"), authn.FromConfig(auth)) + testResolve(t, kc, repo(t, "knative/serving"), authn.FromConfig(auth)) +} + +func TestAuthWithGlobs(t *testing.T) { + auth := authn.AuthConfig{Username: "root", Password: "root"} + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{ + *dockerConfigJSONSecretType.Create(t, "ns", "secret", "*.registry.io", auth), + }) + + if err != nil { + t.Fatalf("New() = %v", err) + } + + testResolve(t, kc, registry(t, "fake.registry.io"), authn.FromConfig(auth)) + testResolve(t, kc, repo(t, "fake.registry.io/repo"), authn.FromConfig(auth)) + testResolve(t, kc, registry(t, "blah.registry.io"), authn.FromConfig(auth)) + testResolve(t, kc, repo(t, "blah.registry.io/repo"), authn.FromConfig(auth)) +} + +func testResolve(t *testing.T, kc authn.Keychain, target authn.Resource, expectedAuth authn.Authenticator) { + t.Helper() + + auth, err := kc.Resolve(target) + if err != nil { + t.Errorf("Resolve(%v) = %v", target, err) + } + got, err := auth.Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + want, err := expectedAuth.Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Error("Resolve() diff (-want, +got)\n", diff) + } +} + +func toJSON(t *testing.T, obj any) []byte { + t.Helper() + + bites, err := json.Marshal(obj) + + if err != nil { + t.Fatal("unable to json marshal", err) + } + return bites +} + +func registry(t *testing.T, registry string) authn.Resource { + t.Helper() + + reg, err := name.NewRegistry(registry, name.WeakValidation) + if err != nil { + t.Fatal("failed to create registry", err) + } + return reg +} + +func repo(t *testing.T, repository string) authn.Resource { + t.Helper() + + repo, err := name.NewRepository(repository, name.WeakValidation) + if err != nil { + t.Fatal("failed to create repo", err) + } + return repo +} + +// TestDockerConfigJSON tests using secrets using the .dockerconfigjson form, +// like you might get from running: +// kubectl create secret docker-registry secret -n ns --docker-server="fake.registry.io" --docker-username="foo" --docker-password="bar" +func TestDockerConfigJSON(t *testing.T) { + username, password := "foo", "bar" + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "ns", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte( + fmt.Sprintf(`{"auths":{"fake.registry.io":{"username":%q,"password":%q,"auth":%q}}}`, + username, password, + base64.StdEncoding.EncodeToString([]byte(username+":"+password))), + ), + }, + }}) + if err != nil { + t.Fatalf("NewFromPullSecrets() = %v", err) + } + + reg, err := name.NewRegistry("fake.registry.io", name.WeakValidation) + if err != nil { + t.Errorf("NewRegistry() = %v", err) + } + + auth, err := kc.Resolve(reg) + if err != nil { + t.Errorf("Resolve(%v) = %v", reg, err) + } + got, err := auth.Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + want, err := (&authn.Basic{Username: username, Password: password}).Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Resolve() = %v, want %v", got, want) + } +} + +func TestKubernetesAuth(t *testing.T) { + // From https://github.com/knative/serving/issues/12761#issuecomment-1097441770 + // All of these should work with K8s' docker auth parsing. + for k, ss := range map[string][]string{ + "registry.gitlab.com/dprotaso/test/nginx": { + "registry.gitlab.com", + "http://registry.gitlab.com", + "https://registry.gitlab.com", + "registry.gitlab.com/dprotaso", + "http://registry.gitlab.com/dprotaso", + "https://registry.gitlab.com/dprotaso", + "registry.gitlab.com/dprotaso/test", + "http://registry.gitlab.com/dprotaso/test", + "https://registry.gitlab.com/dprotaso/test", + "registry.gitlab.com/dprotaso/test/nginx", + "http://registry.gitlab.com/dprotaso/test/nginx", + "https://registry.gitlab.com/dprotaso/test/nginx", + }, + "dtestcontainer.azurecr.io/dave/nginx": { + "dtestcontainer.azurecr.io", + "http://dtestcontainer.azurecr.io", + "https://dtestcontainer.azurecr.io", + "dtestcontainer.azurecr.io/dave", + "http://dtestcontainer.azurecr.io/dave", + "https://dtestcontainer.azurecr.io/dave", + "dtestcontainer.azurecr.io/dave/nginx", + "http://dtestcontainer.azurecr.io/dave/nginx", + "https://dtestcontainer.azurecr.io/dave/nginx", + }} { + repo, err := name.NewRepository(k) + if err != nil { + t.Errorf("parsing %q: %v", k, err) + continue + } + + for _, s := range ss { + t.Run(fmt.Sprintf("%s - %s", k, s), func(t *testing.T) { + username, password := "foo", "bar" + kc, err := NewFromPullSecrets(context.Background(), []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "ns", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte( + fmt.Sprintf(`{"auths":{%q:{"username":%q,"password":%q,"auth":%q}}}`, + s, + username, password, + base64.StdEncoding.EncodeToString([]byte(username+":"+password))), + ), + }, + }}) + if err != nil { + t.Fatalf("NewFromPullSecrets() = %v", err) + } + auth, err := kc.Resolve(repo) + if err != nil { + t.Errorf("Resolve(%v) = %v", repo, err) + } + got, err := auth.Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + want, err := (&authn.Basic{Username: username, Password: password}).Authorization() + if err != nil { + t.Errorf("Authorization() = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Resolve() = %v, want %v", got, want) + } + }) + } + } +} diff --git a/pkg/authn/multikeychain.go b/pkg/authn/multikeychain.go new file mode 100644 index 0000000..3b1804f --- /dev/null +++ b/pkg/authn/multikeychain.go @@ -0,0 +1,41 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +type multiKeychain struct { + keychains []Keychain +} + +// Assert that our multi-keychain implements Keychain. +var _ (Keychain) = (*multiKeychain)(nil) + +// NewMultiKeychain composes a list of keychains into one new keychain. +func NewMultiKeychain(kcs ...Keychain) Keychain { + return &multiKeychain{keychains: kcs} +} + +// Resolve implements Keychain. +func (mk *multiKeychain) Resolve(target Resource) (Authenticator, error) { + for _, kc := range mk.keychains { + auth, err := kc.Resolve(target) + if err != nil { + return nil, err + } + if auth != Anonymous { + return auth, nil + } + } + return Anonymous, nil +} diff --git a/pkg/authn/multikeychain_test.go b/pkg/authn/multikeychain_test.go new file mode 100644 index 0000000..3dff0c9 --- /dev/null +++ b/pkg/authn/multikeychain_test.go @@ -0,0 +1,98 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package authn + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" +) + +func TestMultiKeychain(t *testing.T) { + one := &Basic{Username: "one", Password: "secret"} + two := &Basic{Username: "two", Password: "secret"} + three := &Basic{Username: "three", Password: "secret"} + + regOne, _ := name.NewRegistry("one.gcr.io", name.StrictValidation) + regTwo, _ := name.NewRegistry("two.gcr.io", name.StrictValidation) + regThree, _ := name.NewRegistry("three.gcr.io", name.StrictValidation) + + tests := []struct { + name string + reg name.Registry + kc Keychain + want Authenticator + }{{ + // Make sure our test keychain WAI + name: "simple fixed test (match)", + reg: regOne, + kc: fixedKeychain{regOne: one}, + want: one, + }, { + // Make sure our test keychain WAI + name: "simple fixed test (no match)", + reg: regTwo, + kc: fixedKeychain{regOne: one}, + want: Anonymous, + }, { + name: "match first keychain", + reg: regOne, + kc: NewMultiKeychain( + fixedKeychain{regOne: one}, + fixedKeychain{regOne: three, regTwo: two}, + ), + want: one, + }, { + name: "match second keychain", + reg: regTwo, + kc: NewMultiKeychain( + fixedKeychain{regOne: one}, + fixedKeychain{regOne: three, regTwo: two}, + ), + want: two, + }, { + name: "match no keychain", + reg: regThree, + kc: NewMultiKeychain( + fixedKeychain{regOne: one}, + fixedKeychain{regOne: three, regTwo: two}, + ), + want: Anonymous, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.kc.Resolve(test.reg) + if err != nil { + t.Errorf("Resolve() = %v", err) + } + if got != test.want { + t.Errorf("Resolve() = %v, wanted %v", got, test.want) + } + }) + } +} + +type fixedKeychain map[Resource]Authenticator + +var _ Keychain = (fixedKeychain)(nil) + +// Resolve implements Keychain. +func (fk fixedKeychain) Resolve(target Resource) (Authenticator, error) { + if auth, ok := fk[target]; ok { + return auth, nil + } + return Anonymous, nil +} diff --git a/pkg/compression/compression.go b/pkg/compression/compression.go new file mode 100644 index 0000000..6686c2d --- /dev/null +++ b/pkg/compression/compression.go @@ -0,0 +1,26 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compression abstracts over gzip and zstd. +package compression + +// Compression is an enumeration of the supported compression algorithms +type Compression string + +// The collection of known MediaType values. +const ( + None Compression = "none" + GZip Compression = "gzip" + ZStd Compression = "zstd" +) diff --git a/pkg/crane/append.go b/pkg/crane/append.go new file mode 100644 index 0000000..f1c2ef6 --- /dev/null +++ b/pkg/crane/append.go @@ -0,0 +1,114 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/internal/windows" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func isWindows(img v1.Image) (bool, error) { + cfg, err := img.ConfigFile() + if err != nil { + return false, err + } + return cfg != nil && cfg.OS == "windows", nil +} + +// Append reads a layer from path and appends it the the v1.Image base. +// +// If the base image is a Windows base image (i.e., its config.OS is +// "windows"), the contents of the tarballs will be modified to be suitable for +// a Windows container image.`, +func Append(base v1.Image, paths ...string) (v1.Image, error) { + if base == nil { + return nil, fmt.Errorf("invalid argument: base") + } + + win, err := isWindows(base) + if err != nil { + return nil, fmt.Errorf("getting base image: %w", err) + } + + baseMediaType, err := base.MediaType() + + if err != nil { + return nil, fmt.Errorf("getting base image media type: %w", err) + } + + layerType := types.DockerLayer + + if baseMediaType == types.OCIManifestSchema1 { + layerType = types.OCILayer + } + + layers := make([]v1.Layer, 0, len(paths)) + for _, path := range paths { + layer, err := getLayer(path, layerType) + if err != nil { + return nil, fmt.Errorf("reading layer %q: %w", path, err) + } + + if win { + layer, err = windows.Windows(layer) + if err != nil { + return nil, fmt.Errorf("converting %q for Windows: %w", path, err) + } + } + + layers = append(layers, layer) + } + + return mutate.AppendLayers(base, layers...) +} + +func getLayer(path string, layerType types.MediaType) (v1.Layer, error) { + f, err := streamFile(path) + if err != nil { + return nil, err + } + if f != nil { + return stream.NewLayer(f, stream.WithMediaType(layerType)), nil + } + + return tarball.LayerFromFile(path, tarball.WithMediaType(layerType)) +} + +// If we're dealing with a named pipe, trying to open it multiple times will +// fail, so we need to do a streaming upload. +// +// returns nil, nil for non-streaming files +func streamFile(path string) (*os.File, error) { + if path == "-" { + return os.Stdin, nil + } + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + + if !fi.Mode().IsRegular() { + return os.Open(path) + } + + return nil, nil +} diff --git a/pkg/crane/append_test.go b/pkg/crane/append_test.go new file mode 100644 index 0000000..d894a9d --- /dev/null +++ b/pkg/crane/append_test.go @@ -0,0 +1,73 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane_test + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestAppendWithOCIBaseImage(t *testing.T) { + base := mutate.MediaType(empty.Image, types.OCIManifestSchema1) + img, err := crane.Append(base, "testdata/content.tar") + + if err != nil { + t.Fatalf("crane.Append(): %v", err) + } + + layers, err := img.Layers() + + if err != nil { + t.Fatalf("img.Layers(): %v", err) + } + + mediaType, err := layers[0].MediaType() + + if err != nil { + t.Fatalf("layers[0].MediaType(): %v", err) + } + + if got, want := mediaType, types.OCILayer; got != want { + t.Errorf("MediaType(): want %q, got %q", want, got) + } +} + +func TestAppendWithDockerBaseImage(t *testing.T) { + img, err := crane.Append(empty.Image, "testdata/content.tar") + + if err != nil { + t.Fatalf("crane.Append(): %v", err) + } + + layers, err := img.Layers() + + if err != nil { + t.Fatalf("img.Layers(): %v", err) + } + + mediaType, err := layers[0].MediaType() + + if err != nil { + t.Fatalf("layers[0].MediaType(): %v", err) + } + + if got, want := mediaType, types.DockerLayer; got != want { + t.Errorf("MediaType(): want %q, got %q", want, got) + } +} diff --git a/pkg/crane/catalog.go b/pkg/crane/catalog.go new file mode 100644 index 0000000..f30800c --- /dev/null +++ b/pkg/crane/catalog.go @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "context" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Catalog returns the repositories in a registry's catalog. +func Catalog(src string, opt ...Option) (res []string, err error) { + o := makeOptions(opt...) + reg, err := name.NewRegistry(src, o.Name...) + if err != nil { + return nil, err + } + + // This context gets overridden by remote.WithContext, which is set by + // crane.WithContext. + return remote.Catalog(context.Background(), reg, o.Remote...) +} diff --git a/pkg/crane/config.go b/pkg/crane/config.go new file mode 100644 index 0000000..3e55cc9 --- /dev/null +++ b/pkg/crane/config.go @@ -0,0 +1,24 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +// Config returns the config file for the remote image ref. +func Config(ref string, opt ...Option) ([]byte, error) { + i, _, err := getImage(ref, opt...) + if err != nil { + return nil, err + } + return i.RawConfigFile() +} diff --git a/pkg/crane/copy.go b/pkg/crane/copy.go new file mode 100644 index 0000000..a606f96 --- /dev/null +++ b/pkg/crane/copy.go @@ -0,0 +1,88 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/internal/legacy" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Copy copies a remote image or index from src to dst. +func Copy(src, dst string, opt ...Option) error { + o := makeOptions(opt...) + srcRef, err := name.ParseReference(src, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", src, err) + } + + dstRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference for %q: %w", dst, err) + } + + logs.Progress.Printf("Copying from %v to %v", srcRef, dstRef) + desc, err := remote.Get(srcRef, o.Remote...) + if err != nil { + return fmt.Errorf("fetching %q: %w", src, err) + } + + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + // Handle indexes separately. + if o.Platform != nil { + // If platform is explicitly set, don't copy the whole index, just the appropriate image. + if err := copyImage(desc, dstRef, o); err != nil { + return fmt.Errorf("failed to copy image: %w", err) + } + } else { + if err := copyIndex(desc, dstRef, o); err != nil { + return fmt.Errorf("failed to copy index: %w", err) + } + } + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // Handle schema 1 images separately. + if err := legacy.CopySchema1(desc, srcRef, dstRef, o.Remote...); err != nil { + return fmt.Errorf("failed to copy schema 1 image: %w", err) + } + default: + // Assume anything else is an image, since some registries don't set mediaTypes properly. + if err := copyImage(desc, dstRef, o); err != nil { + return fmt.Errorf("failed to copy image: %w", err) + } + } + + return nil +} + +func copyImage(desc *remote.Descriptor, dstRef name.Reference, o Options) error { + img, err := desc.Image() + if err != nil { + return err + } + return remote.Write(dstRef, img, o.Remote...) +} + +func copyIndex(desc *remote.Descriptor, dstRef name.Reference, o Options) error { + idx, err := desc.ImageIndex() + if err != nil { + return err + } + return remote.WriteIndex(dstRef, idx, o.Remote...) +} diff --git a/pkg/crane/crane_test.go b/pkg/crane/crane_test.go new file mode 100644 index 0000000..9f4d124 --- /dev/null +++ b/pkg/crane/crane_test.go @@ -0,0 +1,574 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane_test + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "strings" + "testing" + + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// TODO(jonjohnsonjr): Test crane.Copy failures. +func TestCraneRegistry(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + src := fmt.Sprintf("%s/test/crane", u.Host) + dst := fmt.Sprintf("%s/test/crane/copy", u.Host) + + // Expected values. + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + digest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + rawManifest, err := img.RawManifest() + if err != nil { + t.Fatal(err) + } + manifest, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + config, err := img.RawConfigFile() + if err != nil { + t.Fatal(err) + } + layer, err := img.LayerByDigest(manifest.Layers[0].Digest) + if err != nil { + t.Fatal(err) + } + + // Load up the registry. + if err := crane.Push(img, src); err != nil { + t.Fatal(err) + } + + // Test that `crane.Foo` returns expected values. + d, err := crane.Digest(src) + if err != nil { + t.Error(err) + } else if d != digest.String() { + t.Errorf("Digest(): %v != %v", d, digest) + } + + m, err := crane.Manifest(src) + if err != nil { + t.Error(err) + } else if string(m) != string(rawManifest) { + t.Errorf("Manifest(): %v != %v", m, rawManifest) + } + + c, err := crane.Config(src) + if err != nil { + t.Error(err) + } else if string(c) != string(config) { + t.Errorf("Config(): %v != %v", c, config) + } + + // Make sure we pull what we pushed. + pulled, err := crane.Pull(src) + if err != nil { + t.Error(err) + } + if err := compare.Images(img, pulled); err != nil { + t.Fatal(err) + } + + // Test that the copied image is the same as the source. + if err := crane.Copy(src, dst); err != nil { + t.Fatal(err) + } + + // Make sure what we copied is equivalent. + // Also, get options coverage in a dumb way. + copied, err := crane.Pull(dst, crane.Insecure, crane.WithTransport(http.DefaultTransport), crane.WithAuth(authn.Anonymous), crane.WithAuthFromKeychain(authn.DefaultKeychain), crane.WithUserAgent("crane/tests")) + if err != nil { + t.Fatal(err) + } + if err := compare.Images(pulled, copied); err != nil { + t.Fatal(err) + } + + if err := crane.Tag(dst, "crane-tag"); err != nil { + t.Fatal(err) + } + + // Make sure what we tagged is equivalent. + tagged, err := crane.Pull(fmt.Sprintf("%s:%s", dst, "crane-tag")) + if err != nil { + t.Fatal(err) + } + if err := compare.Images(pulled, tagged); err != nil { + t.Fatal(err) + } + + layerRef := fmt.Sprintf("%s/test/crane@%s", u.Host, manifest.Layers[0].Digest) + pulledLayer, err := crane.PullLayer(layerRef) + if err != nil { + t.Fatal(err) + } + + if err := compare.Layers(pulledLayer, layer); err != nil { + t.Fatal(err) + } + + // List Tags + // dst variable have: latest and crane-tag + tags, err := crane.ListTags(dst) + if err != nil { + t.Fatal(err) + } + if len(tags) != 2 { + t.Fatalf("wanted 2 tags, got %d", len(tags)) + } + + // create 4 tags for dst + for i := 1; i < 5; i++ { + if err := crane.Tag(dst, fmt.Sprintf("honk-tag-%d", i)); err != nil { + t.Fatal(err) + } + } + + tags, err = crane.ListTags(dst) + if err != nil { + t.Fatal(err) + } + if len(tags) != 6 { + t.Fatalf("wanted 6 tags, got %d", len(tags)) + } + + // Delete the non existing image + if err := crane.Delete(dst + ":honk-image"); err == nil { + t.Fatal("wanted err, got nil") + } + + // Delete the image + if err := crane.Delete(src); err != nil { + t.Fatal(err) + } + + // check if the image was really deleted + if _, err := crane.Pull(src); err == nil { + t.Fatal("wanted err, got nil") + } + + // check if the copied image still exist + dstPulled, err := crane.Pull(dst) + if err != nil { + t.Fatal(err) + } + if err := compare.Images(dstPulled, copied); err != nil { + t.Fatal(err) + } + + // List Catalog + repos, err := crane.Catalog(u.Host) + if err != nil { + t.Fatal(err) + } + if len(repos) != 2 { + t.Fatalf("wanted 2 repos, got %d", len(repos)) + } + + // Test pushing layer + layer, err = img.LayerByDigest(manifest.Layers[1].Digest) + if err != nil { + t.Fatal(err) + } + if err := crane.Upload(layer, dst); err != nil { + t.Fatal(err) + } +} + +func TestCraneCopyIndex(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + src := fmt.Sprintf("%s/test/crane", u.Host) + dst := fmt.Sprintf("%s/test/crane/copy", u.Host) + + // Load up the registry. + idx, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + ref, err := name.ParseReference(src) + if err != nil { + t.Fatal(err) + } + if err := remote.WriteIndex(ref, idx); err != nil { + t.Fatal(err) + } + + // Test that the copied index is the same as the source. + if err := crane.Copy(src, dst); err != nil { + t.Fatal(err) + } + + d, err := crane.Digest(src) + if err != nil { + t.Fatal(err) + } + cp, err := crane.Digest(dst) + if err != nil { + t.Fatal(err) + } + if d != cp { + t.Errorf("Copied Digest(): %v != %v", d, cp) + } +} + +func TestWithPlatform(t *testing.T) { + // Set up a fake registry with a platform-specific image. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + imgs := []mutate.IndexAddendum{} + for _, plat := range []string{ + "linux/amd64", + "linux/arm", + } { + img, err := crane.Image(map[string][]byte{ + "platform.txt": []byte(plat), + }) + if err != nil { + t.Fatal(err) + } + parts := strings.Split(plat, "/") + imgs = append(imgs, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{ + Platform: &v1.Platform{ + OS: parts[0], + Architecture: parts[1], + }, + }, + }) + } + + idx := mutate.AppendManifests(empty.Index, imgs...) + + src := path.Join(u.Host, "src") + dst := path.Join(u.Host, "dst") + + ref, err := name.ParseReference(src) + if err != nil { + t.Fatal(err) + } + + // Populate registry so we can copy from it. + if err := remote.WriteIndex(ref, idx); err != nil { + t.Fatal(err) + } + + if err := crane.Copy(src, dst, crane.WithPlatform(imgs[1].Platform)); err != nil { + t.Fatal(err) + } + + want, err := crane.Manifest(src, crane.WithPlatform(imgs[1].Platform)) + if err != nil { + t.Fatal(err) + } + got, err := crane.Manifest(dst) + if err != nil { + t.Fatal(err) + } + + if string(got) != string(want) { + t.Errorf("Manifest(%q) != Manifest(%q): (\n\n%s\n\n!=\n\n%s\n\n)", dst, src, string(got), string(want)) + } + + arch := "real fake doors" + + // Now do a fake platform, should fail + if _, err := crane.Manifest(src, crane.WithPlatform(&v1.Platform{ + OS: "does-not-exist", + Architecture: arch, + })); err == nil { + t.Error("crane.Manifest(fake platform): got nil want err") + } else if !strings.Contains(err.Error(), arch) { + t.Errorf("crane.Manifest(fake platform): expected %q in error, got: %v", arch, err) + } +} + +func TestCraneTarball(t *testing.T) { + t.Parallel() + // Write an image as a tarball. + tmp, err := os.CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmp.Name()) + + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + digest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + src := fmt.Sprintf("test/crane@%s", digest) + + if err := crane.Save(img, src, tmp.Name()); err != nil { + t.Errorf("Save: %v", err) + } + + // Make sure the image we load has a matching digest. + img, err = crane.Load(tmp.Name()) + if err != nil { + t.Fatal(err) + } + + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + if d != digest { + t.Errorf("digest mismatch: %v != %v", d, digest) + } +} + +func TestCraneSaveLegacy(t *testing.T) { + t.Parallel() + // Write an image as a legacy tarball. + tmp, err := os.CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmp.Name()) + + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + + if err := crane.SaveLegacy(img, "test/crane", tmp.Name()); err != nil { + t.Errorf("SaveOCI: %v", err) + } +} + +func TestCraneSaveOCI(t *testing.T) { + t.Parallel() + // Write an image as an OCI image layout. + tmp := t.TempDir() + + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + if err := crane.SaveOCI(img, tmp); err != nil { + t.Errorf("SaveLegacy: %v", err) + } +} + +func TestCraneFilesystem(t *testing.T) { + t.Parallel() + tmp, err := os.CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + + name := "/some/file" + content := []byte("sentinel") + + tw := tar.NewWriter(tmp) + if err := tw.WriteHeader(&tar.Header{ + Size: int64(len(content)), + Name: name, + }); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(content); err != nil { + t.Fatal(err) + } + tw.Flush() + tw.Close() + + img, err = crane.Append(img, tmp.Name()) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := crane.Export(img, &buf); err != nil { + t.Fatal(err) + } + + tr := tar.NewReader(&buf) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + t.Fatalf("didn't find find") + } else if err != nil { + t.Fatal(err) + } + if header.Name == name { + b, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + if string(b) != string(content) { + t.Fatalf("got back wrong content: %v != %v", string(b), string(content)) + } + break + } + } +} + +func TestStreamingAppend(t *testing.T) { + // Stdin will be an uncompressed layer. + layer, err := crane.Layer(map[string][]byte{ + "hello": []byte(`world`), + }) + if err != nil { + t.Fatal(err) + } + rc, err := layer.Uncompressed() + if err != nil { + t.Fatal(err) + } + + tmp, err := os.CreateTemp("", "crane-append") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmp.Name()) + + if _, err := io.Copy(tmp, rc); err != nil { + t.Fatal(err) + } + + stdin := os.Stdin + defer func() { + os.Stdin = stdin + }() + + os.Stdin = tmp + + img, err := crane.Append(empty.Image, "-") + if err != nil { + t.Fatal(err) + } + ll, err := img.Layers() + if err != nil { + t.Fatal(err) + } + if want, got := 1, len(ll); want != got { + t.Errorf("crane.Append(stdin) - len(layers): want %d != got %d", want, got) + } +} + +func TestBadInputs(t *testing.T) { + t.Parallel() + invalid := "/dev/null/@@@@@@" + + // Create a valid image reference that will fail with not found. + s := httptest.NewServer(http.NotFoundHandler()) + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + valid404 := fmt.Sprintf("%s/some/image", u.Host) + + // e drops the first parameter so we can use the result of a function + // that returns two values as an expression above. This is a bit of a go quirk. + e := func(_ any, err error) error { + return err + } + + for _, tc := range []struct { + desc string + err error + }{ + {"Push(_, invalid)", crane.Push(nil, invalid)}, + {"Upload(_, invalid)", crane.Upload(nil, invalid)}, + {"Delete(invalid)", crane.Delete(invalid)}, + {"Delete: 404", crane.Delete(valid404)}, + {"Save(_, invalid)", crane.Save(nil, invalid, "")}, + {"SaveLegacy(_, invalid)", crane.SaveLegacy(nil, invalid, "")}, + {"SaveLegacy(_, invalid)", crane.SaveLegacy(nil, valid404, invalid)}, + {"SaveOCI(_, invalid)", crane.SaveOCI(nil, "")}, + {"Copy(invalid, invalid)", crane.Copy(invalid, invalid)}, + {"Copy(404, invalid)", crane.Copy(valid404, invalid)}, + {"Copy(404, 404)", crane.Copy(valid404, valid404)}, + {"Tag(invalid, invalid)", crane.Tag(invalid, invalid)}, + {"Tag(404, invalid)", crane.Tag(valid404, invalid)}, + {"Tag(404, 404)", crane.Tag(valid404, valid404)}, + {"Optimize(invalid, invalid)", crane.Optimize(invalid, invalid, []string{})}, + {"Optimize(404, invalid)", crane.Optimize(valid404, invalid, []string{})}, + {"Optimize(404, 404)", crane.Optimize(valid404, valid404, []string{})}, + // These return multiple values, which are hard to use as expressions. + {"Pull(invalid)", e(crane.Pull(invalid))}, + {"Digest(invalid)", e(crane.Digest(invalid))}, + {"Manifest(invalid)", e(crane.Manifest(invalid))}, + {"Config(invalid)", e(crane.Config(invalid))}, + {"Config(404)", e(crane.Config(valid404))}, + {"ListTags(invalid)", e(crane.ListTags(invalid))}, + {"ListTags(404)", e(crane.ListTags(valid404))}, + {"Append(_, invalid)", e(crane.Append(nil, invalid))}, + {"Catalog(invalid)", e(crane.Catalog(invalid))}, + {"Catalog(404)", e(crane.Catalog(u.Host))}, + {"PullLayer(invalid)", e(crane.PullLayer(invalid))}, + {"LoadTag(_, invalid)", e(crane.LoadTag("", invalid))}, + {"LoadTag(invalid, 404)", e(crane.LoadTag(invalid, valid404))}, + } { + if tc.err == nil { + t.Errorf("%s: expected err, got nil", tc.desc) + } + } +} diff --git a/pkg/crane/delete.go b/pkg/crane/delete.go new file mode 100644 index 0000000..58a8be1 --- /dev/null +++ b/pkg/crane/delete.go @@ -0,0 +1,33 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Delete deletes the remote reference at src. +func Delete(src string, opt ...Option) error { + o := makeOptions(opt...) + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", src, err) + } + + return remote.Delete(ref, o.Remote...) +} diff --git a/pkg/crane/digest.go b/pkg/crane/digest.go new file mode 100644 index 0000000..868a570 --- /dev/null +++ b/pkg/crane/digest.go @@ -0,0 +1,52 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import "github.com/google/go-containerregistry/pkg/logs" + +// Digest returns the sha256 hash of the remote image at ref. +func Digest(ref string, opt ...Option) (string, error) { + o := makeOptions(opt...) + if o.Platform != nil { + desc, err := getManifest(ref, opt...) + if err != nil { + return "", err + } + if !desc.MediaType.IsIndex() { + return desc.Digest.String(), nil + } + + // TODO: does not work for indexes which contain schema v1 manifests + img, err := desc.Image() + if err != nil { + return "", err + } + digest, err := img.Digest() + if err != nil { + return "", err + } + return digest.String(), nil + } + desc, err := Head(ref, opt...) + if err != nil { + logs.Warn.Printf("HEAD request failed, falling back on GET: %v", err) + rdesc, err := getManifest(ref, opt...) + if err != nil { + return "", err + } + return rdesc.Digest.String(), nil + } + return desc.Digest.String(), nil +} diff --git a/pkg/crane/digest_test.go b/pkg/crane/digest_test.go new file mode 100644 index 0000000..ac215d3 --- /dev/null +++ b/pkg/crane/digest_test.go @@ -0,0 +1,61 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestDigest_MissingDigest(t *testing.T) { + response := []byte("doesn't matter") + digest := "sha256:477c34d98f9e090a4441cf82d2f1f03e64c8eb730e8c1ef39a8595e685d4df65" // Digest of "doesn't matter" + getCalled := false + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v2/" { + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Type", string(types.DockerManifestSchema2)) + if r.Method == http.MethodGet { + getCalled = true + w.Header().Set("Docker-Content-Digest", digest) + } + // This will automatically set the Content-Length header. + w.Write(response) + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + got, err := Digest(fmt.Sprintf("%s/repo:latest", u.Host)) + if err != nil { + t.Fatalf("Digest: %v", err) + } + if got != digest { + t.Errorf("Digest: got %q, want %q", got, digest) + } + if !getCalled { + t.Errorf("Digest: expected GET to be called") + } +} diff --git a/pkg/crane/doc.go b/pkg/crane/doc.go new file mode 100644 index 0000000..7602d79 --- /dev/null +++ b/pkg/crane/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package crane holds libraries used to implement the crane CLI. +package crane diff --git a/pkg/crane/example_test.go b/pkg/crane/example_test.go new file mode 100644 index 0000000..3a6c182 --- /dev/null +++ b/pkg/crane/example_test.go @@ -0,0 +1,31 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane_test + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" +) + +func Example() { + c := map[string][]byte{ + "/binary": []byte("binary contents"), + } + i, _ := crane.Image(c) + d, _ := i.Digest() + fmt.Println(d) + // Output: sha256:09fb0c6289cefaad8c74c7e5fd6758ad6906ab8f57f1350d9f4eb5a7df45ff8b +} diff --git a/pkg/crane/export.go b/pkg/crane/export.go new file mode 100644 index 0000000..5d6da1d --- /dev/null +++ b/pkg/crane/export.go @@ -0,0 +1,47 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +// Export writes the filesystem contents (as a tarball) of img to w. +// If img has a single layer, just write the (uncompressed) contents to w so +// that this "just works" for images that just wrap a single blob. +func Export(img v1.Image, w io.Writer) error { + layers, err := img.Layers() + if err != nil { + return err + } + if len(layers) == 1 { + // If it's a single layer, we don't have to flatten the filesystem. + // An added perk of skipping mutate.Extract here is that this works + // for non-tarball layers. + l := layers[0] + rc, err := l.Uncompressed() + if err != nil { + return err + } + _, err = io.Copy(w, rc) + return err + } + fs := mutate.Extract(img) + _, err = io.Copy(w, fs) + return err +} diff --git a/pkg/crane/export_test.go b/pkg/crane/export_test.go new file mode 100644 index 0000000..e60e941 --- /dev/null +++ b/pkg/crane/export_test.go @@ -0,0 +1,41 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "bytes" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestExport(t *testing.T) { + want := []byte(`{"foo":"bar"}`) + layer := static.NewLayer(want, types.MediaType("application/json")) + img, err := mutate.AppendLayers(empty.Image, layer) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := Export(img, &buf); err != nil { + t.Fatal(err) + } + if got := buf.Bytes(); !bytes.Equal(got, want) { + t.Errorf("got: %s\nwant: %s", got, want) + } +} diff --git a/pkg/crane/filemap.go b/pkg/crane/filemap.go new file mode 100644 index 0000000..36dfc2a --- /dev/null +++ b/pkg/crane/filemap.go @@ -0,0 +1,72 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "archive/tar" + "bytes" + "io" + "sort" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Layer creates a layer from a single file map. These layers are reproducible and consistent. +// A filemap is a path -> file content map representing a file system. +func Layer(filemap map[string][]byte) (v1.Layer, error) { + b := &bytes.Buffer{} + w := tar.NewWriter(b) + + fn := []string{} + for f := range filemap { + fn = append(fn, f) + } + sort.Strings(fn) + + for _, f := range fn { + c := filemap[f] + if err := w.WriteHeader(&tar.Header{ + Name: f, + Size: int64(len(c)), + }); err != nil { + return nil, err + } + if _, err := w.Write(c); err != nil { + return nil, err + } + } + if err := w.Close(); err != nil { + return nil, err + } + + // Return a new copy of the buffer each time it's opened. + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(b.Bytes())), nil + }) +} + +// Image creates a image with the given filemaps as its contents. These images are reproducible and consistent. +// A filemap is a path -> file content map representing a file system. +func Image(filemap map[string][]byte) (v1.Image, error) { + y, err := Layer(filemap) + if err != nil { + return nil, err + } + + return mutate.AppendLayers(empty.Image, y) +} diff --git a/pkg/crane/filemap_test.go b/pkg/crane/filemap_test.go new file mode 100644 index 0000000..21d8d54 --- /dev/null +++ b/pkg/crane/filemap_test.go @@ -0,0 +1,187 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane_test + +import ( + "archive/tar" + "errors" + "io" + "testing" + + "github.com/google/go-containerregistry/pkg/crane" +) + +func TestLayer(t *testing.T) { + tcs := []struct { + Name string + FileMap map[string][]byte + Digest string + }{{ + Name: "Empty contents", + Digest: "sha256:89732bc7504122601f40269fc9ddfb70982e633ea9caf641ae45736f2846b004", + }, { + Name: "One file", + FileMap: map[string][]byte{ + "/test": []byte("testy"), + }, + Digest: "sha256:ec3ff19f471b99a76fb1c339c1dfdaa944a4fba25be6bcdc99fe7e772103079e", + }, { + Name: "Two files", + FileMap: map[string][]byte{ + "/test": []byte("testy"), + "/testalt": []byte("footesty"), + }, + Digest: "sha256:a48bcb7be3ab3ec608ee56eb80901224e19e31dc096cc06a8fd3a8dae1aa8947", + }, { + Name: "Many files", + FileMap: map[string][]byte{ + "/1": []byte("1"), + "/2": []byte("2"), + "/3": []byte("3"), + "/4": []byte("4"), + "/5": []byte("5"), + "/6": []byte("6"), + "/7": []byte("7"), + "/8": []byte("8"), + "/9": []byte("9"), + }, + Digest: "sha256:1e637602abbcab2dcedcc24e0b7c19763454a47261f1658b57569530b369ccb9", + }} + + for _, tc := range tcs { + t.Run(tc.Name, func(t *testing.T) { + l, err := crane.Layer(tc.FileMap) + if err != nil { + t.Fatalf("Error calling layer: %v", err) + } + + d, err := l.Digest() + if err != nil { + t.Fatalf("Error calling digest: %v", err) + } + if d.String() != tc.Digest { + t.Errorf("Incorrect digest, want %q, got %q", tc.Digest, d.String()) + } + + // Check contents match. + rc, err := l.Uncompressed() + if err != nil { + t.Fatalf("Uncompressed: %v", err) + } + defer rc.Close() + tr := tar.NewReader(rc) + saw := map[string]struct{}{} + for { + th, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("Next: %v", err) + } + saw[th.Name] = struct{}{} + want, found := tc.FileMap[th.Name] + if !found { + t.Errorf("found %q, not in original map", th.Name) + continue + } + got, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("ReadAll(%q): %v", th.Name, err) + } + if string(want) != string(got) { + t.Errorf("File %q: got %v, want %v", th.Name, string(got), string(want)) + } + } + for k := range saw { + delete(tc.FileMap, k) + } + for k := range tc.FileMap { + t.Errorf("Layer did not contain %q", k) + } + }) + t.Run(tc.Name+" is reproducible", func(t *testing.T) { + l1, _ := crane.Layer(tc.FileMap) + l2, _ := crane.Layer(tc.FileMap) + d1, _ := l1.Digest() + d2, _ := l2.Digest() + if d1 != d2 { + t.Fatalf("Non matching digests, want %q, got %q", d1, d2) + } + }) + } +} + +func TestImage(t *testing.T) { + tcs := []struct { + Name string + FileMap map[string][]byte + Digest string + }{{ + Name: "Empty contents", + Digest: "sha256:98132f58b523c391a5788997327cac95e114e3a6609d01163189774510705399", + }, { + Name: "One file", + FileMap: map[string][]byte{ + "/test": []byte("testy"), + }, + Digest: "sha256:d905c03ac635172a96c12b8af6c90cfd028e3edaa3114b31a9e196ab38c16963", + }, { + Name: "Two files", + FileMap: map[string][]byte{ + "/test": []byte("testy"), + "/bar": []byte("not useful"), + }, + Digest: "sha256:20e7e4800e5eb167f170970936c08d9e1bcbe91372420eeb6ab8d1a07752c3a3", + }, { + Name: "Many files", + FileMap: map[string][]byte{ + "/1": []byte("1"), + "/2": []byte("2"), + "/3": []byte("3"), + "/4": []byte("4"), + "/5": []byte("5"), + "/6": []byte("6"), + "/7": []byte("7"), + "/8": []byte("8"), + "/9": []byte("9"), + }, + Digest: "sha256:dfca2803510c8e3b83a3151f7c035c60cfa2a8a52465b802e18b85014de361f1", + }} + for _, tc := range tcs { + t.Run(tc.Name, func(t *testing.T) { + i, err := crane.Image(tc.FileMap) + if err != nil { + t.Fatalf("Error calling image: %v", err) + } + d, err := i.Digest() + if err != nil { + t.Fatalf("Error calling digest: %v", err) + } + if d.String() != tc.Digest { + t.Fatalf("Incorrect digest, want %q, got %q", tc.Digest, d.String()) + } + }) + t.Run(tc.Name+" is reproducible", func(t *testing.T) { + i1, _ := crane.Image(tc.FileMap) + i2, _ := crane.Image(tc.FileMap) + d1, _ := i1.Digest() + d2, _ := i2.Digest() + if d1 != d2 { + t.Fatalf("Non matching digests, want %q, got %q", d1, d2) + } + }) + } +} diff --git a/pkg/crane/get.go b/pkg/crane/get.go new file mode 100644 index 0000000..1f12f01 --- /dev/null +++ b/pkg/crane/get.go @@ -0,0 +1,56 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func getImage(r string, opt ...Option) (v1.Image, name.Reference, error) { + o := makeOptions(opt...) + ref, err := name.ParseReference(r, o.Name...) + if err != nil { + return nil, nil, fmt.Errorf("parsing reference %q: %w", r, err) + } + img, err := remote.Image(ref, o.Remote...) + if err != nil { + return nil, nil, fmt.Errorf("reading image %q: %w", ref, err) + } + return img, ref, nil +} + +func getManifest(r string, opt ...Option) (*remote.Descriptor, error) { + o := makeOptions(opt...) + ref, err := name.ParseReference(r, o.Name...) + if err != nil { + return nil, fmt.Errorf("parsing reference %q: %w", r, err) + } + return remote.Get(ref, o.Remote...) +} + +// Head performs a HEAD request for a manifest and returns a content descriptor +// based on the registry's response. +func Head(r string, opt ...Option) (*v1.Descriptor, error) { + o := makeOptions(opt...) + ref, err := name.ParseReference(r, o.Name...) + if err != nil { + return nil, err + } + return remote.Head(ref, o.Remote...) +} diff --git a/pkg/crane/list.go b/pkg/crane/list.go new file mode 100644 index 0000000..3835215 --- /dev/null +++ b/pkg/crane/list.go @@ -0,0 +1,33 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// ListTags returns the tags in repository src. +func ListTags(src string, opt ...Option) ([]string, error) { + o := makeOptions(opt...) + repo, err := name.NewRepository(src, o.Name...) + if err != nil { + return nil, fmt.Errorf("parsing repo %q: %w", src, err) + } + + return remote.List(repo, o.Remote...) +} diff --git a/pkg/crane/manifest.go b/pkg/crane/manifest.go new file mode 100644 index 0000000..a54926a --- /dev/null +++ b/pkg/crane/manifest.go @@ -0,0 +1,32 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +// Manifest returns the manifest for the remote image or index ref. +func Manifest(ref string, opt ...Option) ([]byte, error) { + desc, err := getManifest(ref, opt...) + if err != nil { + return nil, err + } + o := makeOptions(opt...) + if o.Platform != nil { + img, err := desc.Image() + if err != nil { + return nil, err + } + return img.RawManifest() + } + return desc.Manifest, nil +} diff --git a/pkg/crane/optimize.go b/pkg/crane/optimize.go new file mode 100644 index 0000000..74c665d --- /dev/null +++ b/pkg/crane/optimize.go @@ -0,0 +1,237 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "errors" + "fmt" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Optimize optimizes a remote image or index from src to dst. +// THIS API IS EXPERIMENTAL AND SUBJECT TO CHANGE WITHOUT WARNING. +func Optimize(src, dst string, prioritize []string, opt ...Option) error { + pset := newStringSet(prioritize) + o := makeOptions(opt...) + srcRef, err := name.ParseReference(src, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", src, err) + } + + dstRef, err := name.ParseReference(dst, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference for %q: %w", dst, err) + } + + logs.Progress.Printf("Optimizing from %v to %v", srcRef, dstRef) + desc, err := remote.Get(srcRef, o.Remote...) + if err != nil { + return fmt.Errorf("fetching %q: %w", src, err) + } + + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + // Handle indexes separately. + if o.Platform != nil { + // If platform is explicitly set, don't optimize the whole index, just the appropriate image. + if err := optimizeAndPushImage(desc, dstRef, pset, o); err != nil { + return fmt.Errorf("failed to optimize image: %w", err) + } + } else { + if err := optimizeAndPushIndex(desc, dstRef, pset, o); err != nil { + return fmt.Errorf("failed to optimize index: %w", err) + } + } + + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + return errors.New("docker schema 1 images are not supported") + + default: + // Assume anything else is an image, since some registries don't set mediaTypes properly. + if err := optimizeAndPushImage(desc, dstRef, pset, o); err != nil { + return fmt.Errorf("failed to optimize image: %w", err) + } + } + + return nil +} + +func optimizeAndPushImage(desc *remote.Descriptor, dstRef name.Reference, prioritize stringSet, o Options) error { + img, err := desc.Image() + if err != nil { + return err + } + + missing, oimg, err := optimizeImage(img, prioritize) + if err != nil { + return err + } + + if len(missing) > 0 { + return fmt.Errorf("the following prioritized files were missing from image: %v", missing.List()) + } + + return remote.Write(dstRef, oimg, o.Remote...) +} + +func optimizeImage(img v1.Image, prioritize stringSet) (stringSet, v1.Image, error) { + cfg, err := img.ConfigFile() + if err != nil { + return nil, nil, err + } + ocfg := cfg.DeepCopy() + ocfg.History = nil + ocfg.RootFS.DiffIDs = nil + + oimg, err := mutate.ConfigFile(empty.Image, ocfg) + if err != nil { + return nil, nil, err + } + + layers, err := img.Layers() + if err != nil { + return nil, nil, err + } + + missingFromImage := newStringSet(prioritize.List()) + olayers := make([]mutate.Addendum, 0, len(layers)) + for _, layer := range layers { + missingFromLayer := []string{} + olayer, err := tarball.LayerFromOpener(layer.Uncompressed, + tarball.WithEstargz, + tarball.WithEstargzOptions( + estargz.WithPrioritizedFiles(prioritize.List()), + estargz.WithAllowPrioritizeNotFound(&missingFromLayer), + )) + if err != nil { + return nil, nil, err + } + missingFromImage = missingFromImage.Intersection(newStringSet(missingFromLayer)) + + olayers = append(olayers, mutate.Addendum{ + Layer: olayer, + MediaType: types.DockerLayer, + }) + } + + oimg, err = mutate.Append(oimg, olayers...) + if err != nil { + return nil, nil, err + } + return missingFromImage, oimg, nil +} + +func optimizeAndPushIndex(desc *remote.Descriptor, dstRef name.Reference, prioritize stringSet, o Options) error { + idx, err := desc.ImageIndex() + if err != nil { + return err + } + + missing, oidx, err := optimizeIndex(idx, prioritize) + if err != nil { + return err + } + + if len(missing) > 0 { + return fmt.Errorf("the following prioritized files were missing from all images: %v", missing.List()) + } + + return remote.WriteIndex(dstRef, oidx, o.Remote...) +} + +func optimizeIndex(idx v1.ImageIndex, prioritize stringSet) (stringSet, v1.ImageIndex, error) { + im, err := idx.IndexManifest() + if err != nil { + return nil, nil, err + } + + missingFromIndex := newStringSet(prioritize.List()) + + // Build an image for each child from the base and append it to a new index to produce the result. + adds := make([]mutate.IndexAddendum, 0, len(im.Manifests)) + for _, desc := range im.Manifests { + img, err := idx.Image(desc.Digest) + if err != nil { + return nil, nil, err + } + + missingFromImage, oimg, err := optimizeImage(img, prioritize) + if err != nil { + return nil, nil, err + } + missingFromIndex = missingFromIndex.Intersection(missingFromImage) + adds = append(adds, mutate.IndexAddendum{ + Add: oimg, + Descriptor: v1.Descriptor{ + URLs: desc.URLs, + MediaType: desc.MediaType, + Annotations: desc.Annotations, + Platform: desc.Platform, + }, + }) + } + + idxType, err := idx.MediaType() + if err != nil { + return nil, nil, err + } + + return missingFromIndex, mutate.IndexMediaType(mutate.AppendManifests(empty.Index, adds...), idxType), nil +} + +type stringSet map[string]struct{} + +func newStringSet(in []string) stringSet { + ss := stringSet{} + for _, s := range in { + ss[s] = struct{}{} + } + return ss +} + +func (s stringSet) List() []string { + result := make([]string, 0, len(s)) + for k := range s { + result = append(result, k) + } + return result +} + +func (s stringSet) Intersection(rhs stringSet) stringSet { + // To appease ST1016 + lhs := s + + // Make sure len(lhs) >= len(rhs) + if len(lhs) < len(rhs) { + return rhs.Intersection(lhs) + } + + result := stringSet{} + for k := range lhs { + if _, ok := rhs[k]; ok { + result[k] = struct{}{} + } + } + return result +} diff --git a/pkg/crane/optimize_test.go b/pkg/crane/optimize_test.go new file mode 100644 index 0000000..11aaf57 --- /dev/null +++ b/pkg/crane/optimize_test.go @@ -0,0 +1,179 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "net/http/httptest" + "net/url" + "path" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func TestStringSet(t *testing.T) { + for _, tc := range []struct { + lhs []string + rhs []string + result []string + }{{ + lhs: []string{}, + rhs: []string{}, + result: []string{}, + }, { + lhs: []string{"a"}, + rhs: []string{}, + result: []string{}, + }, { + lhs: []string{}, + rhs: []string{"a"}, + result: []string{}, + }, { + lhs: []string{"a", "b", "c"}, + rhs: []string{"a", "b", "c"}, + result: []string{"a", "b", "c"}, + }, { + lhs: []string{"a", "b"}, + rhs: []string{"a"}, + result: []string{"a"}, + }, { + lhs: []string{"a"}, + rhs: []string{"a", "b"}, + result: []string{"a"}, + }} { + got := newStringSet(tc.lhs).Intersection(newStringSet(tc.rhs)) + want := newStringSet(tc.result) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("%v.intersect(%v) (-want +got): %s", tc.lhs, tc.rhs, diff) + } + + less := func(a, b string) bool { + return strings.Compare(a, b) <= -1 + } + if diff := cmp.Diff(tc.result, got.List(), cmpopts.SortSlices(less)); diff != "" { + t.Errorf("%v.List() (-want +got): = %v", tc.result, diff) + } + } +} + +func TestOptimize(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + imgs := []mutate.IndexAddendum{} + for _, plat := range []string{ + "linux/amd64", + "linux/arm", + } { + img, err := Image(map[string][]byte{ + "unimportant": []byte(strings.Repeat("deadbeef", 128)), + "important": []byte(`abc`), + "platform.txt": []byte(plat), + }) + if err != nil { + t.Fatal(err) + } + parts := strings.Split(plat, "/") + imgs = append(imgs, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{ + Platform: &v1.Platform{ + OS: parts[0], + Architecture: parts[1], + }, + }, + }) + } + + idx := mutate.AppendManifests(empty.Index, imgs...) + + slow := path.Join(u.Host, "slow") + fast := path.Join(u.Host, "fast") + + ref, err := name.ParseReference(slow) + if err != nil { + t.Fatal(err) + } + + if err := remote.WriteIndex(ref, idx); err != nil { + t.Fatal(err) + } + + if err := Optimize(slow, fast, []string{"important"}); err != nil { + t.Fatal(err) + } + + if err := Optimize(slow, fast, []string{"important"}, WithPlatform(imgs[1].Platform)); err != nil { + t.Fatal(err) + } + + // Compare optimize WithPlatform path to optimizing just an image. + got, err := Digest(fast) + if err != nil { + t.Fatal(err) + } + + dig, err := Digest(slow, WithPlatform(imgs[1].Platform)) + if err != nil { + t.Fatal(err) + } + + slowImgRef := slow + "@" + dig + if err := Optimize(slowImgRef, fast, []string{"important"}, WithPlatform(imgs[1].Platform)); err != nil { + t.Fatal(err) + } + + want, err := Digest(fast) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("Optimize(WithPlatform) != Optimize(bydigest): %q != %q", got, want) + } + + for i, ref := range []string{slow, slow, slowImgRef} { + opts := []Option{} + // Silly, but use WithPlatform to get some more coverage. + if i != 0 { + opts = []Option{WithPlatform(imgs[1].Platform)} + } + dig, err := Digest(ref, opts...) + if err != nil { + t.Errorf("Digest(%q): %v", ref, err) + continue + } + // Make sure we fail if there's a missing file in the optimize set + // Use the image digest because it's ~impossible to exist in img. + if err := Optimize(ref, fast, []string{dig}, opts...); err == nil { + t.Errorf("Optimize(%q, prioritize=%q): got nil, want err", ref, dig) + } else if !strings.Contains(err.Error(), dig) { + // Make sure this contains the missing file (dig) + t.Errorf("Optimize(%q) error should contain %q, got: %v", ref, dig, err) + } + } +} diff --git a/pkg/crane/options.go b/pkg/crane/options.go new file mode 100644 index 0000000..5d2e0e4 --- /dev/null +++ b/pkg/crane/options.go @@ -0,0 +1,149 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "context" + "crypto/tls" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Options hold the options that crane uses when calling other packages. +type Options struct { + Name []name.Option + Remote []remote.Option + Platform *v1.Platform + Keychain authn.Keychain + + transport http.RoundTripper + insecure bool +} + +// GetOptions exposes the underlying []remote.Option, []name.Option, and +// platform, based on the passed Option. Generally, you shouldn't need to use +// this unless you've painted yourself into a dependency corner as we have +// with the crane and gcrane cli packages. +func GetOptions(opts ...Option) Options { + return makeOptions(opts...) +} + +func makeOptions(opts ...Option) Options { + opt := Options{ + Remote: []remote.Option{ + remote.WithAuthFromKeychain(authn.DefaultKeychain), + }, + Keychain: authn.DefaultKeychain, + } + + for _, o := range opts { + o(&opt) + } + + // Allow for untrusted certificates if the user + // passed Insecure but no custom transport. + if opt.insecure && opt.transport == nil { + transport := remote.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, //nolint: gosec + } + + WithTransport(transport)(&opt) + } + + return opt +} + +// Option is a functional option for crane. +type Option func(*Options) + +// WithTransport is a functional option for overriding the default transport +// for remote operations. Setting a transport will override the Insecure option's +// configuration allowing for image registries to use untrusted certificates. +func WithTransport(t http.RoundTripper) Option { + return func(o *Options) { + o.Remote = append(o.Remote, remote.WithTransport(t)) + o.transport = t + } +} + +// Insecure is an Option that allows image references to be fetched without TLS. +// This will also allow for untrusted (e.g. self-signed) certificates in cases where +// the default transport is used (i.e. when WithTransport is not used). +func Insecure(o *Options) { + o.Name = append(o.Name, name.Insecure) + o.insecure = true +} + +// WithPlatform is an Option to specify the platform. +func WithPlatform(platform *v1.Platform) Option { + return func(o *Options) { + if platform != nil { + o.Remote = append(o.Remote, remote.WithPlatform(*platform)) + } + o.Platform = platform + } +} + +// WithAuthFromKeychain is a functional option for overriding the default +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// By default, crane will use authn.DefaultKeychain. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *Options) { + // Replace the default keychain at position 0. + o.Remote[0] = remote.WithAuthFromKeychain(keys) + o.Keychain = keys + } +} + +// WithAuth is a functional option for overriding the default authenticator +// for remote operations. +// +// By default, crane will use authn.DefaultKeychain. +func WithAuth(auth authn.Authenticator) Option { + return func(o *Options) { + // Replace the default keychain at position 0. + o.Remote[0] = remote.WithAuth(auth) + } +} + +// WithUserAgent adds the given string to the User-Agent header for any HTTP +// requests. +func WithUserAgent(ua string) Option { + return func(o *Options) { + o.Remote = append(o.Remote, remote.WithUserAgent(ua)) + } +} + +// WithNondistributable is an option that allows pushing non-distributable +// layers. +func WithNondistributable() Option { + return func(o *Options) { + o.Remote = append(o.Remote, remote.WithNondistributable) + } +} + +// WithContext is a functional option for setting the context. +func WithContext(ctx context.Context) Option { + return func(o *Options) { + o.Remote = append(o.Remote, remote.WithContext(ctx)) + } +} diff --git a/pkg/crane/options_test.go b/pkg/crane/options_test.go new file mode 100644 index 0000000..98d7396 --- /dev/null +++ b/pkg/crane/options_test.go @@ -0,0 +1,58 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "errors" + "net/http" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func TestInsecureOptionTracking(t *testing.T) { + want := true + opts := GetOptions(Insecure) + + if got := opts.insecure; got != want { + t.Errorf("got %t\nwant: %t", got, want) + } +} + +func TestTransportSetting(t *testing.T) { + opts := GetOptions(WithTransport(remote.DefaultTransport)) + + if opts.transport == nil { + t.Error("expected crane transport to be set when user passes WithTransport") + } +} + +func TestInsecureTransport(t *testing.T) { + want := true + opts := GetOptions(Insecure) + var transport *http.Transport + var ok bool + if transport, ok = opts.transport.(*http.Transport); !ok { + t.Fatal("Unable to successfully assert default transport") + } + + if transport.TLSClientConfig == nil { + t.Fatal(errors.New("TLSClientConfig was nil and should be set")) + } + + if got := transport.TLSClientConfig.InsecureSkipVerify; got != want { + t.Errorf("got: %t\nwant: %t", got, want) + } +} diff --git a/pkg/crane/pull.go b/pkg/crane/pull.go new file mode 100644 index 0000000..7e6e5b7 --- /dev/null +++ b/pkg/crane/pull.go @@ -0,0 +1,142 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + "os" + + legacy "github.com/google/go-containerregistry/pkg/legacy/tarball" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Tag applied to images that were pulled by digest. This denotes that the +// image was (probably) never tagged with this, but lets us avoid applying the +// ":latest" tag which might be misleading. +const iWasADigestTag = "i-was-a-digest" + +// Pull returns a v1.Image of the remote image src. +func Pull(src string, opt ...Option) (v1.Image, error) { + o := makeOptions(opt...) + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return nil, fmt.Errorf("parsing reference %q: %w", src, err) + } + + return remote.Image(ref, o.Remote...) +} + +// Save writes the v1.Image img as a tarball at path with tag src. +func Save(img v1.Image, src, path string) error { + imgMap := map[string]v1.Image{src: img} + return MultiSave(imgMap, path) +} + +// MultiSave writes collection of v1.Image img with tag as a tarball. +func MultiSave(imgMap map[string]v1.Image, path string, opt ...Option) error { + o := makeOptions(opt...) + tagToImage := map[name.Tag]v1.Image{} + + for src, img := range imgMap { + ref, err := name.ParseReference(src, o.Name...) + if err != nil { + return fmt.Errorf("parsing ref %q: %w", src, err) + } + + // WriteToFile wants a tag to write to the tarball, but we might have + // been given a digest. + // If the original ref was a tag, use that. Otherwise, if it was a + // digest, tag the image with :i-was-a-digest instead. + tag, ok := ref.(name.Tag) + if !ok { + d, ok := ref.(name.Digest) + if !ok { + return fmt.Errorf("ref wasn't a tag or digest") + } + tag = d.Repository.Tag(iWasADigestTag) + } + tagToImage[tag] = img + } + // no progress channel (for now) + return tarball.MultiWriteToFile(path, tagToImage) +} + +// PullLayer returns the given layer from a registry. +func PullLayer(ref string, opt ...Option) (v1.Layer, error) { + o := makeOptions(opt...) + digest, err := name.NewDigest(ref, o.Name...) + if err != nil { + return nil, err + } + + return remote.Layer(digest, o.Remote...) +} + +// SaveLegacy writes the v1.Image img as a legacy tarball at path with tag src. +func SaveLegacy(img v1.Image, src, path string) error { + imgMap := map[string]v1.Image{src: img} + return MultiSave(imgMap, path) +} + +// MultiSaveLegacy writes collection of v1.Image img with tag as a legacy tarball. +func MultiSaveLegacy(imgMap map[string]v1.Image, path string) error { + refToImage := map[name.Reference]v1.Image{} + + for src, img := range imgMap { + ref, err := name.ParseReference(src) + if err != nil { + return fmt.Errorf("parsing ref %q: %w", src, err) + } + refToImage[ref] = img + } + + w, err := os.Create(path) + if err != nil { + return err + } + defer w.Close() + + return legacy.MultiWrite(refToImage, w) +} + +// SaveOCI writes the v1.Image img as an OCI Image Layout at path. If a layout +// already exists at that path, it will add the image to the index. +func SaveOCI(img v1.Image, path string) error { + imgMap := map[string]v1.Image{"": img} + return MultiSaveOCI(imgMap, path) +} + +// MultiSaveOCI writes collection of v1.Image img as an OCI Image Layout at path. If a layout +// already exists at that path, it will add the image to the index. +func MultiSaveOCI(imgMap map[string]v1.Image, path string) error { + p, err := layout.FromPath(path) + if err != nil { + p, err = layout.Write(path, empty.Index) + if err != nil { + return err + } + } + for _, img := range imgMap { + if err = p.AppendImage(img); err != nil { + return err + } + } + return nil +} diff --git a/pkg/crane/push.go b/pkg/crane/push.go new file mode 100644 index 0000000..6d1fbd6 --- /dev/null +++ b/pkg/crane/push.go @@ -0,0 +1,65 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Load reads the tarball at path as a v1.Image. +func Load(path string, opt ...Option) (v1.Image, error) { + return LoadTag(path, "") +} + +// LoadTag reads a tag from the tarball at path as a v1.Image. +// If tag is "", will attempt to read the tarball as a single image. +func LoadTag(path, tag string, opt ...Option) (v1.Image, error) { + if tag == "" { + return tarball.ImageFromPath(path, nil) + } + + o := makeOptions(opt...) + t, err := name.NewTag(tag, o.Name...) + if err != nil { + return nil, fmt.Errorf("parsing tag %q: %w", tag, err) + } + return tarball.ImageFromPath(path, &t) +} + +// Push pushes the v1.Image img to a registry as dst. +func Push(img v1.Image, dst string, opt ...Option) error { + o := makeOptions(opt...) + tag, err := name.ParseReference(dst, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", dst, err) + } + return remote.Write(tag, img, o.Remote...) +} + +// Upload pushes the v1.Layer to a given repo. +func Upload(layer v1.Layer, repo string, opt ...Option) error { + o := makeOptions(opt...) + ref, err := name.NewRepository(repo, o.Name...) + if err != nil { + return fmt.Errorf("parsing repo %q: %w", repo, err) + } + + return remote.WriteLayer(ref, layer, o.Remote...) +} diff --git a/pkg/crane/tag.go b/pkg/crane/tag.go new file mode 100644 index 0000000..13bc395 --- /dev/null +++ b/pkg/crane/tag.go @@ -0,0 +1,39 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Tag adds tag to the remote img. +func Tag(img, tag string, opt ...Option) error { + o := makeOptions(opt...) + ref, err := name.ParseReference(img, o.Name...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", img, err) + } + desc, err := remote.Get(ref, o.Remote...) + if err != nil { + return fmt.Errorf("fetching %q: %w", img, err) + } + + dst := ref.Context().Tag(tag) + + return remote.Tag(dst, desc, o.Remote...) +} diff --git a/pkg/crane/testdata/content.tar b/pkg/crane/testdata/content.tar new file mode 100755 index 0000000..55f4d1d Binary files /dev/null and b/pkg/crane/testdata/content.tar differ diff --git a/pkg/gcrane/copy.go b/pkg/gcrane/copy.go new file mode 100644 index 0000000..a3362c4 --- /dev/null +++ b/pkg/gcrane/copy.go @@ -0,0 +1,347 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcrane + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/google/go-containerregistry/internal/retry" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "golang.org/x/sync/errgroup" +) + +// Keychain tries to use google-specific credential sources, falling back to +// the DefaultKeychain (config-file based). +var Keychain = authn.NewMultiKeychain(google.Keychain, authn.DefaultKeychain) + +// GCRBackoff returns a retry.Backoff that is suitable for use with gcr.io. +// +// These numbers are based on GCR's posted quotas: +// https://cloud.google.com/container-registry/quotas +// - 30k requests per 10 minutes. +// - 500k requests per 24 hours. +// +// On error, we will wait for: +// - 6 seconds (in case of very short term 429s from GCS), then +// - 1 minute (in case of temporary network issues), then +// - 10 minutes (to get around GCR 10 minute quotas), then fail. +// +// TODO: In theory, we could keep retrying until the next day to get around the 500k limit. +func GCRBackoff() retry.Backoff { + return retry.Backoff{ + Duration: 6 * time.Second, + Factor: 10.0, + Jitter: 0.1, + Steps: 3, + Cap: 1 * time.Hour, + } +} + +// Copy copies a remote image or index from src to dst. +func Copy(src, dst string, opts ...Option) error { + o := makeOptions(opts...) + // Just reuse crane's copy logic with gcrane's credential logic. + return crane.Copy(src, dst, o.crane...) +} + +// CopyRepository copies everything from the src GCR repository to the +// dst GCR repository. +func CopyRepository(ctx context.Context, src, dst string, opts ...Option) error { + o := makeOptions(opts...) + return recursiveCopy(ctx, src, dst, o) +} + +type task struct { + digest string + manifest google.ManifestInfo + oldRepo name.Repository + newRepo name.Repository +} + +type copier struct { + srcRepo name.Repository + dstRepo name.Repository + + tasks chan task + opt *options +} + +func newCopier(src, dst string, o *options) (*copier, error) { + srcRepo, err := name.NewRepository(src) + if err != nil { + return nil, fmt.Errorf("parsing repo %q: %w", src, err) + } + + dstRepo, err := name.NewRepository(dst) + if err != nil { + return nil, fmt.Errorf("parsing repo %q: %w", dst, err) + } + + // A queue of size 2*jobs should keep each goroutine busy. + tasks := make(chan task, o.jobs*2) + + return &copier{srcRepo, dstRepo, tasks, o}, nil +} + +// recursiveCopy copies images from repo src to repo dst. +func recursiveCopy(ctx context.Context, src, dst string, o *options) error { + c, err := newCopier(src, dst, o) + if err != nil { + return err + } + + g, ctx := errgroup.WithContext(ctx) + walkFn := func(repo name.Repository, tags *google.Tags, err error) error { + if err != nil { + logs.Warn.Printf("failed walkFn for repo %s: %v", repo, err) + // If we hit an error when listing the repo, try re-listing with backoff. + if err := backoffErrors(GCRBackoff(), func() error { + tags, err = google.List(repo, o.google...) + return err + }); err != nil { + return fmt.Errorf("failed List for repo %s: %w", repo, err) + } + } + + // If we hit an error when trying to diff the repo, re-diff with backoff. + if err := backoffErrors(GCRBackoff(), func() error { + return c.copyRepo(ctx, repo, tags) + }); err != nil { + return fmt.Errorf("failed to copy repo %q: %w", repo, err) + } + + return nil + } + + // Start walking the repo, enqueuing items in c.tasks. + g.Go(func() error { + defer close(c.tasks) + if err := google.Walk(c.srcRepo, walkFn, o.google...); err != nil { + return fmt.Errorf("failed to Walk: %w", err) + } + return nil + }) + + // Pull items off of c.tasks and copy the images. + for i := 0; i < o.jobs; i++ { + g.Go(func() error { + for task := range c.tasks { + // If we hit an error when trying to copy the images, + // retry with backoff. + if err := backoffErrors(GCRBackoff(), func() error { + return c.copyImages(ctx, task) + }); err != nil { + return fmt.Errorf("failed to copy %q: %w", task.digest, err) + } + } + return nil + }) + } + + return g.Wait() +} + +// copyRepo figures out the name for our destination repo (newRepo), lists the +// contents of newRepo, calculates the diff of what needs to be copied, then +// starts a goroutine to copy each image we need, and waits for them to finish. +func (c *copier) copyRepo(ctx context.Context, oldRepo name.Repository, tags *google.Tags) error { + newRepo, err := c.rename(oldRepo) + if err != nil { + return fmt.Errorf("rename failed: %w", err) + } + + // Figure out what we actually need to copy. + want := tags.Manifests + have := make(map[string]google.ManifestInfo) + haveTags, err := google.List(newRepo, c.opt.google...) + if err != nil { + if !hasStatusCode(err, http.StatusNotFound) { + return err + } + // This is a 404 code, so we just need to copy everything. + logs.Warn.Printf("failed to list %s: %v", newRepo, err) + } else { + have = haveTags.Manifests + } + need := diffImages(want, have) + + // Queue up every image as a task. + for digest, manifest := range need { + t := task{ + digest: digest, + manifest: manifest, + oldRepo: oldRepo, + newRepo: newRepo, + } + select { + case c.tasks <- t: + case <-ctx.Done(): + return ctx.Err() + } + } + + return nil +} + +// copyImages starts a goroutine for each tag that points to the image +// oldRepo@digest, or just copies the image by digest if there are no tags. +func (c *copier) copyImages(_ context.Context, t task) error { + // We only have to explicitly copy by digest if there are no tags pointing to this manifest. + if len(t.manifest.Tags) == 0 { + srcImg := fmt.Sprintf("%s@%s", t.oldRepo, t.digest) + dstImg := fmt.Sprintf("%s@%s", t.newRepo, t.digest) + + return crane.Copy(srcImg, dstImg, c.opt.crane...) + } + + // We only need to push the whole image once. + tag := t.manifest.Tags[0] + srcImg := fmt.Sprintf("%s:%s", t.oldRepo, tag) + dstImg := fmt.Sprintf("%s:%s", t.newRepo, tag) + + if err := crane.Copy(srcImg, dstImg, c.opt.crane...); err != nil { + return err + } + + if len(t.manifest.Tags) <= 1 { + // If there's only one tag, we're done. + return nil + } + + // Add the rest of the tags. + srcRef, err := name.ParseReference(srcImg) + if err != nil { + return err + } + desc, err := remote.Get(srcRef, c.opt.remote...) + if err != nil { + return err + } + + for _, tag := range t.manifest.Tags[1:] { + dstImg := t.newRepo.Tag(tag) + + if err := remote.Tag(dstImg, desc, c.opt.remote...); err != nil { + return err + } + } + + return nil +} + +// Retry temporary errors, 429, and 500+ with backoff. +func backoffErrors(bo retry.Backoff, f func() error) error { + p := func(err error) bool { + b := retry.IsTemporary(err) || hasStatusCode(err, http.StatusTooManyRequests) || isServerError(err) + if b { + logs.Warn.Printf("Retrying %v", err) + } + return b + } + return retry.Retry(f, p, bo) +} + +func hasStatusCode(err error, code int) bool { + if err == nil { + return false + } + var terr *transport.Error + if errors.As(err, &terr) { + if terr.StatusCode == code { + return true + } + } + return false +} + +func isServerError(err error) bool { + if err == nil { + return false + } + var terr *transport.Error + if errors.As(err, &terr) { + return terr.StatusCode >= 500 + } + return false +} + +// rename figures out the name of the new repository to copy to, e.g.: +// +// $ gcrane cp -r gcr.io/foo gcr.io/baz +// +// rename("gcr.io/foo/bar") == "gcr.io/baz/bar" +func (c *copier) rename(repo name.Repository) (name.Repository, error) { + replaced := strings.Replace(repo.String(), c.srcRepo.String(), c.dstRepo.String(), 1) + return name.NewRepository(replaced, name.StrictValidation) +} + +// diffImages returns a map of digests to google.ManifestInfos for images or +// tags that are present in "want" but not in "have". +func diffImages(want, have map[string]google.ManifestInfo) map[string]google.ManifestInfo { + need := make(map[string]google.ManifestInfo) + + for digest, wantManifest := range want { + if haveManifest, ok := have[digest]; !ok { + // Missing the whole image, we need to copy everything. + need[digest] = wantManifest + } else { + missingTags := subtractStringLists(wantManifest.Tags, haveManifest.Tags) + if len(missingTags) == 0 { + continue + } + + // Missing just some tags, add the ones we need to copy. + todo := wantManifest + todo.Tags = missingTags + need[digest] = todo + } + } + + return need +} + +// subtractStringLists returns a list of strings that are in minuend and not +// in subtrahend; order is unimportant. +func subtractStringLists(minuend, subtrahend []string) []string { + bSet := toStringSet(subtrahend) + difference := []string{} + + for _, a := range minuend { + if _, ok := bSet[a]; !ok { + difference = append(difference, a) + } + } + + return difference +} + +func toStringSet(slice []string) map[string]struct{} { + set := make(map[string]struct{}, len(slice)) + for _, s := range slice { + set[s] = struct{}{} + } + return set +} diff --git a/pkg/gcrane/copy_test.go b/pkg/gcrane/copy_test.go new file mode 100644 index 0000000..e50564a --- /dev/null +++ b/pkg/gcrane/copy_test.go @@ -0,0 +1,428 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcrane + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type fakeXCR struct { + h http.Handler + repos map[string]google.Tags + t *testing.T +} + +func (xcr *fakeXCR) ServeHTTP(w http.ResponseWriter, r *http.Request) { + xcr.t.Logf("%s %s", r.Method, r.URL) + if strings.HasPrefix(r.URL.Path, "/v2/") && strings.HasSuffix(r.URL.Path, "/tags/list") { + repo := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list") + if tags, ok := xcr.repos[repo]; !ok { + w.WriteHeader(http.StatusNotFound) + } else { + xcr.t.Logf("%+v", tags) + if err := json.NewEncoder(w).Encode(tags); err != nil { + xcr.t.Fatal(err) + } + } + } else { + xcr.h.ServeHTTP(w, r) + } +} + +func newFakeXCR(t *testing.T) *fakeXCR { + h := registry.New() + return &fakeXCR{h: h, t: t} +} + +func (xcr *fakeXCR) setRefs(stuff map[name.Reference]partial.Describable) error { + repos := make(map[string]google.Tags) + + for ref, thing := range stuff { + repo := ref.Context().RepositoryStr() + tags, ok := repos[repo] + if !ok { + tags = google.Tags{ + Name: repo, + Children: []string{}, + } + } + + // Populate the "child" field. + for parentPath := repo; parentPath != "."; parentPath = path.Dir(parentPath) { + child, parent := path.Base(parentPath), path.Dir(parentPath) + tags, ok := repos[parent] + if !ok { + tags = google.Tags{} + } + for _, c := range repos[parent].Children { + if c == child { + break + } + } + tags.Children = append(tags.Children, child) + repos[parent] = tags + } + + // Populate the "manifests" and "tags" field. + d, err := thing.Digest() + if err != nil { + return err + } + mt, err := thing.MediaType() + if err != nil { + return err + } + if tags.Manifests == nil { + tags.Manifests = make(map[string]google.ManifestInfo) + } + mi, ok := tags.Manifests[d.String()] + if !ok { + mi = google.ManifestInfo{ + MediaType: string(mt), + Tags: []string{}, + } + } + if tag, ok := ref.(name.Tag); ok { + tags.Tags = append(tags.Tags, tag.Identifier()) + mi.Tags = append(mi.Tags, tag.Identifier()) + } + tags.Manifests[d.String()] = mi + repos[repo] = tags + } + xcr.repos = repos + return nil +} + +func TestCopy(t *testing.T) { + logs.Warn.SetOutput(os.Stderr) + xcr := newFakeXCR(t) + s := httptest.NewServer(xcr) + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + defer s.Close() + src := path.Join(u.Host, "test/gcrane") + dst := path.Join(u.Host, "test/gcrane/copy") + + oneTag, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + twoTags, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + noTags, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + + latestRef, err := name.ParseReference(src) + if err != nil { + t.Fatal(err) + } + oneTagRef := latestRef.Context().Tag("bar") + + d, err := noTags.Digest() + if err != nil { + t.Fatal(err) + } + noTagsRef := latestRef.Context().Digest(d.String()) + fooRef := latestRef.Context().Tag("foo") + + // Populate this after we create it so we know the hostname. + if err := xcr.setRefs(map[name.Reference]partial.Describable{ + oneTagRef: oneTag, + latestRef: twoTags, + fooRef: twoTags, + noTagsRef: noTags, + }); err != nil { + t.Fatal(err) + } + + if err := remote.Write(latestRef, twoTags); err != nil { + t.Fatal(err) + } + if err := remote.Write(fooRef, twoTags); err != nil { + t.Fatal(err) + } + if err := remote.Write(oneTagRef, oneTag); err != nil { + t.Fatal(err) + } + if err := remote.Write(noTagsRef, noTags); err != nil { + t.Fatal(err) + } + + if err := Copy(src, dst); err != nil { + t.Fatal(err) + } + + if err := CopyRepository(context.Background(), src, dst); err != nil { + t.Fatal(err) + } +} + +func TestRename(t *testing.T) { + c := copier{ + srcRepo: name.MustParseReference("registry.example.com/foo").Context(), + dstRepo: name.MustParseReference("registry.example.com/bar").Context(), + } + + got, err := c.rename(name.MustParseReference("registry.example.com/foo/sub/repo").Context()) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + want := name.MustParseReference("registry.example.com/bar/sub/repo").Context() + + if want.String() != got.String() { + t.Errorf("%s != %s", want, got) + } +} + +func TestSubtractStringLists(t *testing.T) { + cases := []struct { + minuend []string + subtrahend []string + result []string + }{{ + minuend: []string{"a", "b", "c"}, + subtrahend: []string{"a"}, + result: []string{"b", "c"}, + }, { + minuend: []string{"a", "a", "a"}, + subtrahend: []string{"a", "b"}, + result: []string{}, + }, { + minuend: []string{}, + subtrahend: []string{"a", "b"}, + result: []string{}, + }, { + minuend: []string{"a", "b"}, + subtrahend: []string{}, + result: []string{"a", "b"}, + }} + + for _, tc := range cases { + want, got := tc.result, subtractStringLists(tc.minuend, tc.subtrahend) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("subtracting string lists: %v - %v: (-want +got)\n%s", tc.minuend, tc.subtrahend, diff) + } + } +} + +func TestDiffImages(t *testing.T) { + cases := []struct { + want map[string]google.ManifestInfo + have map[string]google.ManifestInfo + need map[string]google.ManifestInfo + }{{ + // Have everything we need. + want: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "c"}, + }, + }, + have: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "c"}, + }, + }, + need: map[string]google.ManifestInfo{}, + }, { + // Missing image a. + want: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "c", "d"}, + }, + }, + have: map[string]google.ManifestInfo{}, + need: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "c", "d"}, + }, + }, + }, { + // Missing tags "b" and "d" + want: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "c", "d"}, + }, + }, + have: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"c"}, + }, + }, + need: map[string]google.ManifestInfo{ + "a": { + Tags: []string{"b", "d"}, + }, + }, + }, { + // Make sure all properties get copied over. + want: map[string]google.ManifestInfo{ + "a": { + Size: 123, + MediaType: string(types.DockerManifestSchema2), + Created: time.Date(1992, time.January, 7, 6, 40, 00, 5e8, time.UTC), + Uploaded: time.Date(2018, time.November, 29, 4, 13, 30, 5e8, time.UTC), + Tags: []string{"b", "c", "d"}, + }, + }, + have: map[string]google.ManifestInfo{}, + need: map[string]google.ManifestInfo{ + "a": { + Size: 123, + MediaType: string(types.DockerManifestSchema2), + Created: time.Date(1992, time.January, 7, 6, 40, 00, 5e8, time.UTC), + Uploaded: time.Date(2018, time.November, 29, 4, 13, 30, 5e8, time.UTC), + Tags: []string{"b", "c", "d"}, + }, + }, + }} + + for _, tc := range cases { + want, got := tc.need, diffImages(tc.want, tc.have) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("diffing images: %v - %v: (-want +got)\n%s", tc.want, tc.have, diff) + } + } +} + +// Test that our backoff works the way we expect. +func TestBackoff(t *testing.T) { + backoff := GCRBackoff() + + if d := backoff.Step(); d > 10*time.Second { + t.Errorf("Duration too long: %v", d) + } + if d := backoff.Step(); d > 100*time.Second { + t.Errorf("Duration too long: %v", d) + } + if d := backoff.Step(); d > 1000*time.Second { + t.Errorf("Duration too long: %v", d) + } + if s := backoff.Steps; s != 0 { + t.Errorf("backoff.Steps should be 0, got %d", s) + } +} + +func TestErrors(t *testing.T) { + if hasStatusCode(nil, http.StatusOK) { + t.Fatal("nil error should not have any status code") + } + if !hasStatusCode(&transport.Error{StatusCode: http.StatusOK}, http.StatusOK) { + t.Fatal("200 should be 200") + } + if hasStatusCode(&transport.Error{StatusCode: http.StatusOK}, http.StatusNotFound) { + t.Fatal("200 should not be 404") + } + + if isServerError(nil) { + t.Fatal("nil should not be server error") + } + if isServerError(fmt.Errorf("i am a string")) { + t.Fatal("string should not be server error") + } + if !isServerError(&transport.Error{StatusCode: http.StatusServiceUnavailable}) { + t.Fatal("503 should be server error") + } + if isServerError(&transport.Error{StatusCode: http.StatusTooManyRequests}) { + t.Fatal("429 should not be server error") + } +} + +func TestRetryErrors(t *testing.T) { + // We log a warning during retries, so we can tell if something retried by checking logs.Warn. + var b bytes.Buffer + logs.Warn.SetOutput(&b) + + err := backoffErrors(retry.Backoff{ + Duration: 1 * time.Millisecond, + Steps: 3, + }, func() error { + return &transport.Error{StatusCode: http.StatusTooManyRequests} + }) + + if err == nil { + t.Fatal("backoffErrors should return internal err, got nil") + } + var terr *transport.Error + if !errors.As(err, &terr) { + t.Fatalf("backoffErrors should return internal err, got different error: %v", err) + } else if terr.StatusCode != http.StatusTooManyRequests { + t.Fatalf("backoffErrors should return internal err, got different status code: %v", terr.StatusCode) + } + + if b.Len() == 0 { + t.Fatal("backoffErrors didn't log to logs.Warn") + } +} + +func TestBadInputs(t *testing.T) { + t.Parallel() + invalid := "@@@@@@" + + // Create a valid image reference that will fail with not found. + s := httptest.NewServer(http.NotFoundHandler()) + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + valid404 := fmt.Sprintf("%s/some/image", u.Host) + + ctx := context.Background() + + for _, tc := range []struct { + desc string + err error + }{ + {"Copy(invalid, invalid)", Copy(invalid, invalid)}, + {"Copy(404, invalid)", Copy(valid404, invalid)}, + {"Copy(404, 404)", Copy(valid404, valid404)}, + {"CopyRepository(invalid, invalid)", CopyRepository(ctx, invalid, invalid)}, + {"CopyRepository(404, invalid)", CopyRepository(ctx, valid404, invalid)}, + {"CopyRepository(404, 404)", CopyRepository(ctx, valid404, valid404, WithJobs(1))}, + } { + if tc.err == nil { + t.Errorf("%s: expected err, got nil", tc.desc) + } + } +} diff --git a/pkg/gcrane/doc.go b/pkg/gcrane/doc.go new file mode 100644 index 0000000..63a1bbb --- /dev/null +++ b/pkg/gcrane/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gcrane holds libraries used to implement the gcrane CLI. +package gcrane diff --git a/pkg/gcrane/options.go b/pkg/gcrane/options.go new file mode 100644 index 0000000..9d34c7d --- /dev/null +++ b/pkg/gcrane/options.go @@ -0,0 +1,122 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcrane + +import ( + "context" + "net/http" + "runtime" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Option is a functional option for gcrane operations. +type Option func(*options) + +type options struct { + jobs int + remote []remote.Option + google []google.Option + crane []crane.Option +} + +func makeOptions(opts ...Option) *options { + o := &options{ + jobs: runtime.GOMAXPROCS(0), + remote: []remote.Option{ + remote.WithAuthFromKeychain(Keychain), + }, + google: []google.Option{ + google.WithAuthFromKeychain(Keychain), + }, + crane: []crane.Option{ + crane.WithAuthFromKeychain(Keychain), + }, + } + + for _, option := range opts { + option(o) + } + + return o +} + +// WithJobs sets the number of concurrent jobs to run. +// +// The default number of jobs is GOMAXPROCS. +func WithJobs(jobs int) Option { + return func(o *options) { + o.jobs = jobs + } +} + +// WithTransport is a functional option for overriding the default transport +// for remote operations. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) { + o.remote = append(o.remote, remote.WithTransport(t)) + o.google = append(o.google, google.WithTransport(t)) + o.crane = append(o.crane, crane.WithTransport(t)) + } +} + +// WithUserAgent adds the given string to the User-Agent header for any HTTP +// requests. +func WithUserAgent(ua string) Option { + return func(o *options) { + o.remote = append(o.remote, remote.WithUserAgent(ua)) + o.google = append(o.google, google.WithUserAgent(ua)) + o.crane = append(o.crane, crane.WithUserAgent(ua)) + } +} + +// WithContext is a functional option for setting the context. +func WithContext(ctx context.Context) Option { + return func(o *options) { + o.remote = append(o.remote, remote.WithContext(ctx)) + o.google = append(o.google, google.WithContext(ctx)) + o.crane = append(o.crane, crane.WithContext(ctx)) + } +} + +// WithKeychain is a functional option for overriding the default +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// By default, gcrane will use gcrane.Keychain. +func WithKeychain(keys authn.Keychain) Option { + return func(o *options) { + // Replace the default keychain at position 0. + o.remote[0] = remote.WithAuthFromKeychain(keys) + o.google[0] = google.WithAuthFromKeychain(keys) + o.crane[0] = crane.WithAuthFromKeychain(keys) + } +} + +// WithAuth is a functional option for overriding the default authenticator +// for remote operations. +// +// By default, gcrane will use gcrane.Keychain. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) { + // Replace the default keychain at position 0. + o.remote[0] = remote.WithAuth(auth) + o.google[0] = google.WithAuth(auth) + o.crane[0] = crane.WithAuth(auth) + } +} diff --git a/pkg/gcrane/options_test.go b/pkg/gcrane/options_test.go new file mode 100644 index 0000000..c2ce2f9 --- /dev/null +++ b/pkg/gcrane/options_test.go @@ -0,0 +1,58 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcrane + +import ( + "context" + "net/http" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" +) + +func TestOptions(t *testing.T) { + o := makeOptions() + if len(o.remote) != 1 { + t.Errorf("remote should default to Keychain") + } + if len(o.crane) != 1 { + t.Errorf("crane should default to Keychain") + } + if len(o.google) != 1 { + t.Errorf("google should default to Keychain") + } + + o = makeOptions(WithAuth(authn.Anonymous), WithKeychain(authn.DefaultKeychain)) + if len(o.remote) != 1 { + t.Errorf("WithKeychain should replace remote[0]") + } + if len(o.crane) != 1 { + t.Errorf("WithKeychain should replace crane[0]") + } + if len(o.google) != 1 { + t.Errorf("WithKeychain should replace google[0]") + } + + o = makeOptions(WithTransport(http.DefaultTransport), WithUserAgent("hi"), WithContext(context.TODO())) + if len(o.remote) != 4 { + t.Errorf("wrong number of options: %d", len(o.remote)) + } + if len(o.crane) != 4 { + t.Errorf("wrong number of options: %d", len(o.crane)) + } + if len(o.google) != 4 { + t.Errorf("wrong number of options: %d", len(o.google)) + } +} diff --git a/pkg/legacy/config.go b/pkg/legacy/config.go new file mode 100644 index 0000000..3364bec --- /dev/null +++ b/pkg/legacy/config.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package legacy + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// LayerConfigFile is the configuration file that holds the metadata describing +// a v1 layer. See: +// https://github.com/moby/moby/blob/master/image/spec/v1.md +type LayerConfigFile struct { + v1.ConfigFile + + ContainerConfig v1.Config `json:"container_config,omitempty"` + + ID string `json:"id,omitempty"` + Parent string `json:"parent,omitempty"` + Throwaway bool `json:"throwaway,omitempty"` + Comment string `json:"comment,omitempty"` +} diff --git a/pkg/legacy/doc.go b/pkg/legacy/doc.go new file mode 100644 index 0000000..1d16688 --- /dev/null +++ b/pkg/legacy/doc.go @@ -0,0 +1,18 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package legacy provides functionality to work with docker images in the v1 +// format. +// See: https://github.com/moby/moby/blob/master/image/spec/v1.md +package legacy diff --git a/pkg/legacy/tarball/README.md b/pkg/legacy/tarball/README.md new file mode 100644 index 0000000..90b88c7 --- /dev/null +++ b/pkg/legacy/tarball/README.md @@ -0,0 +1,6 @@ +# `legacy/tarball` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/legacy/tarball?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/legacy/tarball) + +This package implements support for writing legacy tarballs, as described +[here](https://github.com/moby/moby/blob/749d90e10f989802638ae542daf54257f3bf71f2/image/spec/v1.2.md#combined-image-json--filesystem-changeset-format). diff --git a/pkg/legacy/tarball/doc.go b/pkg/legacy/tarball/doc.go new file mode 100644 index 0000000..62684d6 --- /dev/null +++ b/pkg/legacy/tarball/doc.go @@ -0,0 +1,18 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarball provides facilities for writing v1 docker images +// (https://github.com/moby/moby/blob/master/image/spec/v1.md) from/to a tarball +// on-disk. +package tarball diff --git a/pkg/legacy/tarball/write.go b/pkg/legacy/tarball/write.go new file mode 100644 index 0000000..e3f173c --- /dev/null +++ b/pkg/legacy/tarball/write.go @@ -0,0 +1,374 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/google/go-containerregistry/pkg/legacy" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// repositoriesTarDescriptor represents the repositories file inside a `docker save` tarball. +type repositoriesTarDescriptor map[string]map[string]string + +// v1Layer represents a layer with metadata needed by the v1 image spec https://github.com/moby/moby/blob/master/image/spec/v1.md. +type v1Layer struct { + // config is the layer metadata. + config *legacy.LayerConfigFile + // layer is the v1.Layer object this v1Layer represents. + layer v1.Layer +} + +// json returns the raw bytes of the json metadata of the given v1Layer. +func (l *v1Layer) json() ([]byte, error) { + return json.Marshal(l.config) +} + +// version returns the raw bytes of the "VERSION" file of the given v1Layer. +func (l *v1Layer) version() []byte { + return []byte("1.0") +} + +// v1LayerID computes the v1 image format layer id for the given v1.Layer with the given v1 parent ID and raw image config. +func v1LayerID(layer v1.Layer, parentID string, rawConfig []byte) (string, error) { + d, err := layer.Digest() + if err != nil { + return "", fmt.Errorf("unable to get layer digest to generate v1 layer ID: %w", err) + } + s := fmt.Sprintf("%s %s", d.Hex, parentID) + if len(rawConfig) != 0 { + s = fmt.Sprintf("%s %s", s, string(rawConfig)) + } + + h, _, _ := v1.SHA256(strings.NewReader(s)) + return h.Hex, nil +} + +// newTopV1Layer creates a new v1Layer for a layer other than the top layer in a v1 image tarball. +func newV1Layer(layer v1.Layer, parent *v1Layer, history v1.History) (*v1Layer, error) { + parentID := "" + if parent != nil { + parentID = parent.config.ID + } + id, err := v1LayerID(layer, parentID, nil) + if err != nil { + return nil, fmt.Errorf("unable to generate v1 layer ID: %w", err) + } + result := &v1Layer{ + layer: layer, + config: &legacy.LayerConfigFile{ + ConfigFile: v1.ConfigFile{ + Created: history.Created, + Author: history.Author, + }, + ContainerConfig: v1.Config{ + Cmd: []string{history.CreatedBy}, + }, + ID: id, + Parent: parentID, + Throwaway: history.EmptyLayer, + Comment: history.Comment, + }, + } + return result, nil +} + +// newTopV1Layer creates a new v1Layer for the top layer in a v1 image tarball. +func newTopV1Layer(layer v1.Layer, parent *v1Layer, history v1.History, imgConfig *v1.ConfigFile, rawConfig []byte) (*v1Layer, error) { + result, err := newV1Layer(layer, parent, history) + if err != nil { + return nil, err + } + id, err := v1LayerID(layer, result.config.Parent, rawConfig) + if err != nil { + return nil, fmt.Errorf("unable to generate v1 layer ID for top layer: %w", err) + } + result.config.ID = id + result.config.Architecture = imgConfig.Architecture + result.config.Container = imgConfig.Container + result.config.DockerVersion = imgConfig.DockerVersion + result.config.OS = imgConfig.OS + result.config.Config = imgConfig.Config + result.config.Created = imgConfig.Created + return result, nil +} + +// splitTag splits the given tagged image name /: +// into / and . +func splitTag(name string) (string, string) { + // Split on ":" + parts := strings.Split(name, ":") + // Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation. + if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], "/") { + base := strings.Join(parts[:len(parts)-1], ":") + tag := parts[len(parts)-1] + return base, tag + } + return name, "" +} + +// addTags adds the given image tags to the given "repositories" file descriptor in a v1 image tarball. +func addTags(repos repositoriesTarDescriptor, tags []string, topLayerID string) { + for _, t := range tags { + base, tag := splitTag(t) + tagToID, ok := repos[base] + if !ok { + tagToID = make(map[string]string) + repos[base] = tagToID + } + tagToID[tag] = topLayerID + } +} + +// updateLayerSources updates the given layer digest to descriptor map with the descriptor of the given layer in the given image if it's an undistributable layer. +func updateLayerSources(layerSources map[v1.Hash]v1.Descriptor, layer v1.Layer, img v1.Image) error { + d, err := layer.Digest() + if err != nil { + return err + } + // Add to LayerSources if it's a foreign layer. + desc, err := partial.BlobDescriptor(img, d) + if err != nil { + return err + } + if !desc.MediaType.IsDistributable() { + diffid, err := partial.BlobToDiffID(img, d) + if err != nil { + return err + } + layerSources[diffid] = *desc + } + return nil +} + +// Write is a wrapper to write a single image in V1 format and tag to a tarball. +func Write(ref name.Reference, img v1.Image, w io.Writer) error { + return MultiWrite(map[name.Reference]v1.Image{ref: img}, w) +} + +// filterEmpty filters out the history corresponding to empty layers from the +// given history. +func filterEmpty(h []v1.History) []v1.History { + result := []v1.History{} + for _, i := range h { + if i.EmptyLayer { + continue + } + result = append(result, i) + } + return result +} + +// MultiWrite writes the contents of each image to the provided reader, in the V1 image tarball format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One repositories file mapping from the image / to to the id of the top most layer. +// For every layer, a directory named with the layer ID is created with the following contents: +// +// layer.tar - The uncompressed layer tarball. +// .json- Layer metadata json. +// VERSION- Schema version string. Always set to "1.0". +// +// One file for the config blob, named after its SHA. +func MultiWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error { + tf := tar.NewWriter(w) + defer tf.Close() + + sortedImages, imageToTags := dedupRefToImage(refToImage) + var m tarball.Manifest + repos := make(repositoriesTarDescriptor) + + seenLayerIDs := make(map[string]struct{}) + for _, img := range sortedImages { + tags := imageToTags[img] + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgFileName := fmt.Sprintf("%s.json", cfgName.Hex) + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := writeTarEntry(tf, cfgFileName, bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil { + return err + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + + // Store foreign layer info. + layerSources := make(map[v1.Hash]v1.Descriptor) + + // Write the layers. + layers, err := img.Layers() + if err != nil { + return err + } + history := filterEmpty(cfg.History) + // Create a blank config history if the config didn't have a history. + if len(history) == 0 && len(layers) != 0 { + history = make([]v1.History, len(layers)) + } else if len(layers) != len(history) { + return fmt.Errorf("image config had layer history which did not match the number of layers, got len(history)=%d, len(layers)=%d, want len(history)=len(layers)", len(history), len(layers)) + } + layerFiles := make([]string, len(layers)) + var prev *v1Layer + for i, l := range layers { + if err := updateLayerSources(layerSources, l, img); err != nil { + return fmt.Errorf("unable to update image metadata to include undistributable layer source information: %w", err) + } + var cur *v1Layer + if i < (len(layers) - 1) { + cur, err = newV1Layer(l, prev, history[i]) + } else { + cur, err = newTopV1Layer(l, prev, history[i], cfg, cfgBlob) + } + if err != nil { + return err + } + layerFiles[i] = fmt.Sprintf("%s/layer.tar", cur.config.ID) + if _, ok := seenLayerIDs[cur.config.ID]; ok { + prev = cur + continue + } + seenLayerIDs[cur.config.ID] = struct{}{} + + // If the v1.Layer implements UncompressedSize efficiently, use that + // for the tar header. Otherwise, this iterates over Uncompressed(). + // NOTE: If using a streaming layer, this may consume the layer. + size, err := partial.UncompressedSize(l) + if err != nil { + return err + } + u, err := l.Uncompressed() + if err != nil { + return err + } + defer u.Close() + if err := writeTarEntry(tf, layerFiles[i], u, size); err != nil { + return err + } + + j, err := cur.json() + if err != nil { + return err + } + if err := writeTarEntry(tf, fmt.Sprintf("%s/json", cur.config.ID), bytes.NewReader(j), int64(len(j))); err != nil { + return err + } + v := cur.version() + if err := writeTarEntry(tf, fmt.Sprintf("%s/VERSION", cur.config.ID), bytes.NewReader(v), int64(len(v))); err != nil { + return err + } + prev = cur + } + + // Generate the tar descriptor and write it. + m = append(m, tarball.Descriptor{ + Config: cfgFileName, + RepoTags: tags, + Layers: layerFiles, + LayerSources: layerSources, + }) + // prev should be the top layer here. Use it to add the image tags + // to the tarball repositories file. + addTags(repos, tags, prev.config.ID) + } + + mBytes, err := json.Marshal(m) + if err != nil { + return err + } + + if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(mBytes), int64(len(mBytes))); err != nil { + return err + } + reposBytes, err := json.Marshal(&repos) + if err != nil { + return err + } + if err := writeTarEntry(tf, "repositories", bytes.NewReader(reposBytes), int64(len(reposBytes))); err != nil { + return err + } + return nil +} + +func dedupRefToImage(refToImage map[name.Reference]v1.Image) ([]v1.Image, map[v1.Image][]string) { + imageToTags := make(map[v1.Image][]string) + + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; ok && tags != nil { + imageToTags[img] = append(tags, tag.String()) + } else { + imageToTags[img] = []string{tag.String()} + } + } else { + if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } + } + } + + // Force specific order on tags + imgs := []v1.Image{} + for img, tags := range imageToTags { + sort.Strings(tags) + imgs = append(imgs, img) + } + + sort.Slice(imgs, func(i, j int) bool { + cfI, err := imgs[i].ConfigName() + if err != nil { + return false + } + cfJ, err := imgs[j].ConfigName() + if err != nil { + return false + } + return cfI.Hex < cfJ.Hex + }) + + return imgs, imageToTags +} + +// Writes a file to the provided writer with a corresponding tar header +func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error { + hdr := &tar.Header{ + Mode: 0644, + Typeflag: tar.TypeReg, + Size: size, + Name: path, + } + if err := tf.WriteHeader(hdr); err != nil { + return err + } + _, err := io.Copy(tf, r) + return err +} diff --git a/pkg/legacy/tarball/write_test.go b/pkg/legacy/tarball/write_test.go new file mode 100644 index 0000000..33e658c --- /dev/null +++ b/pkg/legacy/tarball/write_test.go @@ -0,0 +1,615 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestWrite(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + if err := Write(tag, randImage, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + + // Make sure the image is valid and can be loaded. + // Load it both by nil and by its name. + for _, it := range []*name.Tag{nil, &tag} { + tarImage, err := tarball.ImageFromPath(fp.Name(), it) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + if err := compare.Images(randImage, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } + + // Try loading a different tag, it should error. + fakeTag, err := name.NewTag("gcr.io/notthistag:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error generating tag: %v", err) + } + if _, err := tarball.ImageFromPath(fp.Name(), &fakeTag); err == nil { + t.Errorf("Expected error loading tag %v from image", fakeTag) + } +} + +func TestMultiWriteSameImage(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image.") + } + + // Make two tags that point to the random image above. + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1.") + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2.") + } + dig3, err := name.NewDigest("gcr.io/baz/baz@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test dig3.") + } + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage + refToImage[tag2] = randImage + refToImage[dig3] = randImage + + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + + // Write the images with both tags to the tarball + if err := MultiWrite(refToImage, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + if err := compare.Images(randImage, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } +} + +func TestMultiWriteDifferentImages(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage1, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 1: %v", err) + } + + // Make another random image + randImage2, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 2: %v", err) + } + + // Make another random image + randImage3, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 3: %v", err) + } + + // Create two tags, one pointing to each image created. + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1: %v", err) + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2: %v", err) + } + dig3, err := name.NewDigest("gcr.io/baz/baz@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test dig3: %v", err) + } + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage1 + refToImage[tag2] = randImage2 + refToImage[dig3] = randImage3 + + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + + // Write both images to the tarball. + if err := MultiWrite(refToImage, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref, img := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + if err := compare.Images(img, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } +} + +func TestWriteForeignLayers(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 1) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + randLayer, err := random.Layer(512, types.DockerForeignLayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + img, err := mutate.Append(randImage, mutate.Addendum{ + Layer: randLayer, + URLs: []string{ + "example.com", + }, + }) + if err != nil { + t.Fatalf("Unable to mutate image to add foreign layer: %v", err) + } + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + if err := Write(tag, img, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Fatalf("validate.Image(): %v", err) + } + + m, err := tarImage.Manifest() + if err != nil { + t.Fatal(err) + } + + if got, want := m.Layers[1].MediaType, types.DockerForeignLayer; got != want { + t.Errorf("Wrong MediaType: %s != %s", got, want) + } + if got, want := m.Layers[1].URLs[0], "example.com"; got != want { + t.Errorf("Wrong URLs: %s != %s", got, want) + } +} + +func TestMultiWriteNoHistory(t *testing.T) { + // Make a random image. + img, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + cfg, err := img.ConfigFile() + if err != nil { + t.Fatalf("Error getting image config: %v", err) + } + // Blank out the layer history. + cfg.History = nil + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + if err := Write(tag, img, fp); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Fatalf("validate.Image(): %v", err) + } +} + +func TestMultiWriteHistoryEmptyLayers(t *testing.T) { + // Build a history for 2 layers that is interspersed with empty layer + // history. + h := []v1.History{ + {EmptyLayer: true}, + {EmptyLayer: false}, + {EmptyLayer: true}, + {EmptyLayer: false}, + {EmptyLayer: true}, + } + // Make a random image with the number of non-empty layers from the history + // above. + img, err := random.Image(256, int64(len(filterEmpty(h)))) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + cfg, err := img.ConfigFile() + if err != nil { + t.Fatalf("Error getting image config: %v", err) + } + // Override the config history with our custom built history that includes + // history for empty layers. + cfg.History = h + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + if err := Write(tag, img, fp); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Fatalf("validate.Image(): %v", err) + } +} + +func TestMultiWriteMismatchedHistory(t *testing.T) { + // Make a random image + img, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + cfg, err := img.ConfigFile() + if err != nil { + t.Fatalf("Error getting image config: %v", err) + } + + // Set the history such that number of history entries != layers. This + // should trigger an error during the image write. + cfg.History = make([]v1.History, 1) + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatalf("mutate.ConfigFile() = %v", err) + } + + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + err = Write(tag, img, fp) + if err == nil { + t.Fatal("Unexpected success writing tarball, got nil, want error.") + } + want := "image config had layer history which did not match the number of layers" + if !strings.Contains(err.Error(), want) { + t.Errorf("Got unexpected error when writing image with mismatched history & layer, got %v, want substring %q", err, want) + } +} + +type fastSizeLayer struct { + v1.Layer + size int64 + called bool +} + +func (l *fastSizeLayer) UncompressedSize() (int64, error) { + l.called = true + return l.size, nil +} + +func TestUncompressedSize(t *testing.T) { + // Make a random image + img, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + + rand, err := random.Layer(1000, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + + size, err := partial.UncompressedSize(rand) + if err != nil { + t.Fatal(err) + } + + l := &fastSizeLayer{Layer: rand, size: size} + + img, err = mutate.AppendLayers(img, l) + if err != nil { + t.Fatal(err) + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + if err := Write(tag, img, fp); err != nil { + t.Fatalf("Write(): %v", err) + } + if !l.called { + t.Errorf("expected UncompressedSize to be called, but it wasn't") + } +} + +// TestWriteSharedLayers tests that writing a tarball of multiple images that +// share some layers only writes those shared layers once. +func TestWriteSharedLayers(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file: %v", err) + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + const baseImageLayerCount = 8 + + // Make a random image + baseImage, err := random.Image(256, baseImageLayerCount) + if err != nil { + t.Fatalf("Error creating base image: %v", err) + } + + // Make another random image + randLayer, err := random.Layer(256, types.DockerLayer) + if err != nil { + t.Fatalf("Error creating random layer %v", err) + } + extendedImage, err := mutate.Append(baseImage, mutate.Addendum{ + Layer: randLayer, + }) + if err != nil { + t.Fatalf("Error mutating base image %v", err) + } + + // Create two tags, one pointing to each image created. + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1: %v", err) + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2: %v", err) + } + refToImage := map[name.Reference]v1.Image{ + tag1: baseImage, + tag2: extendedImage, + } + + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + + // Write both images to the tarball. + if err := MultiWrite(refToImage, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref, img := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + if err := compare.Images(img, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } + + wantIDs := make(map[string]struct{}) + ids, err := v1LayerIDs(baseImage) + if err != nil { + t.Fatalf("Error getting base image IDs: %v", err) + } + for _, id := range ids { + wantIDs[id] = struct{}{} + } + ids, err = v1LayerIDs(extendedImage) + if err != nil { + t.Fatalf("Error getting extended image IDs: %v", err) + } + for _, id := range ids { + wantIDs[id] = struct{}{} + } + + // base + extended layer + different top base layer + if len(wantIDs) != baseImageLayerCount+2 { + t.Errorf("Expected to have %d unique layer IDs but have %d", baseImageLayerCount+2, len(wantIDs)) + } + + const layerFileName = "layer.tar" + r := tar.NewReader(fp) + for { + hdr, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("Get tar header: %v", err) + } + if filepath.Base(hdr.Name) == layerFileName { + id := filepath.Dir(hdr.Name) + if _, ok := wantIDs[id]; ok { + delete(wantIDs, id) + } else { + t.Errorf("Found unwanted layer with ID %q", id) + } + } + } + if len(wantIDs) != 0 { + for id := range wantIDs { + t.Errorf("Expected to find layer with ID %q but it didn't exist", id) + } + } +} + +func v1LayerIDs(img v1.Image) ([]string, error) { + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("get layers: %w", err) + } + ids := make([]string, len(layers)) + parentID := "" + for i, layer := range layers { + var rawCfg []byte + if i == len(layers)-1 { + rawCfg, err = img.RawConfigFile() + if err != nil { + return nil, fmt.Errorf("get raw config file: %w", err) + } + } + id, err := v1LayerID(layer, parentID, rawCfg) + if err != nil { + return nil, fmt.Errorf("get v1 layer ID: %w", err) + } + + ids[i] = id + parentID = id + } + return ids, nil +} diff --git a/pkg/logs/logs.go b/pkg/logs/logs.go new file mode 100644 index 0000000..a5d25b1 --- /dev/null +++ b/pkg/logs/logs.go @@ -0,0 +1,39 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logs exposes the loggers used by this library. +package logs + +import ( + "io" + "log" +) + +var ( + // Warn is used to log non-fatal errors. + Warn = log.New(io.Discard, "", log.LstdFlags) + + // Progress is used to log notable, successful events. + Progress = log.New(io.Discard, "", log.LstdFlags) + + // Debug is used to log information that is useful for debugging. + Debug = log.New(io.Discard, "", log.LstdFlags) +) + +// Enabled checks to see if the logger's writer is set to something other +// than io.Discard. This allows callers to avoid expensive operations +// that will end up in /dev/null anyway. +func Enabled(l *log.Logger) bool { + return l.Writer() != io.Discard +} diff --git a/pkg/name/README.md b/pkg/name/README.md new file mode 100644 index 0000000..4889b84 --- /dev/null +++ b/pkg/name/README.md @@ -0,0 +1,3 @@ +# `name` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/name?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/name) diff --git a/pkg/name/check.go b/pkg/name/check.go new file mode 100644 index 0000000..e9a240a --- /dev/null +++ b/pkg/name/check.go @@ -0,0 +1,43 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "strings" + "unicode/utf8" +) + +// stripRunesFn returns a function which returns -1 (i.e. a value which +// signals deletion in strings.Map) for runes in 'runes', and the rune otherwise. +func stripRunesFn(runes string) func(rune) rune { + return func(r rune) rune { + if strings.ContainsRune(runes, r) { + return -1 + } + return r + } +} + +// checkElement checks a given named element matches character and length restrictions. +// Returns true if the given element adheres to the given restrictions, false otherwise. +func checkElement(name, element, allowedRunes string, minRunes, maxRunes int) error { + numRunes := utf8.RuneCountInString(element) + if (numRunes < minRunes) || (maxRunes < numRunes) { + return newErrBadName("%s must be between %d and %d characters in length: %s", name, minRunes, maxRunes, element) + } else if len(strings.Map(stripRunesFn(allowedRunes), element)) != 0 { + return newErrBadName("%s can only contain the characters `%s`: %s", name, allowedRunes, element) + } + return nil +} diff --git a/pkg/name/digest.go b/pkg/name/digest.go new file mode 100644 index 0000000..c049c1e --- /dev/null +++ b/pkg/name/digest.go @@ -0,0 +1,94 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + // nolint: depguard + _ "crypto/sha256" // Recommended by go-digest. + "strings" + + "github.com/opencontainers/go-digest" +) + +const digestDelim = "@" + +// Digest stores a digest name in a structured form. +type Digest struct { + Repository + digest string + original string +} + +// Ensure Digest implements Reference +var _ Reference = (*Digest)(nil) + +// Context implements Reference. +func (d Digest) Context() Repository { + return d.Repository +} + +// Identifier implements Reference. +func (d Digest) Identifier() string { + return d.DigestStr() +} + +// DigestStr returns the digest component of the Digest. +func (d Digest) DigestStr() string { + return d.digest +} + +// Name returns the name from which the Digest was derived. +func (d Digest) Name() string { + return d.Repository.Name() + digestDelim + d.DigestStr() +} + +// String returns the original input string. +func (d Digest) String() string { + return d.original +} + +// NewDigest returns a new Digest representing the given name. +func NewDigest(name string, opts ...Option) (Digest, error) { + // Split on "@" + parts := strings.Split(name, digestDelim) + if len(parts) != 2 { + return Digest{}, newErrBadName("a digest must contain exactly one '@' separator (e.g. registry/repository@digest) saw: %s", name) + } + base := parts[0] + dig := parts[1] + prefix := digest.Canonical.String() + ":" + if !strings.HasPrefix(dig, prefix) { + return Digest{}, newErrBadName("unsupported digest algorithm: %s", dig) + } + hex := strings.TrimPrefix(dig, prefix) + if err := digest.Canonical.Validate(hex); err != nil { + return Digest{}, err + } + + tag, err := NewTag(base, opts...) + if err == nil { + base = tag.Repository.Name() + } + + repo, err := NewRepository(base, opts...) + if err != nil { + return Digest{}, err + } + return Digest{ + Repository: repo, + digest: dig, + original: name, + }, nil +} diff --git a/pkg/name/digest_test.go b/pkg/name/digest_test.go new file mode 100644 index 0000000..85775cc --- /dev/null +++ b/pkg/name/digest_test.go @@ -0,0 +1,152 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "path" + "strings" + "testing" +) + +const validDigest = "sha256:deadb33fdeadb33fdeadb33fdeadb33fdeadb33fdeadb33fdeadb33fdeadb33f" + +var goodStrictValidationDigestNames = []string{ + "gcr.io/g-convoy/hello-world@" + validDigest, + "gcr.io/google.com/project-id/hello-world@" + validDigest, + "us.gcr.io/project-id/sub-repo@" + validDigest, + "example.text/foo/bar@" + validDigest, +} + +var goodStrictValidationTagDigestNames = []string{ + "example.text/foo/bar:latest@" + validDigest, + "example.text:8443/foo/bar:latest@" + validDigest, + "example.text/foo/bar:v1.0.0-alpine@" + validDigest, +} + +var goodWeakValidationDigestNames = []string{ + "namespace/pathcomponent/image@" + validDigest, + "library/ubuntu@" + validDigest, +} + +var goodWeakValidationTagDigestNames = []string{ + "nginx:latest@" + validDigest, + "library/nginx:latest@" + validDigest, +} + +var badDigestNames = []string{ + "gcr.io/project-id/unknown-alg@unknown:abc123", + "gcr.io/project-id/wrong-length@sha256:d34db33fd34db33f", + "gcr.io/project-id/missing-digest@", + // https://github.com/google/go-containerregistry/issues/1394 + "repo@sha256:" + strings.Repeat(":", 64), + "repo@sha256:" + strings.Repeat("sh", 32), + "repo@sha256:" + validDigest + "@" + validDigest, +} + +func TestNewDigestStrictValidation(t *testing.T) { + t.Parallel() + + for _, name := range goodStrictValidationDigestNames { + if digest, err := NewDigest(name, StrictValidation); err != nil { + t.Errorf("`%s` should be a valid Digest name, got error: %v", name, err) + } else if digest.Name() != name { + t.Errorf("`%v` .Name() should reproduce the original name. Wanted: %s Got: %s", digest, name, digest.Name()) + } + } + + for _, name := range goodStrictValidationTagDigestNames { + if _, err := NewDigest(name, StrictValidation); err != nil { + t.Errorf("`%s` should be a valid Digest name, got error: %v", name, err) + } + } + + for _, name := range append(goodWeakValidationDigestNames, badDigestNames...) { + if repo, err := NewDigest(name, StrictValidation); err == nil { + t.Errorf("`%s` should be an invalid Digest name, got Digest: %#v", name, repo) + } + } +} + +func TestNewDigest(t *testing.T) { + t.Parallel() + + for _, name := range append(goodStrictValidationDigestNames, append(goodWeakValidationDigestNames, goodWeakValidationTagDigestNames...)...) { + if _, err := NewDigest(name, WeakValidation); err != nil { + t.Errorf("`%s` should be a valid Digest name, got error: %v", name, err) + } + } + + for _, name := range badDigestNames { + if repo, err := NewDigest(name, WeakValidation); err == nil { + t.Errorf("`%s` should be an invalid Digest name, got Digest: %#v", name, repo) + } + } +} + +func TestDigestComponents(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepository := "project-id/image" + fullRepo := path.Join(testRegistry, testRepository) + + digestNameStr := testRegistry + "/" + testRepository + "@" + validDigest + digest, err := NewDigest(digestNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Digest name, got error: %v", digestNameStr, err) + } + + if got := digest.String(); got != digestNameStr { + t.Errorf("String() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, digestNameStr, got) + } + if got := digest.Identifier(); got != validDigest { + t.Errorf("Identifier() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, validDigest, got) + } + actualRegistry := digest.RegistryStr() + if actualRegistry != testRegistry { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, testRegistry, actualRegistry) + } + actualRepository := digest.RepositoryStr() + if actualRepository != testRepository { + t.Errorf("RepositoryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, testRepository, actualRepository) + } + contextRepo := digest.Context().String() + if contextRepo != fullRepo { + t.Errorf("Context().String() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, fullRepo, contextRepo) + } + actualDigest := digest.DigestStr() + if actualDigest != validDigest { + t.Errorf("DigestStr() was incorrect for %v. Wanted: `%s` Got: `%s`", digest, validDigest, actualDigest) + } +} + +func TestDigestScopes(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepo := "project-id/image" + testAction := "pull" + + expectedScope := strings.Join([]string{"repository", testRepo, testAction}, ":") + + digestNameStr := testRegistry + "/" + testRepo + "@" + validDigest + digest, err := NewDigest(digestNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Digest name, got error: %v", digestNameStr, err) + } + + actualScope := digest.Scope(testAction) + if actualScope != expectedScope { + t.Errorf("scope was incorrect for %v. Wanted: `%s` Got: `%s`", digest, expectedScope, actualScope) + } +} diff --git a/pkg/name/doc.go b/pkg/name/doc.go new file mode 100644 index 0000000..b294794 --- /dev/null +++ b/pkg/name/doc.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package name defines structured types for representing image references. +// +// What's in a name? For image references, not nearly enough! +// +// Image references look a lot like URLs, but they differ in that they don't +// contain the scheme (http or https), they can end with a :tag or a @digest +// (the latter being validated), and they perform defaulting for missing +// components. +// +// Since image references don't contain the scheme, we do our best to infer +// if we use http or https from the given hostname. We allow http fallback for +// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in +// ".local", or is in the "private" address space per RFC 1918. For everything +// else, we assume https only. To override this heuristic, use the Insecure +// option. +// +// Image references with a digest signal to us that we should verify the content +// of the image matches the digest. E.g. when pulling a Digest reference, we'll +// calculate the sha256 of the manifest returned by the registry and error out +// if it doesn't match what we asked for. +// +// For defaulting, we interpret "ubuntu" as +// "index.docker.io/library/ubuntu:latest" because we add the missing repo +// "library", the missing registry "index.docker.io", and the missing tag +// "latest". To disable this defaulting, use the StrictValidation option. This +// is useful e.g. to only allow image references that explicitly set a tag or +// digest, so that you don't accidentally pull "latest". +package name diff --git a/pkg/name/errors.go b/pkg/name/errors.go new file mode 100644 index 0000000..bf004ff --- /dev/null +++ b/pkg/name/errors.go @@ -0,0 +1,48 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "errors" + "fmt" +) + +// ErrBadName is an error for when a bad docker name is supplied. +type ErrBadName struct { + info string +} + +func (e *ErrBadName) Error() string { + return e.info +} + +// Is reports whether target is an error of type ErrBadName +func (e *ErrBadName) Is(target error) bool { + var berr *ErrBadName + return errors.As(target, &berr) +} + +// newErrBadName returns a ErrBadName which returns the given formatted string from Error(). +func newErrBadName(fmtStr string, args ...any) *ErrBadName { + return &ErrBadName{fmt.Sprintf(fmtStr, args...)} +} + +// IsErrBadName returns true if the given error is an ErrBadName. +// +// Deprecated: Use errors.Is. +func IsErrBadName(err error) bool { + var berr *ErrBadName + return errors.As(err, &berr) +} diff --git a/pkg/name/errors_test.go b/pkg/name/errors_test.go new file mode 100644 index 0000000..a9ea4da --- /dev/null +++ b/pkg/name/errors_test.go @@ -0,0 +1,37 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "errors" + "testing" +) + +func TestBadName(t *testing.T) { + _, err := ParseReference("@@") + if !IsErrBadName(err) { + t.Errorf("Not an ErrBadName: %v", err) + } + var berr *ErrBadName + if !errors.As(err, &berr) { + t.Errorf("Not an ErrBadName using errors.As: %v", err) + } + if err.Error() != "could not parse reference: @@" { + t.Errorf("Unexpected string: %v", err) + } + if !errors.Is(err, &ErrBadName{}) { + t.Errorf("Not an ErrBadName using errors.Is: %v", err) + } +} diff --git a/pkg/name/internal/must_test.go b/pkg/name/internal/must_test.go new file mode 100644 index 0000000..d77d3ec --- /dev/null +++ b/pkg/name/internal/must_test.go @@ -0,0 +1,27 @@ +//go:build compile +// +build compile + +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "strings" + + "github.com/google/go-containerregistry/pkg/name" +) + +// This shouldn't compile. +var _ = name.MustParseReference(strings.Join([]string{"valid", "string"}, "/")) diff --git a/pkg/name/internal/must_test.sh b/pkg/name/internal/must_test.sh new file mode 100755 index 0000000..91a4fd1 --- /dev/null +++ b/pkg/name/internal/must_test.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# Trying to compile without the build tag should work. +go test ./pkg/name/internal + +# Actually trying to compile should fail. +go test -tags=compile ./pkg/name/internal 2>&1 > /dev/null +if [[ $? -eq 0 ]]; then + echo "pkg/name/internal test compiled successfully, expected failure" + exit 1 +fi +echo "pkg/name/internal test successfully did not compile" diff --git a/pkg/name/options.go b/pkg/name/options.go new file mode 100644 index 0000000..d14fedc --- /dev/null +++ b/pkg/name/options.go @@ -0,0 +1,83 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +const ( + // DefaultRegistry is the registry name that will be used if no registry + // provided and the default is not overridden. + DefaultRegistry = "index.docker.io" + defaultRegistryAlias = "docker.io" + + // DefaultTag is the tag name that will be used if no tag provided and the + // default is not overridden. + DefaultTag = "latest" +) + +type options struct { + strict bool // weak by default + insecure bool // secure by default + defaultRegistry string + defaultTag string +} + +func makeOptions(opts ...Option) options { + opt := options{ + defaultRegistry: DefaultRegistry, + defaultTag: DefaultTag, + } + for _, o := range opts { + o(&opt) + } + return opt +} + +// Option is a functional option for name parsing. +type Option func(*options) + +// StrictValidation is an Option that requires image references to be fully +// specified; i.e. no defaulting for registry (dockerhub), repo (library), +// or tag (latest). +func StrictValidation(opts *options) { + opts.strict = true +} + +// WeakValidation is an Option that sets defaults when parsing names, see +// StrictValidation. +func WeakValidation(opts *options) { + opts.strict = false +} + +// Insecure is an Option that allows image references to be fetched without TLS. +func Insecure(opts *options) { + opts.insecure = true +} + +// OptionFn is a function that returns an option. +type OptionFn func() Option + +// WithDefaultRegistry sets the default registry that will be used if one is not +// provided. +func WithDefaultRegistry(r string) Option { + return func(opts *options) { + opts.defaultRegistry = r + } +} + +// WithDefaultTag sets the default tag that will be used if one is not provided. +func WithDefaultTag(t string) Option { + return func(opts *options) { + opts.defaultTag = t + } +} diff --git a/pkg/name/ref.go b/pkg/name/ref.go new file mode 100644 index 0000000..912ab33 --- /dev/null +++ b/pkg/name/ref.go @@ -0,0 +1,75 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "fmt" +) + +// Reference defines the interface that consumers use when they can +// take either a tag or a digest. +type Reference interface { + fmt.Stringer + + // Context accesses the Repository context of the reference. + Context() Repository + + // Identifier accesses the type-specific portion of the reference. + Identifier() string + + // Name is the fully-qualified reference name. + Name() string + + // Scope is the scope needed to access this reference. + Scope(string) string +} + +// ParseReference parses the string as a reference, either by tag or digest. +func ParseReference(s string, opts ...Option) (Reference, error) { + if t, err := NewTag(s, opts...); err == nil { + return t, nil + } + if d, err := NewDigest(s, opts...); err == nil { + return d, nil + } + return nil, newErrBadName("could not parse reference: " + s) +} + +type stringConst string + +// MustParseReference behaves like ParseReference, but panics instead of +// returning an error. It's intended for use in tests, or when a value is +// expected to be valid at code authoring time. +// +// To discourage its use in scenarios where the value is not known at code +// authoring time, it must be passed a string constant: +// +// const str = "valid/string" +// MustParseReference(str) +// MustParseReference("another/valid/string") +// MustParseReference(str + "/and/more") +// +// These will not compile: +// +// var str = "valid/string" +// MustParseReference(str) +// MustParseReference(strings.Join([]string{"valid", "string"}, "/")) +func MustParseReference(s stringConst, opts ...Option) Reference { + ref, err := ParseReference(string(s), opts...) + if err != nil { + panic(err) + } + return ref +} diff --git a/pkg/name/ref_test.go b/pkg/name/ref_test.go new file mode 100644 index 0000000..c47283c --- /dev/null +++ b/pkg/name/ref_test.go @@ -0,0 +1,157 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "testing" +) + +var ( + testDefaultRegistry = "registry.upbound.io" + testDefaultTag = "stable" + inputDefaultNames = []string{ + "crossplane/provider-gcp", + "crossplane/provider-gcp:v0.14.0", + "ubuntu", + "gcr.io/crossplane/provider-gcp:latest", + } + outputDefaultNames = []string{ + "registry.upbound.io/crossplane/provider-gcp:stable", + "registry.upbound.io/crossplane/provider-gcp:v0.14.0", + "registry.upbound.io/ubuntu:stable", + "gcr.io/crossplane/provider-gcp:latest", + } +) + +func TestParseReferenceDefaulting(t *testing.T) { + for i, name := range inputDefaultNames { + ref, err := ParseReference(name, WithDefaultRegistry(testDefaultRegistry), WithDefaultTag(testDefaultTag)) + if err != nil { + t.Errorf("ParseReference(%q); %v", name, err) + } + if ref.Name() != outputDefaultNames[i] { + t.Errorf("ParseReference(%q); got %v, want %v", name, ref.String(), outputDefaultNames[i]) + } + } +} + +func TestParseReference(t *testing.T) { + for _, name := range goodWeakValidationDigestNames { + ref, err := ParseReference(name, WeakValidation) + if err != nil { + t.Errorf("ParseReference(%q); %v", name, err) + } + dig, err := NewDigest(name, WeakValidation) + if err != nil { + t.Errorf("NewDigest(%q); %v", name, err) + } + if ref != dig { + t.Errorf("ParseReference(%q) != NewDigest(%q); got %v, want %v", name, name, ref, dig) + } + } + + for _, name := range goodStrictValidationDigestNames { + ref, err := ParseReference(name, StrictValidation) + if err != nil { + t.Errorf("ParseReference(%q); %v", name, err) + } + dig, err := NewDigest(name, StrictValidation) + if err != nil { + t.Errorf("NewDigest(%q); %v", name, err) + } + if ref != dig { + t.Errorf("ParseReference(%q) != NewDigest(%q); got %v, want %v", name, name, ref, dig) + } + } + + for _, name := range badDigestNames { + if _, err := ParseReference(name, WeakValidation); err == nil { + t.Errorf("ParseReference(%q); expected error, got none", name) + } + } + + for _, name := range goodWeakValidationTagNames { + ref, err := ParseReference(name, WeakValidation) + if err != nil { + t.Errorf("ParseReference(%q); %v", name, err) + } + tag, err := NewTag(name, WeakValidation) + if err != nil { + t.Errorf("NewTag(%q); %v", name, err) + } + if ref != tag { + t.Errorf("ParseReference(%q) != NewTag(%q); got %v, want %v", name, name, ref, tag) + } + } + + for _, name := range goodStrictValidationTagNames { + ref, err := ParseReference(name, StrictValidation) + if err != nil { + t.Errorf("ParseReference(%q); %v", name, err) + } + tag, err := NewTag(name, StrictValidation) + if err != nil { + t.Errorf("NewTag(%q); %v", name, err) + } + if ref != tag { + t.Errorf("ParseReference(%q) != NewTag(%q); got %v, want %v", name, name, ref, tag) + } + } + + for _, name := range badTagNames { + if _, err := ParseReference(name, WeakValidation); err == nil { + t.Errorf("ParseReference(%q); expected error, got none", name) + } + } +} + +func TestMustParseReference(t *testing.T) { + for _, name := range append(goodWeakValidationTagNames, goodWeakValidationDigestNames...) { + func() { + defer func() { + if err := recover(); err != nil { + t.Errorf("MustParseReference(%q, WeakValidation); panic: %v", name, err) + } + }() + MustParseReference(stringConst(name), WeakValidation) + }() + } + + for _, name := range append(goodStrictValidationTagNames, goodStrictValidationDigestNames...) { + func() { + defer func() { + if err := recover(); err != nil { + t.Errorf("MustParseReference(%q, StrictValidation); panic: %v", name, err) + } + }() + MustParseReference(stringConst(name), StrictValidation) + }() + } + + for _, name := range append(badTagNames, badDigestNames...) { + func() { + defer func() { recover() }() + ref := MustParseReference(stringConst(name), WeakValidation) + t.Errorf("MustParseReference(%q, WeakValidation) should panic, got: %#v", name, ref) + }() + } +} + +// Test that MustParseReference can accept a const string or string value. +const str = "valid/string" + +var _ = MustParseReference(str) +var _ = MustParseReference("valid/string") +var _ = MustParseReference("valid/prefix/" + str) diff --git a/pkg/name/registry.go b/pkg/name/registry.go new file mode 100644 index 0000000..2a26b66 --- /dev/null +++ b/pkg/name/registry.go @@ -0,0 +1,136 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "net" + "net/url" + "regexp" + "strings" +) + +// Detect more complex forms of local references. +var reLocal = regexp.MustCompile(`.*\.local(?:host)?(?::\d{1,5})?$`) + +// Detect the loopback IP (127.0.0.1) +var reLoopback = regexp.MustCompile(regexp.QuoteMeta("127.0.0.1")) + +// Detect the loopback IPV6 (::1) +var reipv6Loopback = regexp.MustCompile(regexp.QuoteMeta("::1")) + +// Registry stores a docker registry name in a structured form. +type Registry struct { + insecure bool + registry string +} + +// RegistryStr returns the registry component of the Registry. +func (r Registry) RegistryStr() string { + return r.registry +} + +// Name returns the name from which the Registry was derived. +func (r Registry) Name() string { + return r.RegistryStr() +} + +func (r Registry) String() string { + return r.Name() +} + +// Scope returns the scope required to access the registry. +func (r Registry) Scope(string) string { + // The only resource under 'registry' is 'catalog'. http://goo.gl/N9cN9Z + return "registry:catalog:*" +} + +func (r Registry) isRFC1918() bool { + ipStr := strings.Split(r.Name(), ":")[0] + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + for _, cidr := range []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} { + _, block, _ := net.ParseCIDR(cidr) + if block.Contains(ip) { + return true + } + } + return false +} + +// Scheme returns https scheme for all the endpoints except localhost or when explicitly defined. +func (r Registry) Scheme() string { + if r.insecure { + return "http" + } + if r.isRFC1918() { + return "http" + } + if strings.HasPrefix(r.Name(), "localhost:") { + return "http" + } + if reLocal.MatchString(r.Name()) { + return "http" + } + if reLoopback.MatchString(r.Name()) { + return "http" + } + if reipv6Loopback.MatchString(r.Name()) { + return "http" + } + return "https" +} + +func checkRegistry(name string) error { + // Per RFC 3986, registries (authorities) are required to be prefixed with "//" + // url.Host == hostname[:port] == authority + if url, err := url.Parse("//" + name); err != nil || url.Host != name { + return newErrBadName("registries must be valid RFC 3986 URI authorities: %s", name) + } + return nil +} + +// NewRegistry returns a Registry based on the given name. +// Strict validation requires explicit, valid RFC 3986 URI authorities to be given. +func NewRegistry(name string, opts ...Option) (Registry, error) { + opt := makeOptions(opts...) + if opt.strict && len(name) == 0 { + return Registry{}, newErrBadName("strict validation requires the registry to be explicitly defined") + } + + if err := checkRegistry(name); err != nil { + return Registry{}, err + } + + if name == "" { + name = opt.defaultRegistry + } + // Rewrite "docker.io" to "index.docker.io". + // See: https://github.com/google/go-containerregistry/issues/68 + if name == defaultRegistryAlias { + name = DefaultRegistry + } + + return Registry{registry: name, insecure: opt.insecure}, nil +} + +// NewInsecureRegistry returns an Insecure Registry based on the given name. +// +// Deprecated: Use the Insecure Option with NewRegistry instead. +func NewInsecureRegistry(name string, opts ...Option) (Registry, error) { + opts = append(opts, Insecure) + return NewRegistry(name, opts...) +} diff --git a/pkg/name/registry_test.go b/pkg/name/registry_test.go new file mode 100644 index 0000000..9986ee6 --- /dev/null +++ b/pkg/name/registry_test.go @@ -0,0 +1,252 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "testing" +) + +var goodStrictValidationRegistryNames = []string{ + "gcr.io", + "gcr.io:9001", + "index.docker.io", + "us.gcr.io", + "example.text", + "localhost", + "localhost:9090", +} + +var goodWeakValidationRegistryNames = []string{ + "", +} + +var badRegistryNames = []string{ + "white space", + "gcr?com", +} + +func TestNewRegistryStrictValidation(t *testing.T) { + t.Parallel() + + for _, name := range goodStrictValidationRegistryNames { + if registry, err := NewRegistry(name, StrictValidation); err != nil { + t.Errorf("`%s` should be a valid Registry name, got error: %v", name, err) + } else { + if registry.Name() != name { + t.Errorf("`%v` .Name() should reproduce the original name. Wanted: %s Got: %s", registry, name, registry.Name()) + } + if registry.String() != name { + t.Errorf("`%v` .String() should reproduce the original name. Wanted: %s Got: %s", registry, name, registry.String()) + } + } + } + + for _, name := range append(goodWeakValidationRegistryNames, badRegistryNames...) { + if repo, err := NewRegistry(name, StrictValidation); err == nil { + t.Errorf("`%s` should be an invalid Registry name, got Registry: %#v", name, repo) + } + } +} + +func TestNewRegistry(t *testing.T) { + t.Parallel() + + for _, name := range append(goodStrictValidationRegistryNames, goodWeakValidationRegistryNames...) { + if _, err := NewRegistry(name, WeakValidation); err != nil { + t.Errorf("`%s` should be a valid Registry name, got error: %v", name, err) + } + } + + for _, name := range badRegistryNames { + if repo, err := NewRegistry(name, WeakValidation); err == nil { + t.Errorf("`%s` should be an invalid Registry name, got Registry: %#v", name, repo) + } + } +} + +func TestNewInsecureRegistry(t *testing.T) { + t.Parallel() + + for _, name := range append(goodStrictValidationRegistryNames, goodWeakValidationRegistryNames...) { + if _, err := NewInsecureRegistry(name, WeakValidation); err != nil { + t.Errorf("`%s` should be a valid Registry name, got error: %v", name, err) + } + } + + for _, name := range badRegistryNames { + if repo, err := NewInsecureRegistry(name, WeakValidation); err == nil { + t.Errorf("`%s` should be an invalid Registry name, got Registry: %#v", name, repo) + } + } +} + +func TestDefaultRegistryNames(t *testing.T) { + testRegistries := []string{"docker.io", ""} + + for _, testRegistry := range testRegistries { + registry, err := NewRegistry(testRegistry, WeakValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Registry name, got error: %v", testRegistry, err) + } + + actualRegistry := registry.RegistryStr() + if actualRegistry != DefaultRegistry { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", registry, DefaultRegistry, actualRegistry) + } + } +} + +func TestOverrideDefaultRegistryNames(t *testing.T) { + testRegistries := []string{"docker.io", ""} + expectedRegistries := []string{"index.docker.io", "gcr.io"} + overrideDefault := "gcr.io" + + for i, testRegistry := range testRegistries { + registry, err := NewRegistry(testRegistry, WeakValidation, WithDefaultRegistry(overrideDefault)) + if err != nil { + t.Fatalf("`%s` should be a valid Registry name, got error: %v", testRegistry, err) + } + + actualRegistry := registry.RegistryStr() + if actualRegistry != expectedRegistries[i] { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", registry, expectedRegistries[i], actualRegistry) + } + } +} + +func TestRegistryComponents(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + + registry, err := NewRegistry(testRegistry, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Registry name, got error: %v", testRegistry, err) + } + + actualRegistry := registry.RegistryStr() + if actualRegistry != testRegistry { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", registry, testRegistry, actualRegistry) + } +} + +func TestRegistryScopes(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testAction := "whatever" + + expectedScope := "registry:catalog:*" + + registry, err := NewRegistry(testRegistry, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Registry name, got error: %v", testRegistry, err) + } + + actualScope := registry.Scope(testAction) + if actualScope != expectedScope { + t.Errorf("scope was incorrect for %v. Wanted: `%s` Got: `%s`", registry, expectedScope, actualScope) + } +} + +func TestIsRFC1918(t *testing.T) { + t.Parallel() + tests := []struct { + reg string + result bool + }{{ + reg: "index.docker.io", + result: false, + }, { + reg: "10.2.3.4:5000", + result: true, + }, { + reg: "8.8.8.8", + result: false, + }, { + reg: "172.16.3.4:3000", + result: true, + }, { + reg: "192.168.3.4", + result: true, + }, { + reg: "10.256.0.0:5000", + result: false, + }} + for _, test := range tests { + reg, err := NewRegistry(test.reg, WeakValidation) + if err != nil { + t.Errorf("NewRegistry(%s) = %v", test.reg, err) + } + got := reg.isRFC1918() + if got != test.result { + t.Errorf("isRFC1918(); got %v, want %v", got, test.result) + } + } +} + +func TestRegistryScheme(t *testing.T) { + t.Parallel() + tests := []struct { + domain string + scheme string + }{{ + domain: "foo.svc.local:1234", + scheme: "http", + }, { + domain: "127.0.0.1:1234", + scheme: "http", + }, { + domain: "127.0.0.1", + scheme: "http", + }, { + domain: "localhost:8080", + scheme: "http", + }, { + domain: "gcr.io", + scheme: "https", + }, { + domain: "index.docker.io", + scheme: "https", + }, { + domain: "::1", + scheme: "http", + }, { + domain: "10.2.3.4:5000", + scheme: "http", + }} + + for _, test := range tests { + reg, err := NewRegistry(test.domain, WeakValidation) + if err != nil { + t.Errorf("NewRegistry(%s) = %v", test.domain, err) + } + if got, want := reg.Scheme(), test.scheme; got != want { + t.Errorf("scheme(%v); got %v, want %v", reg, got, want) + } + } +} + +func TestRegistryInsecureScheme(t *testing.T) { + t.Parallel() + domain := "gcr.io" + + reg, err := NewInsecureRegistry(domain, WeakValidation) + if err != nil { + t.Errorf("NewRegistry(%s) = %v", domain, err) + } + + if got := reg.Scheme(); got != "http" { + t.Errorf("scheme(%v); got %v, want http", reg, got) + } +} diff --git a/pkg/name/repository.go b/pkg/name/repository.go new file mode 100644 index 0000000..9250e36 --- /dev/null +++ b/pkg/name/repository.go @@ -0,0 +1,121 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "fmt" + "strings" +) + +const ( + defaultNamespace = "library" + repositoryChars = "abcdefghijklmnopqrstuvwxyz0123456789_-./" + regRepoDelimiter = "/" +) + +// Repository stores a docker repository name in a structured form. +type Repository struct { + Registry + repository string +} + +// See https://docs.docker.com/docker-hub/official_repos +func hasImplicitNamespace(repo string, reg Registry) bool { + return !strings.ContainsRune(repo, '/') && reg.RegistryStr() == DefaultRegistry +} + +// RepositoryStr returns the repository component of the Repository. +func (r Repository) RepositoryStr() string { + if hasImplicitNamespace(r.repository, r.Registry) { + return fmt.Sprintf("%s/%s", defaultNamespace, r.repository) + } + return r.repository +} + +// Name returns the name from which the Repository was derived. +func (r Repository) Name() string { + regName := r.Registry.Name() + if regName != "" { + return regName + regRepoDelimiter + r.RepositoryStr() + } + // TODO: As far as I can tell, this is unreachable. + return r.RepositoryStr() +} + +func (r Repository) String() string { + return r.Name() +} + +// Scope returns the scope required to perform the given action on the registry. +// TODO(jonjohnsonjr): consider moving scopes to a separate package. +func (r Repository) Scope(action string) string { + return fmt.Sprintf("repository:%s:%s", r.RepositoryStr(), action) +} + +func checkRepository(repository string) error { + return checkElement("repository", repository, repositoryChars, 2, 255) +} + +// NewRepository returns a new Repository representing the given name, according to the given strictness. +func NewRepository(name string, opts ...Option) (Repository, error) { + opt := makeOptions(opts...) + if len(name) == 0 { + return Repository{}, newErrBadName("a repository name must be specified") + } + + var registry string + repo := name + parts := strings.SplitN(name, regRepoDelimiter, 2) + if len(parts) == 2 && (strings.ContainsRune(parts[0], '.') || strings.ContainsRune(parts[0], ':')) { + // The first part of the repository is treated as the registry domain + // iff it contains a '.' or ':' character, otherwise it is all repository + // and the domain defaults to Docker Hub. + registry = parts[0] + repo = parts[1] + } + + if err := checkRepository(repo); err != nil { + return Repository{}, err + } + + reg, err := NewRegistry(registry, opts...) + if err != nil { + return Repository{}, err + } + if hasImplicitNamespace(repo, reg) && opt.strict { + return Repository{}, newErrBadName("strict validation requires the full repository path (missing 'library')") + } + return Repository{reg, repo}, nil +} + +// Tag returns a Tag in this Repository. +func (r Repository) Tag(identifier string) Tag { + t := Tag{ + tag: identifier, + Repository: r, + } + t.original = t.Name() + return t +} + +// Digest returns a Digest in this Repository. +func (r Repository) Digest(identifier string) Digest { + d := Digest{ + digest: identifier, + Repository: r, + } + d.original = d.Name() + return d +} diff --git a/pkg/name/repository_test.go b/pkg/name/repository_test.go new file mode 100644 index 0000000..790cab6 --- /dev/null +++ b/pkg/name/repository_test.go @@ -0,0 +1,145 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "errors" + "strings" + "testing" +) + +var goodStrictValidationRepositoryNames = []string{ + "gcr.io/g-convoy/hello-world", + "gcr.io/google.com/project-id/hello-world", + "us.gcr.io/project-id/sub-repo", + "example.text/foo/bar", + "mirror.gcr.io/ubuntu", + "index.docker.io/library/ubuntu", +} + +var goodWeakValidationRepositoryNames = []string{ + "namespace/pathcomponent/image", + "library/ubuntu", + "ubuntu", +} + +var badRepositoryNames = []string{ + "white space", + "b@char/image", + "", +} + +func TestNewRepositoryStrictValidation(t *testing.T) { + t.Parallel() + + for _, name := range goodStrictValidationRepositoryNames { + if repository, err := NewRepository(name, StrictValidation); err != nil { + t.Errorf("`%s` should be a valid Repository name, got error: %v", name, err) + } else if repository.Name() != name { + t.Errorf("`%v` .Name() should reproduce the original name. Wanted: %s Got: %s", repository, name, repository.Name()) + } + } + + for _, name := range append(goodWeakValidationRepositoryNames, badRepositoryNames...) { + if repo, err := NewRepository(name, StrictValidation); err == nil { + t.Errorf("`%s` should be an invalid repository name, got Repository: %#v", name, repo) + } + } +} + +func TestNewRepository(t *testing.T) { + t.Parallel() + + for _, name := range append(goodStrictValidationRepositoryNames, goodWeakValidationRepositoryNames...) { + if _, err := NewRepository(name, WeakValidation); err != nil { + t.Errorf("`%s` should be a valid repository name, got error: %v", name, err) + } + } + + for _, name := range badRepositoryNames { + if repo, err := NewRepository(name, WeakValidation); err == nil { + t.Errorf("`%s` should be an invalid repository name, got Repository: %#v", name, repo) + } + } +} + +func TestRepositoryComponents(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepository := "project-id/image" + + repositoryNameStr := testRegistry + "/" + testRepository + repository, err := NewRepository(repositoryNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Repository name, got error: %v", repositoryNameStr, err) + } + + actualRegistry := repository.RegistryStr() + if actualRegistry != testRegistry { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", repository, testRegistry, actualRegistry) + } + actualRepository := repository.RepositoryStr() + if actualRepository != testRepository { + t.Errorf("RepositoryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", repository, testRepository, actualRepository) + } +} + +func TestRepositoryScopes(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepo := "project-id/image" + testAction := "pull" + + expectedScope := strings.Join([]string{"repository", testRepo, testAction}, ":") + + repositoryNameStr := testRegistry + "/" + testRepo + repository, err := NewRepository(repositoryNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Repository name, got error: %v", repositoryNameStr, err) + } + + actualScope := repository.Scope(testAction) + if actualScope != expectedScope { + t.Errorf("scope was incorrect for %v. Wanted: `%s` Got: `%s`", repository, expectedScope, actualScope) + } +} + +func TestRepositoryBadDefaulting(t *testing.T) { + var berr *ErrBadName + if _, err := NewRepository("index.docker.io/foo", StrictValidation); !errors.As(err, &berr) { + t.Errorf("Not an ErrBadName: %v", err) + } +} + +func TestRepositoryChildren(t *testing.T) { + repo, err := NewRepository("example.com/repo", Insecure) + if err != nil { + t.Fatal(err) + } + tag := repo.Tag("foo") + if got, want := tag.Scheme(), "http"; got != want { + t.Errorf("tag.Scheme(): got %s want %s", got, want) + } + if got, want := tag.String(), "example.com/repo:foo"; got != want { + t.Errorf("tag.String(): got %s want %s", got, want) + } + digest := repo.Digest("badf00d") + if got, want := digest.Scheme(), "http"; got != want { + t.Errorf("digest.Scheme(): got %s want %s", got, want) + } + if got, want := digest.String(), "example.com/repo@badf00d"; got != want { + t.Errorf("digest.String(): got %s want %s", got, want) + } +} diff --git a/pkg/name/tag.go b/pkg/name/tag.go new file mode 100644 index 0000000..66bd1be --- /dev/null +++ b/pkg/name/tag.go @@ -0,0 +1,108 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "strings" +) + +const ( + // TODO(dekkagaijin): use the docker/distribution regexes for validation. + tagChars = "abcdefghijklmnopqrstuvwxyz0123456789_-.ABCDEFGHIJKLMNOPQRSTUVWXYZ" + tagDelim = ":" +) + +// Tag stores a docker tag name in a structured form. +type Tag struct { + Repository + tag string + original string +} + +// Ensure Tag implements Reference +var _ Reference = (*Tag)(nil) + +// Context implements Reference. +func (t Tag) Context() Repository { + return t.Repository +} + +// Identifier implements Reference. +func (t Tag) Identifier() string { + return t.TagStr() +} + +// TagStr returns the tag component of the Tag. +func (t Tag) TagStr() string { + return t.tag +} + +// Name returns the name from which the Tag was derived. +func (t Tag) Name() string { + return t.Repository.Name() + tagDelim + t.TagStr() +} + +// String returns the original input string. +func (t Tag) String() string { + return t.original +} + +// Scope returns the scope required to perform the given action on the tag. +func (t Tag) Scope(action string) string { + return t.Repository.Scope(action) +} + +func checkTag(name string) error { + return checkElement("tag", name, tagChars, 1, 128) +} + +// NewTag returns a new Tag representing the given name, according to the given strictness. +func NewTag(name string, opts ...Option) (Tag, error) { + opt := makeOptions(opts...) + base := name + tag := "" + + // Split on ":" + parts := strings.Split(name, tagDelim) + // Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation. + if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) { + base = strings.Join(parts[:len(parts)-1], tagDelim) + tag = parts[len(parts)-1] + } + + // We don't require a tag, but if we get one check it's valid, + // even when not being strict. + // If we are being strict, we want to validate the tag regardless in case + // it's empty. + if tag != "" || opt.strict { + if err := checkTag(tag); err != nil { + return Tag{}, err + } + } + + if tag == "" { + tag = opt.defaultTag + } + + repo, err := NewRepository(base, opts...) + if err != nil { + return Tag{}, err + } + return Tag{ + Repository: repo, + tag: tag, + original: name, + }, nil +} diff --git a/pkg/name/tag_test.go b/pkg/name/tag_test.go new file mode 100644 index 0000000..e340566 --- /dev/null +++ b/pkg/name/tag_test.go @@ -0,0 +1,162 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +import ( + "path" + "strings" + "testing" +) + +var goodStrictValidationTagNames = []string{ + "gcr.io/g-convoy/hello-world:latest", + "gcr.io/google.com/g-convoy/hello-world:latest", + "gcr.io/project-id/with-nums:v2", + "us.gcr.io/project-id/image:with.period.in.tag", + "gcr.io/project-id/image:w1th-alpha_num3ric.PLUScaps", + "domain.with.port:9001/image:latest", +} + +var goodWeakValidationTagNames = []string{ + "namespace/pathcomponent/image", + "library/ubuntu", + "gcr.io/project-id/implicit-latest", + "www.example.test:12345/repo/path", +} + +var badTagNames = []string{ + "gcr.io/project-id/bad_chars:c@n'tuse", + "gcr.io/project-id/wrong-length:white space", + "gcr.io/project-id/too-many-chars:thisisthetagthatneverendsitgoesonandonmyfriendsomepeoplestartedtaggingitnotknowingwhatitwasandtheyllcontinuetaggingitforeverjustbecausethisisthetagthatneverends", +} + +func TestNewTagStrictValidation(t *testing.T) { + t.Parallel() + + for _, name := range goodStrictValidationTagNames { + if tag, err := NewTag(name, StrictValidation); err != nil { + t.Errorf("`%s` should be a valid Tag name, got error: %v", name, err) + } else if tag.Name() != name { + t.Errorf("`%v` .Name() should reproduce the original name. Wanted: %s Got: %s", tag, name, tag.Name()) + } + } + + for _, name := range append(goodWeakValidationTagNames, badTagNames...) { + if tag, err := NewTag(name, StrictValidation); err == nil { + t.Errorf("`%s` should be an invalid Tag name, got Tag: %#v", name, tag) + } + } +} + +func TestNewTag(t *testing.T) { + t.Parallel() + + for _, name := range append(goodStrictValidationTagNames, goodWeakValidationTagNames...) { + if _, err := NewTag(name, WeakValidation); err != nil { + t.Errorf("`%s` should be a valid Tag name, got error: %v", name, err) + } + } + + for _, name := range badTagNames { + if tag, err := NewTag(name, WeakValidation); err == nil { + t.Errorf("`%s` should be an invalid Tag name, got Tag: %#v", name, tag) + } + } +} + +func TestTagComponents(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepository := "project-id/image" + testTag := "latest" + fullRepo := path.Join(testRegistry, testRepository) + + tagNameStr := testRegistry + "/" + testRepository + ":" + testTag + tag, err := NewTag(tagNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Tag name, got error: %v", tagNameStr, err) + } + + actualRegistry := tag.RegistryStr() + if actualRegistry != testRegistry { + t.Errorf("RegistryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, testRegistry, actualRegistry) + } + actualRepository := tag.RepositoryStr() + if actualRepository != testRepository { + t.Errorf("RepositoryStr() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, testRepository, actualRepository) + } + actualTag := tag.TagStr() + if actualTag != testTag { + t.Errorf("TagStr() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, testTag, actualTag) + } + if got, want := tag.Context().String(), fullRepo; got != want { + t.Errorf("Context.String() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, want, got) + } + if got, want := tag.Identifier(), testTag; got != want { + t.Errorf("Identifier() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, want, got) + } + if got, want := tag.String(), tagNameStr; got != want { + t.Errorf("String() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, want, got) + } +} + +func TestTagScopes(t *testing.T) { + t.Parallel() + testRegistry := "gcr.io" + testRepo := "project-id/image" + testTag := "latest" + testAction := "pull" + + expectedScope := strings.Join([]string{"repository", testRepo, testAction}, ":") + + tagNameStr := testRegistry + "/" + testRepo + ":" + testTag + tag, err := NewTag(tagNameStr, StrictValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Tag name, got error: %v", tagNameStr, err) + } + + actualScope := tag.Scope(testAction) + if actualScope != expectedScope { + t.Errorf("scope was incorrect for %v. Wanted: `%s` Got: `%s`", tag, expectedScope, actualScope) + } +} + +func TestAllDefaults(t *testing.T) { + tagNameStr := "ubuntu" + tag, err := NewTag(tagNameStr, WeakValidation) + if err != nil { + t.Fatalf("`%s` should be a valid Tag name, got error: %v", tagNameStr, err) + } + + expectedName := "index.docker.io/library/ubuntu:latest" + actualName := tag.Name() + if actualName != expectedName { + t.Errorf("Name() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, expectedName, actualName) + } +} + +func TestOverrideDefault(t *testing.T) { + tagNameStr := "ubuntu" + tag, err := NewTag(tagNameStr, WeakValidation, WithDefaultTag("other")) + if err != nil { + t.Fatalf("`%s` should be a valid Tag name, got error: %v", tagNameStr, err) + } + + expectedName := "index.docker.io/library/ubuntu:other" + actualName := tag.Name() + if actualName != expectedName { + t.Errorf("Name() was incorrect for %v. Wanted: `%s` Got: `%s`", tag, expectedName, actualName) + } +} diff --git a/pkg/registry/README.md b/pkg/registry/README.md new file mode 100644 index 0000000..5e58bbc --- /dev/null +++ b/pkg/registry/README.md @@ -0,0 +1,14 @@ +# `pkg/registry` + +This package implements a Docker v2 registry and the OCI distribution specification. + +It is designed to be used anywhere a low dependency container registry is needed, with an initial focus on tests. + +Its goal is to be standards compliant and its strictness will increase over time. + +This is currently a low flightmiles system. It's likely quite safe to use in tests; If you're using it in production, please let us know how and send us PRs for integration tests. + +Before sending a PR, understand that the expectation of this package is that it remain free of extraneous dependencies. +This means that we expect `pkg/registry` to only have dependencies on Go's standard library, and other packages in `go-containerregistry`. + +You may be asked to change your code to reduce dependencies, and your PR might be rejected if this is deemed impossible. diff --git a/pkg/registry/blobs.go b/pkg/registry/blobs.go new file mode 100644 index 0000000..4bf2c65 --- /dev/null +++ b/pkg/registry/blobs.go @@ -0,0 +1,483 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net/http" + "path" + "strings" + "sync" + + "github.com/google/go-containerregistry/internal/verify" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Returns whether this url should be handled by the blob handler +// This is complicated because blob is indicated by the trailing path, not the leading path. +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-a-layer +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-a-layer +func isBlob(req *http.Request) bool { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + if len(elem) < 3 { + return false + } + return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" && + elem[len(elem)-2] == "uploads") +} + +// blobHandler represents a minimal blob storage backend, capable of serving +// blob contents. +type blobHandler interface { + // Get gets the blob contents, or errNotFound if the blob wasn't found. + Get(ctx context.Context, repo string, h v1.Hash) (io.ReadCloser, error) +} + +// blobStatHandler is an extension interface representing a blob storage +// backend that can serve metadata about blobs. +type blobStatHandler interface { + // Stat returns the size of the blob, or errNotFound if the blob wasn't + // found, or redirectError if the blob can be found elsewhere. + Stat(ctx context.Context, repo string, h v1.Hash) (int64, error) +} + +// blobPutHandler is an extension interface representing a blob storage backend +// that can write blob contents. +type blobPutHandler interface { + // Put puts the blob contents. + // + // The contents will be verified against the expected size and digest + // as the contents are read, and an error will be returned if these + // don't match. Implementations should return that error, or a wrapper + // around that error, to return the correct error when these don't match. + Put(ctx context.Context, repo string, h v1.Hash, rc io.ReadCloser) error +} + +// blobDeleteHandler is an extension interface representing a blob storage +// backend that can delete blob contents. +type blobDeleteHandler interface { + // Delete the blob contents. + Delete(ctx context.Context, repo string, h v1.Hash) error +} + +// redirectError represents a signal that the blob handler doesn't have the blob +// contents, but that those contents are at another location which registry +// clients should redirect to. +type redirectError struct { + // Location is the location to find the contents. + Location string + + // Code is the HTTP redirect status code to return to clients. + Code int +} + +func (e redirectError) Error() string { return fmt.Sprintf("redirecting (%d): %s", e.Code, e.Location) } + +// errNotFound represents an error locating the blob. +var errNotFound = errors.New("not found") + +type memHandler struct { + m map[string][]byte + lock sync.Mutex +} + +func (m *memHandler) Stat(_ context.Context, _ string, h v1.Hash) (int64, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return 0, errNotFound + } + return int64(len(b)), nil +} +func (m *memHandler) Get(_ context.Context, _ string, h v1.Hash) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return nil, errNotFound + } + return io.NopCloser(bytes.NewReader(b)), nil +} +func (m *memHandler) Put(_ context.Context, _ string, h v1.Hash, rc io.ReadCloser) error { + m.lock.Lock() + defer m.lock.Unlock() + + defer rc.Close() + all, err := io.ReadAll(rc) + if err != nil { + return err + } + m.m[h.String()] = all + return nil +} +func (m *memHandler) Delete(_ context.Context, _ string, h v1.Hash) error { + m.lock.Lock() + defer m.lock.Unlock() + + if _, found := m.m[h.String()]; !found { + return errNotFound + } + + delete(m.m, h.String()) + return nil +} + +// blobs +type blobs struct { + blobHandler blobHandler + + // Each upload gets a unique id that writes occur to until finalized. + uploads map[string][]byte + lock sync.Mutex + log *log.Logger +} + +func (b *blobs) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + // Must have a path of form /v2/{name}/blobs/{upload,sha256:} + if len(elem) < 4 { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "blobs must be attached to a repo", + } + } + target := elem[len(elem)-1] + service := elem[len(elem)-2] + digest := req.URL.Query().Get("digest") + contentRange := req.Header.Get("Content-Range") + + repo := req.URL.Host + path.Join(elem[1:len(elem)-2]...) + + switch req.Method { + case http.MethodHead: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + if bsh, ok := b.blobHandler.(blobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + } else { + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + defer rc.Close() + size, err = io.Copy(io.Discard, rc) + if err != nil { + return regErrInternal(err) + } + } + + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + return nil + + case http.MethodGet: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + var r io.Reader + if bsh, ok := b.blobHandler.(blobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + defer rc.Close() + r = rc + } else { + tmp, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + defer tmp.Close() + var buf bytes.Buffer + io.Copy(&buf, tmp) + size = int64(buf.Len()) + r = &buf + } + + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, r) + return nil + + case http.MethodPost: + bph, ok := b.blobHandler.(blobPutHandler) + if !ok { + return regErrUnsupported + } + + // It is weird that this is "target" instead of "service", but + // that's how the index math works out above. + if target != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target), + } + } + + if digest != "" { + h, err := v1.NewHash(digest) + if err != nil { + return regErrDigestInvalid + } + + vrc, err := verify.ReadCloser(req.Body, req.ContentLength, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err = bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + } + + id := fmt.Sprint(rand.Int63()) + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id)) + resp.Header().Set("Range", "0-0") + resp.WriteHeader(http.StatusAccepted) + return nil + + case http.MethodPatch: + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service), + } + } + + if contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "We don't understand your Content-Range", + } + } + b.lock.Lock() + defer b.lock.Unlock() + if start != len(b.uploads[target]) { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "Your content range doesn't match what we have", + } + } + l := bytes.NewBuffer(b.uploads[target]) + io.Copy(l, req.Body) + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + } + + b.lock.Lock() + defer b.lock.Unlock() + if _, ok := b.uploads[target]; ok { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BLOB_UPLOAD_INVALID", + Message: "Stream uploads after first write are not allowed", + } + } + + l := &bytes.Buffer{} + io.Copy(l, req.Body) + + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + + case http.MethodPut: + bph, ok := b.blobHandler.(blobPutHandler) + if !ok { + return regErrUnsupported + } + + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service), + } + } + + if digest == "" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest not specified", + } + } + + b.lock.Lock() + defer b.lock.Unlock() + + h, err := v1.NewHash(digest) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + defer req.Body.Close() + in := io.NopCloser(io.MultiReader(bytes.NewBuffer(b.uploads[target]), req.Body)) + + size := int64(verify.SizeUnknown) + if req.ContentLength > 0 { + size = int64(len(b.uploads[target])) + req.ContentLength + } + + vrc, err := verify.ReadCloser(in, size, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err := bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + + delete(b.uploads, target) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + + case http.MethodDelete: + bdh, ok := b.blobHandler.(blobDeleteHandler) + if !ok { + return regErrUnsupported + } + + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + if err := bdh.Delete(req.Context(), repo, h); err != nil { + return regErrInternal(err) + } + resp.WriteHeader(http.StatusAccepted) + return nil + + default: + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } +} diff --git a/pkg/registry/compatibility_test.go b/pkg/registry/compatibility_test.go new file mode 100644 index 0000000..eff22c2 --- /dev/null +++ b/pkg/registry/compatibility_test.go @@ -0,0 +1,63 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry_test + +import ( + "bytes" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func TestPushAndPullContainer(t *testing.T) { + s := httptest.NewServer(registry.New()) + defer s.Close() + + r := strings.TrimPrefix(s.URL, "http://") + "/foo:latest" + d, err := name.NewTag(r) + if err != nil { + t.Fatalf("Unable to create tag: %v", err) + } + + i, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("Unable to make random image: %v", err) + } + + if err := remote.Write(d, i); err != nil { + t.Fatalf("Error writing image: %v", err) + } + + ref, err := name.ParseReference(r) + if err != nil { + t.Fatalf("Error parsing tag: %v", err) + } + + ri, err := remote.Image(ref) + if err != nil { + t.Fatalf("Error reading image: %v", err) + } + + b := &bytes.Buffer{} + if err := tarball.Write(ref, ri, b); err != nil { + t.Fatalf("Error writing image to tarball: %v", err) + } +} diff --git a/pkg/registry/depcheck_test.go b/pkg/registry/depcheck_test.go new file mode 100644 index 0000000..ca0bec5 --- /dev/null +++ b/pkg/registry/depcheck_test.go @@ -0,0 +1,38 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "testing" + + "github.com/google/go-containerregistry/internal/depcheck" +) + +func TestDeps(t *testing.T) { + if testing.Short() { + t.Skip("skipping slow depcheck") + } + depcheck.AssertOnlyDependencies(t, map[string][]string{ + "github.com/google/go-containerregistry/pkg/registry": append( + depcheck.StdlibPackages(), + "github.com/google/go-containerregistry/internal/httptest", + "github.com/google/go-containerregistry/pkg/v1", + "github.com/google/go-containerregistry/pkg/v1/types", + + "github.com/google/go-containerregistry/internal/verify", + "github.com/google/go-containerregistry/internal/and", + ), + }) +} diff --git a/pkg/registry/error.go b/pkg/registry/error.go new file mode 100644 index 0000000..f8e126d --- /dev/null +++ b/pkg/registry/error.go @@ -0,0 +1,79 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "encoding/json" + "net/http" +) + +type regError struct { + Status int + Code string + Message string +} + +func (r *regError) Write(resp http.ResponseWriter) error { + resp.WriteHeader(r.Status) + + type err struct { + Code string `json:"code"` + Message string `json:"message"` + } + type wrap struct { + Errors []err `json:"errors"` + } + return json.NewEncoder(resp).Encode(wrap{ + Errors: []err{ + { + Code: r.Code, + Message: r.Message, + }, + }, + }) +} + +// regErrInternal returns an internal server error. +func regErrInternal(err error) *regError { + return ®Error{ + Status: http.StatusInternalServerError, + Code: "INTERNAL_SERVER_ERROR", + Message: err.Error(), + } +} + +var regErrBlobUnknown = ®Error{ + Status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: "Unknown blob", +} + +var regErrUnsupported = ®Error{ + Status: http.StatusMethodNotAllowed, + Code: "UNSUPPORTED", + Message: "Unsupported operation", +} + +var regErrDigestMismatch = ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest does not match contents", +} + +var regErrDigestInvalid = ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", +} diff --git a/pkg/registry/manifest.go b/pkg/registry/manifest.go new file mode 100644 index 0000000..cd788f7 --- /dev/null +++ b/pkg/registry/manifest.go @@ -0,0 +1,430 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + "strconv" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type catalog struct { + Repos []string `json:"repositories"` +} + +type listTags struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +type manifest struct { + contentType string + blob []byte +} + +type manifests struct { + // maps repo -> manifest tag/digest -> manifest + manifests map[string]map[string]manifest + lock sync.Mutex + log *log.Logger +} + +func isManifest(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "manifests" +} + +func isTags(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "tags" +} + +func isCatalog(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 2 { + return false + } + + return elems[len(elems)-1] == "_catalog" +} + +// Returns whether this url should be handled by the referrers handler +func isReferrers(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "referrers" +} + +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-an-image-manifest +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-an-image +func (m *manifests) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + target := elem[len(elem)-1] + repo := strings.Join(elem[1:len(elem)-2], "/") + + switch req.Method { + case http.MethodGet: + m.lock.Lock() + defer m.lock.Unlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := c[target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + h, _, _ := v1.SHA256(bytes.NewReader(m.blob)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader(m.blob)) + return nil + + case http.MethodHead: + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + h, _, _ := v1.SHA256(bytes.NewReader(m.blob)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + return nil + + case http.MethodPut: + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + m.manifests[repo] = map[string]manifest{} + } + b := &bytes.Buffer{} + io.Copy(b, req.Body) + h, _, _ := v1.SHA256(bytes.NewReader(b.Bytes())) + digest := h.String() + mf := manifest{ + blob: b.Bytes(), + contentType: req.Header.Get("Content-Type"), + } + + // If the manifest is a manifest list, check that the manifest + // list's constituent manifests are already uploaded. + // This isn't strictly required by the registry API, but some + // registries require this. + if types.MediaType(mf.contentType).IsIndex() { + im, err := v1.ParseIndexManifest(b) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "MANIFEST_INVALID", + Message: err.Error(), + } + } + for _, desc := range im.Manifests { + if !desc.MediaType.IsDistributable() { + continue + } + if desc.MediaType.IsIndex() || desc.MediaType.IsImage() { + if _, found := m.manifests[repo][desc.Digest.String()]; !found { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: fmt.Sprintf("Sub-manifest %q not found", desc.Digest), + } + } + } else { + // TODO: Probably want to do an existence check for blobs. + m.log.Printf("TODO: Check blobs for %q", desc.Digest) + } + } + } + + // Allow future references by target (tag) and immutable digest. + // See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-by-digest-immutable-identifier. + m.manifests[repo][target] = mf + m.manifests[repo][digest] = mf + resp.Header().Set("Docker-Content-Digest", digest) + resp.WriteHeader(http.StatusCreated) + return nil + + case http.MethodDelete: + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + _, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + + delete(m.manifests[repo], target) + resp.WriteHeader(http.StatusAccepted) + return nil + + default: + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } +} + +func (m *manifests) handleTags(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + repo := strings.Join(elem[1:len(elem)-2], "/") + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + var tags []string + for tag := range c { + if !strings.Contains(tag, "sha256:") { + tags = append(tags, tag) + } + } + sort.Strings(tags) + + // https://github.com/opencontainers/distribution-spec/blob/b505e9cc53ec499edbd9c1be32298388921bb705/detail.md#tags-paginated + // Offset using last query parameter. + if last := req.URL.Query().Get("last"); last != "" { + for i, t := range tags { + if t > last { + tags = tags[i:] + break + } + } + } + + // Limit using n query parameter. + if ns := req.URL.Query().Get("n"); ns != "" { + if n, err := strconv.Atoi(ns); err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BAD_REQUEST", + Message: fmt.Sprintf("parsing n: %v", err), + } + } else if n < len(tags) { + tags = tags[:n] + } + } + + tagsToList := listTags{ + Name: repo, + Tags: tags, + } + + msg, _ := json.Marshal(tagsToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +func (m *manifests) handleCatalog(resp http.ResponseWriter, req *http.Request) *regError { + query := req.URL.Query() + nStr := query.Get("n") + n := 10000 + if nStr != "" { + n, _ = strconv.Atoi(nStr) + } + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + var repos []string + countRepos := 0 + // TODO: implement pagination + for key := range m.manifests { + if countRepos >= n { + break + } + countRepos++ + + repos = append(repos, key) + } + + repositoriesToList := catalog{ + Repos: repos, + } + + msg, _ := json.Marshal(repositoriesToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +// TODO: implement handling of artifactType querystring +func (m *manifests) handleReferrers(resp http.ResponseWriter, req *http.Request) *regError { + // Ensure this is a GET request + if req.Method != "GET" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } + + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + target := elem[len(elem)-1] + repo := strings.Join(elem[1:len(elem)-2], "/") + + // Validate that incoming target is a valid digest + if _, err := v1.NewHash(target); err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "UNSUPPORTED", + Message: "Target must be a valid digest", + } + } + + m.lock.Lock() + defer m.lock.Unlock() + + digestToManifestMap, repoExists := m.manifests[repo] + if !repoExists { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + im := v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + Manifests: []v1.Descriptor{}, + } + for digest, manifest := range digestToManifestMap { + h, err := v1.NewHash(digest) + if err != nil { + continue + } + var refPointer struct { + Subject *v1.Descriptor `json:"subject"` + } + json.Unmarshal(manifest.blob, &refPointer) + if refPointer.Subject == nil { + continue + } + referenceDigest := refPointer.Subject.Digest + if referenceDigest.String() != target { + continue + } + // At this point, we know the current digest references the target + var imageAsArtifact struct { + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + } + json.Unmarshal(manifest.blob, &imageAsArtifact) + im.Manifests = append(im.Manifests, v1.Descriptor{ + MediaType: types.MediaType(manifest.contentType), + Size: int64(len(manifest.blob)), + Digest: h, + ArtifactType: imageAsArtifact.Config.MediaType, + }) + } + msg, _ := json.Marshal(&im) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil +} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 0000000..303e6e7 --- /dev/null +++ b/pkg/registry/registry.go @@ -0,0 +1,117 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package registry implements a docker V2 registry and the OCI distribution specification. +// +// It is designed to be used anywhere a low dependency container registry is needed, with an +// initial focus on tests. +// +// Its goal is to be standards compliant and its strictness will increase over time. +// +// This is currently a low flightmiles system. It's likely quite safe to use in tests; If you're using it +// in production, please let us know how and send us CL's for integration tests. +package registry + +import ( + "log" + "net/http" + "os" +) + +type registry struct { + log *log.Logger + blobs blobs + manifests manifests + referrersEnabled bool +} + +// https://docs.docker.com/registry/spec/api/#api-version-check +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#api-version-check +func (r *registry) v2(resp http.ResponseWriter, req *http.Request) *regError { + if isBlob(req) { + return r.blobs.handle(resp, req) + } + if isManifest(req) { + return r.manifests.handle(resp, req) + } + if isTags(req) { + return r.manifests.handleTags(resp, req) + } + if isCatalog(req) { + return r.manifests.handleCatalog(resp, req) + } + if r.referrersEnabled && isReferrers(req) { + return r.manifests.handleReferrers(resp, req) + } + resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + if req.URL.Path != "/v2/" && req.URL.Path != "/v2" { + return ®Error{ + Status: http.StatusNotFound, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } + resp.WriteHeader(200) + return nil +} + +func (r *registry) root(resp http.ResponseWriter, req *http.Request) { + if rerr := r.v2(resp, req); rerr != nil { + r.log.Printf("%s %s %d %s %s", req.Method, req.URL, rerr.Status, rerr.Code, rerr.Message) + rerr.Write(resp) + return + } + r.log.Printf("%s %s", req.Method, req.URL) +} + +// New returns a handler which implements the docker registry protocol. +// It should be registered at the site root. +func New(opts ...Option) http.Handler { + r := ®istry{ + log: log.New(os.Stderr, "", log.LstdFlags), + blobs: blobs{ + blobHandler: &memHandler{m: map[string][]byte{}}, + uploads: map[string][]byte{}, + log: log.New(os.Stderr, "", log.LstdFlags), + }, + manifests: manifests{ + manifests: map[string]map[string]manifest{}, + log: log.New(os.Stderr, "", log.LstdFlags), + }, + } + for _, o := range opts { + o(r) + } + return http.HandlerFunc(r.root) +} + +// Option describes the available options +// for creating the registry. +type Option func(r *registry) + +// Logger overrides the logger used to record requests to the registry. +func Logger(l *log.Logger) Option { + return func(r *registry) { + r.log = l + r.manifests.log = l + r.blobs.log = l + } +} + +// WithReferrersSupport enables the referrers API endpoint (OCI 1.1+) +func WithReferrersSupport(enabled bool) Option { + return func(r *registry) { + r.referrersEnabled = enabled + } +} diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go new file mode 100644 index 0000000..0ee492e --- /dev/null +++ b/pkg/registry/registry_test.go @@ -0,0 +1,609 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry_test + +import ( + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +const ( + weirdIndex = `{ + "manifests": [ + { + "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + "mediaType":"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" + },{ + "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + "mediaType":"application/xml" + },{ + "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + "mediaType":"application/vnd.oci.image.manifest.v1+json" + } + ] +}` +) + +func sha256String(s string) string { + h, _, _ := v1.SHA256(strings.NewReader(s)) + return h.Hex +} + +func TestCalls(t *testing.T) { + tcs := []struct { + Description string + + // Request / setup + URL string + Digests map[string]string + Manifests map[string]string + BlobStream map[string]string + RequestHeader map[string]string + + // Response + Code int + Header map[string]string + Method string + Body string // request body to send + Want string // response body to expect + }{ + { + Description: "/v2 returns 200", + Method: "GET", + URL: "/v2", + Code: http.StatusOK, + Header: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, + }, + { + Description: "/v2/ returns 200", + Method: "GET", + URL: "/v2/", + Code: http.StatusOK, + Header: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, + }, + { + Description: "/v2/bad returns 404", + Method: "GET", + URL: "/v2/bad", + Code: http.StatusNotFound, + Header: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, + }, + { + Description: "GET non existent blob", + Method: "GET", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusNotFound, + }, + { + Description: "HEAD non existent blob", + Method: "HEAD", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusNotFound, + }, + { + Description: "GET bad digest", + Method: "GET", + URL: "/v2/foo/blobs/sha256:asd", + Code: http.StatusBadRequest, + }, + { + Description: "HEAD bad digest", + Method: "HEAD", + URL: "/v2/foo/blobs/sha256:asd", + Code: http.StatusBadRequest, + }, + { + Description: "bad blob verb", + Method: "FOO", + URL: "/v2/foo/blobs/sha256:asd", + Code: http.StatusBadRequest, + }, + { + Description: "GET containerless blob", + Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, + Method: "GET", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusOK, + Header: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, + Want: "foo", + }, + { + Description: "GET blob", + Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, + Method: "GET", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusOK, + Header: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, + Want: "foo", + }, + { + Description: "HEAD blob", + Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, + Method: "HEAD", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusOK, + Header: map[string]string{ + "Content-Length": "3", + "Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + }, + }, + { + Description: "DELETE blob", + Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, + Method: "DELETE", + URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusAccepted, + }, + { + Description: "blob url with no container", + Method: "GET", + URL: "/v2/blobs/sha256:asd", + Code: http.StatusBadRequest, + }, + { + Description: "uploadurl", + Method: "POST", + URL: "/v2/foo/blobs/uploads", + Code: http.StatusAccepted, + Header: map[string]string{"Range": "0-0"}, + }, + { + Description: "uploadurl", + Method: "POST", + URL: "/v2/foo/blobs/uploads/", + Code: http.StatusAccepted, + Header: map[string]string{"Range": "0-0"}, + }, + { + Description: "upload put missing digest", + Method: "PUT", + URL: "/v2/foo/blobs/uploads/1", + Code: http.StatusBadRequest, + }, + { + Description: "monolithic upload good digest", + Method: "POST", + URL: "/v2/foo/blobs/uploads?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusCreated, + Body: "foo", + Header: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, + }, + { + Description: "monolithic upload bad digest", + Method: "POST", + URL: "/v2/foo/blobs/uploads?digest=sha256:fake", + Code: http.StatusBadRequest, + Body: "foo", + }, + { + Description: "upload good digest", + Method: "PUT", + URL: "/v2/foo/blobs/uploads/1?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + Code: http.StatusCreated, + Body: "foo", + Header: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, + }, + { + Description: "upload bad digest", + Method: "PUT", + URL: "/v2/foo/blobs/uploads/1?digest=sha256:baddigest", + Code: http.StatusBadRequest, + Body: "foo", + }, + { + Description: "stream upload", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + Code: http.StatusNoContent, + Body: "foo", + Header: map[string]string{ + "Range": "0-2", + "Location": "/v2/foo/blobs/uploads/1", + }, + }, + { + Description: "stream duplicate upload", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + Code: http.StatusBadRequest, + Body: "foo", + BlobStream: map[string]string{"1": "foo"}, + }, + { + Description: "stream finish upload", + Method: "PUT", + URL: "/v2/foo/blobs/uploads/1?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + BlobStream: map[string]string{"1": "foo"}, + Code: http.StatusCreated, + Header: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, + }, + { + Description: "get missing manifest", + Method: "GET", + URL: "/v2/foo/manifests/latest", + Code: http.StatusNotFound, + }, + { + Description: "head missing manifest", + Method: "HEAD", + URL: "/v2/foo/manifests/latest", + Code: http.StatusNotFound, + }, + { + Description: "get missing manifest good container", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "GET", + URL: "/v2/foo/manifests/bar", + Code: http.StatusNotFound, + }, + { + Description: "head missing manifest good container", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "HEAD", + URL: "/v2/foo/manifests/bar", + Code: http.StatusNotFound, + }, + { + Description: "get manifest by tag", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "GET", + URL: "/v2/foo/manifests/latest", + Code: http.StatusOK, + Want: "foo", + }, + { + Description: "get manifest by digest", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "GET", + URL: "/v2/foo/manifests/sha256:" + sha256String("foo"), + Code: http.StatusOK, + Want: "foo", + }, + { + Description: "head manifest", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "HEAD", + URL: "/v2/foo/manifests/latest", + Code: http.StatusOK, + }, + { + Description: "create manifest", + Method: "PUT", + URL: "/v2/foo/manifests/latest", + Code: http.StatusCreated, + Body: "foo", + }, + { + Description: "create index", + Method: "PUT", + URL: "/v2/foo/manifests/latest", + Code: http.StatusCreated, + Body: weirdIndex, + RequestHeader: map[string]string{ + "Content-Type": "application/vnd.oci.image.index.v1+json", + }, + Manifests: map[string]string{"foo/manifests/image": "foo"}, + }, + { + Description: "create index missing child", + Method: "PUT", + URL: "/v2/foo/manifests/latest", + Code: http.StatusNotFound, + Body: weirdIndex, + RequestHeader: map[string]string{ + "Content-Type": "application/vnd.oci.image.index.v1+json", + }, + }, + { + Description: "bad index body", + Method: "PUT", + URL: "/v2/foo/manifests/latest", + Code: http.StatusBadRequest, + Body: "foo", + RequestHeader: map[string]string{ + "Content-Type": "application/vnd.oci.image.index.v1+json", + }, + }, + { + Description: "bad manifest method", + Method: "BAR", + URL: "/v2/foo/manifests/latest", + Code: http.StatusBadRequest, + }, + { + Description: "Chunk upload start", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + RequestHeader: map[string]string{"Content-Range": "0-3"}, + Code: http.StatusNoContent, + Body: "foo", + Header: map[string]string{ + "Range": "0-2", + "Location": "/v2/foo/blobs/uploads/1", + }, + }, + { + Description: "Chunk upload bad content range", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + RequestHeader: map[string]string{"Content-Range": "0-bar"}, + Code: http.StatusRequestedRangeNotSatisfiable, + Body: "foo", + }, + { + Description: "Chunk upload overlaps previous data", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + BlobStream: map[string]string{"1": "foo"}, + RequestHeader: map[string]string{"Content-Range": "2-5"}, + Code: http.StatusRequestedRangeNotSatisfiable, + Body: "bar", + }, + { + Description: "Chunk upload after previous data", + Method: "PATCH", + URL: "/v2/foo/blobs/uploads/1", + BlobStream: map[string]string{"1": "foo"}, + RequestHeader: map[string]string{"Content-Range": "3-6"}, + Code: http.StatusNoContent, + Body: "bar", + Header: map[string]string{ + "Range": "0-5", + "Location": "/v2/foo/blobs/uploads/1", + }, + }, + { + Description: "DELETE Unknown name", + Method: "DELETE", + URL: "/v2/test/honk/manifests/latest", + Code: http.StatusNotFound, + }, + { + Description: "DELETE Unknown manifest", + Manifests: map[string]string{"honk/manifests/latest": "honk"}, + Method: "DELETE", + URL: "/v2/honk/manifests/tag-honk", + Code: http.StatusNotFound, + }, + { + Description: "DELETE existing manifest", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "DELETE", + URL: "/v2/foo/manifests/latest", + Code: http.StatusAccepted, + }, + { + Description: "DELETE existing manifest by digest", + Manifests: map[string]string{"foo/manifests/latest": "foo"}, + Method: "DELETE", + URL: "/v2/foo/manifests/sha256:" + sha256String("foo"), + Code: http.StatusAccepted, + }, + { + Description: "list tags", + Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, + Method: "GET", + URL: "/v2/foo/tags/list?n=1000", + Code: http.StatusOK, + Want: `{"name":"foo","tags":["latest","tag1"]}`, + }, + { + Description: "limit tags", + Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, + Method: "GET", + URL: "/v2/foo/tags/list?n=1", + Code: http.StatusOK, + Want: `{"name":"foo","tags":["latest"]}`, + }, + { + Description: "offset tags", + Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, + Method: "GET", + URL: "/v2/foo/tags/list?last=latest", + Code: http.StatusOK, + Want: `{"name":"foo","tags":["tag1"]}`, + }, + { + Description: "list non existing tags", + Method: "GET", + URL: "/v2/foo/tags/list?n=1000", + Code: http.StatusNotFound, + }, + { + Description: "list repos", + Manifests: map[string]string{"foo/manifests/latest": "foo", "bar/manifests/latest": "bar"}, + Method: "GET", + URL: "/v2/_catalog?n=1000", + Code: http.StatusOK, + }, + { + Description: "fetch references", + Method: "GET", + URL: "/v2/foo/referrers/sha256:" + sha256String("foo"), + Code: http.StatusOK, + Manifests: map[string]string{ + "foo/manifests/image": "foo", + "foo/manifests/points-to-image": "{\"subject\": {\"digest\": \"sha256:" + sha256String("foo") + "\"}}", + }, + }, + { + Description: "fetch references, subject pointing elsewhere", + Method: "GET", + URL: "/v2/foo/referrers/sha256:" + sha256String("foo"), + Code: http.StatusOK, + Manifests: map[string]string{ + "foo/manifests/image": "foo", + "foo/manifests/points-to-image": "{\"subject\": {\"digest\": \"sha256:" + sha256String("nonexistant") + "\"}}", + }, + }, + { + Description: "fetch references, no results", + Method: "GET", + URL: "/v2/foo/referrers/sha256:" + sha256String("foo"), + Code: http.StatusOK, + Manifests: map[string]string{ + "foo/manifests/image": "foo", + }, + }, + { + Description: "fetch references, missing repo", + Method: "GET", + URL: "/v2/does-not-exist/referrers/sha256:" + sha256String("foo"), + Code: http.StatusNotFound, + }, + { + Description: "fetch references, bad target (tag vs. digest)", + Method: "GET", + URL: "/v2/foo/referrers/latest", + Code: http.StatusBadRequest, + }, + { + Description: "fetch references, bad method", + Method: "POST", + URL: "/v2/foo/referrers/sha256:" + sha256String("foo"), + Code: http.StatusBadRequest, + }, + } + + for _, tc := range tcs { + + var logger *log.Logger + testf := func(t *testing.T) { + + opts := []registry.Option{registry.WithReferrersSupport(true)} + if logger != nil { + opts = append(opts, registry.Logger(logger)) + } + r := registry.New(opts...) + s := httptest.NewServer(r) + defer s.Close() + + for manifest, contents := range tc.Manifests { + u, err := url.Parse(s.URL + "/v2/" + manifest) + if err != nil { + t.Fatalf("Error parsing %q: %v", s.URL+"/v2", err) + } + req := &http.Request{ + Method: "PUT", + URL: u, + Body: io.NopCloser(strings.NewReader(contents)), + } + t.Log(req.Method, req.URL) + resp, err := s.Client().Do(req) + if err != nil { + t.Fatalf("Error uploading manifest: %v", err) + } + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("Error uploading manifest got status: %d %s", resp.StatusCode, body) + } + t.Logf("created manifest with digest %v", resp.Header.Get("Docker-Content-Digest")) + } + + for digest, contents := range tc.Digests { + u, err := url.Parse(fmt.Sprintf("%s/v2/foo/blobs/uploads/1?digest=%s", s.URL, digest)) + if err != nil { + t.Fatalf("Error parsing %q: %v", s.URL+tc.URL, err) + } + req := &http.Request{ + Method: "PUT", + URL: u, + Body: io.NopCloser(strings.NewReader(contents)), + } + t.Log(req.Method, req.URL) + resp, err := s.Client().Do(req) + if err != nil { + t.Fatalf("Error uploading digest: %v", err) + } + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("Error uploading digest got status: %d %s", resp.StatusCode, body) + } + } + + for upload, contents := range tc.BlobStream { + u, err := url.Parse(fmt.Sprintf("%s/v2/foo/blobs/uploads/%s", s.URL, upload)) + if err != nil { + t.Fatalf("Error parsing %q: %v", s.URL+tc.URL, err) + } + req := &http.Request{ + Method: "PATCH", + URL: u, + Body: io.NopCloser(strings.NewReader(contents)), + } + t.Log(req.Method, req.URL) + resp, err := s.Client().Do(req) + if err != nil { + t.Fatalf("Error streaming blob: %v", err) + } + if resp.StatusCode != http.StatusNoContent { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("Error streaming blob: %d %s", resp.StatusCode, body) + } + + } + + u, err := url.Parse(s.URL + tc.URL) + if err != nil { + t.Fatalf("Error parsing %q: %v", s.URL+tc.URL, err) + } + req := &http.Request{ + Method: tc.Method, + URL: u, + Body: io.NopCloser(strings.NewReader(tc.Body)), + Header: map[string][]string{}, + } + for k, v := range tc.RequestHeader { + req.Header.Set(k, v) + } + t.Log(req.Method, req.URL) + resp, err := s.Client().Do(req) + if err != nil { + t.Fatalf("Error getting %q: %v", tc.URL, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Reading response body: %v", err) + } + if resp.StatusCode != tc.Code { + t.Errorf("Incorrect status code, got %d, want %d; body: %s", resp.StatusCode, tc.Code, body) + } + + for k, v := range tc.Header { + r := resp.Header.Get(k) + if r != v { + t.Errorf("Incorrect header %q received, got %q, want %q", k, r, v) + } + } + + if tc.Want != "" && string(body) != tc.Want { + t.Errorf("Incorrect response body, got %q, want %q", body, tc.Want) + } + } + t.Run(tc.Description, testf) + logger = log.New(io.Discard, "", log.Ldate) + t.Run(tc.Description+" - custom log", testf) + } +} diff --git a/pkg/registry/tls.go b/pkg/registry/tls.go new file mode 100644 index 0000000..cb2644e --- /dev/null +++ b/pkg/registry/tls.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "net/http/httptest" + + ggcrtest "github.com/google/go-containerregistry/internal/httptest" +) + +// TLS returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain +// which should correspond to the domain the image is stored in. +// If you need a transport, Client().Transport is correctly configured. +func TLS(domain string) (*httptest.Server, error) { + return ggcrtest.NewTLSServer(domain, New()) +} diff --git a/pkg/registry/tls_test.go b/pkg/registry/tls_test.go new file mode 100644 index 0000000..0f65b70 --- /dev/null +++ b/pkg/registry/tls_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry_test + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func TestTLS(t *testing.T) { + s, err := registry.TLS("registry.example.com") + if err != nil { + t.Fatal(err) + } + defer s.Close() + + i, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("Unable to make image: %v", err) + } + rd, err := i.Digest() + if err != nil { + t.Fatalf("Unable to get image digest: %v", err) + } + + d, err := name.NewDigest("registry.example.com/foo@" + rd.String()) + if err != nil { + t.Fatalf("Unable to parse digest: %v", err) + } + if err := remote.Write(d, i, remote.WithTransport(s.Client().Transport)); err != nil { + t.Fatalf("Unable to write image to remote: %s", err) + } +} diff --git a/pkg/v1/cache/cache.go b/pkg/v1/cache/cache.go new file mode 100644 index 0000000..31d9c93 --- /dev/null +++ b/pkg/v1/cache/cache.go @@ -0,0 +1,194 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cache provides methods to cache layers. +package cache + +import ( + "errors" + "io" + + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Cache encapsulates methods to interact with cached layers. +type Cache interface { + // Put writes the Layer to the Cache. + // + // The returned Layer should be used for future operations, since lazy + // cachers might only populate the cache when the layer is actually + // consumed. + // + // The returned layer can be consumed, and the cache entry populated, + // by calling either Compressed or Uncompressed and consuming the + // returned io.ReadCloser. + Put(v1.Layer) (v1.Layer, error) + + // Get returns the Layer cached by the given Hash, or ErrNotFound if no + // such layer was found. + Get(v1.Hash) (v1.Layer, error) + + // Delete removes the Layer with the given Hash from the Cache. + Delete(v1.Hash) error +} + +// ErrNotFound is returned by Get when no layer with the given Hash is found. +var ErrNotFound = errors.New("layer was not found") + +// Image returns a new Image which wraps the given Image, whose layers will be +// pulled from the Cache if they are found, and written to the Cache as they +// are read from the underlying Image. +func Image(i v1.Image, c Cache) v1.Image { + return &image{ + Image: i, + c: c, + } +} + +type image struct { + v1.Image + c Cache +} + +func (i *image) Layers() ([]v1.Layer, error) { + ls, err := i.Image.Layers() + if err != nil { + return nil, err + } + + out := make([]v1.Layer, len(ls)) + for idx, l := range ls { + out[idx] = &lazyLayer{inner: l, c: i.c} + } + return out, nil +} + +type lazyLayer struct { + inner v1.Layer + c Cache +} + +func (l *lazyLayer) Compressed() (io.ReadCloser, error) { + digest, err := l.inner.Digest() + if err != nil { + return nil, err + } + + if cl, err := l.c.Get(digest); err == nil { + // Layer found in the cache. + logs.Progress.Printf("Layer %s found (compressed) in cache", digest) + return cl.Compressed() + } else if !errors.Is(err, ErrNotFound) { + return nil, err + } + + // Not cached, pull and return the real layer. + logs.Progress.Printf("Layer %s not found (compressed) in cache, getting", digest) + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + return rl.Compressed() +} + +func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { + diffID, err := l.inner.DiffID() + if err != nil { + return nil, err + } + if cl, err := l.c.Get(diffID); err == nil { + // Layer found in the cache. + logs.Progress.Printf("Layer %s found (uncompressed) in cache", diffID) + return cl.Uncompressed() + } else if !errors.Is(err, ErrNotFound) { + return nil, err + } + + // Not cached, pull and return the real layer. + logs.Progress.Printf("Layer %s not found (uncompressed) in cache, getting", diffID) + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + return rl.Uncompressed() +} + +func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } +func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.DiffID() } +func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } + +func (i *image) LayerByDigest(h v1.Hash) (v1.Layer, error) { + l, err := i.c.Get(h) + if errors.Is(err, ErrNotFound) { + // Not cached, get it and write it. + l, err := i.Image.LayerByDigest(h) + if err != nil { + return nil, err + } + return i.c.Put(l) + } + return l, err +} + +func (i *image) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + l, err := i.c.Get(h) + if errors.Is(err, ErrNotFound) { + // Not cached, get it and write it. + l, err := i.Image.LayerByDiffID(h) + if err != nil { + return nil, err + } + return i.c.Put(l) + } + return l, err +} + +// ImageIndex returns a new ImageIndex which wraps the given ImageIndex's +// children with either Image(child, c) or ImageIndex(child, c) depending on type. +func ImageIndex(ii v1.ImageIndex, c Cache) v1.ImageIndex { + return &imageIndex{ + inner: ii, + c: c, + } +} + +type imageIndex struct { + inner v1.ImageIndex + c Cache +} + +func (ii *imageIndex) MediaType() (types.MediaType, error) { return ii.inner.MediaType() } +func (ii *imageIndex) Digest() (v1.Hash, error) { return ii.inner.Digest() } +func (ii *imageIndex) Size() (int64, error) { return ii.inner.Size() } +func (ii *imageIndex) IndexManifest() (*v1.IndexManifest, error) { return ii.inner.IndexManifest() } +func (ii *imageIndex) RawManifest() ([]byte, error) { return ii.inner.RawManifest() } + +func (ii *imageIndex) Image(h v1.Hash) (v1.Image, error) { + i, err := ii.inner.Image(h) + if err != nil { + return nil, err + } + return Image(i, ii.c), nil +} + +func (ii *imageIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + idx, err := ii.inner.ImageIndex(h) + if err != nil { + return nil, err + } + return ImageIndex(idx, ii.c), nil +} diff --git a/pkg/v1/cache/cache_test.go b/pkg/v1/cache/cache_test.go new file mode 100644 index 0000000..ee5091b --- /dev/null +++ b/pkg/v1/cache/cache_test.go @@ -0,0 +1,154 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "errors" + "io" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestImage(t *testing.T) { + img, err := random.Image(1024, 5) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + m := &memcache{map[v1.Hash]v1.Layer{}} + img = Image(img, m) + + // Validate twice to hit the cache. + if err := validate.Image(img); err != nil { + t.Errorf("Validate: %v", err) + } + if err := validate.Image(img); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestImageIndex(t *testing.T) { + // ImageIndex with child Image and ImageIndex manifests. + ii, err := random.Index(1024, 5, 2) + if err != nil { + t.Fatalf("random.Index: %v", err) + } + iiChild, err := random.Index(1024, 5, 2) + if err != nil { + t.Fatalf("random.Index: %v", err) + } + ii = mutate.AppendManifests(ii, mutate.IndexAddendum{Add: iiChild}) + + m := &memcache{map[v1.Hash]v1.Layer{}} + ii = ImageIndex(ii, m) + + // Validate twice to hit the cache. + if err := validate.Index(ii); err != nil { + t.Errorf("Validate: %v", err) + } + if err := validate.Index(ii); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestLayersLazy(t *testing.T) { + img, err := random.Image(1024, 5) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + m := &memcache{map[v1.Hash]v1.Layer{}} + img = Image(img, m) + + layers, err := img.Layers() + if err != nil { + t.Fatalf("img.Layers: %v", err) + } + + // After calling Layers, nothing is cached. + if got, want := len(m.m), 0; got != want { + t.Errorf("Cache has %d entries, want %d", got, want) + } + + rc, err := layers[0].Uncompressed() + if err != nil { + t.Fatalf("layer.Uncompressed: %v", err) + } + io.Copy(io.Discard, rc) + + if got, expected := len(m.m), 1; got != expected { + t.Errorf("expected %v layers in cache after reading, got %v", expected, got) + } +} + +// TestCacheShortCircuit tests that if a layer is found in the cache, +// LayerByDigest is not called in the underlying Image implementation. +func TestCacheShortCircuit(t *testing.T) { + l := &fakeLayer{} + m := &memcache{map[v1.Hash]v1.Layer{ + fakeHash: l, + }} + img := Image(&fakeImage{}, m) + + for i := 0; i < 10; i++ { + if _, err := img.LayerByDigest(fakeHash); err != nil { + t.Errorf("LayerByDigest[%d]: %v", i, err) + } + } +} + +var fakeHash = v1.Hash{Algorithm: "fake", Hex: "data"} + +type fakeLayer struct{ v1.Layer } +type fakeImage struct{ v1.Image } + +func (f *fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return nil, errors.New("LayerByDigest was called") +} + +// memcache is an in-memory Cache implementation. +// +// It doesn't intend to actually write layer data, it just keeps a reference +// to the original Layer. +// +// It only assumes/considers compressed layers, and so only writes layers by +// digest. +type memcache struct { + m map[v1.Hash]v1.Layer +} + +func (m *memcache) Put(l v1.Layer) (v1.Layer, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + m.m[digest] = l + return l, nil +} + +func (m *memcache) Get(h v1.Hash) (v1.Layer, error) { + l, found := m.m[h] + if !found { + return nil, ErrNotFound + } + return l, nil +} + +func (m *memcache) Delete(h v1.Hash) error { + delete(m.m, h) + return nil +} diff --git a/pkg/v1/cache/example_test.go b/pkg/v1/cache/example_test.go new file mode 100644 index 0000000..7f20474 --- /dev/null +++ b/pkg/v1/cache/example_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache_test + +import ( + "fmt" + "log" + "os" + + "github.com/google/go-containerregistry/pkg/v1/cache" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func ExampleImage() { + img, err := random.Image(1024*1024, 3) + if err != nil { + log.Fatal(err) + } + dir, err := os.MkdirTemp("", "") + if err != nil { + log.Fatal(err) + } + fs := cache.NewFilesystemCache(dir) + + // cached will cache layers from img using the fs cache + cached := cache.Image(img, fs) + + // Use cached as you would use img. + digest, err := cached.Digest() + if err != nil { + log.Fatal(err) + } + fmt.Println(digest) +} diff --git a/pkg/v1/cache/fs.go b/pkg/v1/cache/fs.go new file mode 100644 index 0000000..75b826e --- /dev/null +++ b/pkg/v1/cache/fs.go @@ -0,0 +1,151 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type fscache struct { + path string +} + +// NewFilesystemCache returns a Cache implementation backed by files. +func NewFilesystemCache(path string) Cache { + return &fscache{path} +} + +func (fs *fscache) Put(l v1.Layer) (v1.Layer, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + diffID, err := l.DiffID() + if err != nil { + return nil, err + } + return &layer{ + Layer: l, + path: fs.path, + digest: digest, + diffID: diffID, + }, nil +} + +type layer struct { + v1.Layer + path string + digest, diffID v1.Hash +} + +func (l *layer) create(h v1.Hash) (io.WriteCloser, error) { + if err := os.MkdirAll(l.path, 0700); err != nil { + return nil, err + } + return os.Create(cachepath(l.path, h)) +} + +func (l *layer) Compressed() (io.ReadCloser, error) { + f, err := l.create(l.digest) + if err != nil { + return nil, err + } + rc, err := l.Layer.Compressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func (l *layer) Uncompressed() (io.ReadCloser, error) { + f, err := l.create(l.diffID) + if err != nil { + return nil, err + } + rc, err := l.Layer.Uncompressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +type readcloser struct { + t io.Reader + closes []func() error +} + +func (rc *readcloser) Read(b []byte) (int, error) { + return rc.t.Read(b) +} + +func (rc *readcloser) Close() error { + // Call all Close methods, even if any returned an error. Return the + // first returned error. + var err error + for _, c := range rc.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} + +func (fs *fscache) Get(h v1.Hash) (v1.Layer, error) { + l, err := tarball.LayerFromFile(cachepath(fs.path, h)) + if os.IsNotExist(err) { + return nil, ErrNotFound + } + if errors.Is(err, io.ErrUnexpectedEOF) { + // Delete and return ErrNotFound because the layer was incomplete. + if err := fs.Delete(h); err != nil { + return nil, err + } + return nil, ErrNotFound + } + return l, err +} + +func (fs *fscache) Delete(h v1.Hash) error { + err := os.Remove(cachepath(fs.path, h)) + if os.IsNotExist(err) { + return ErrNotFound + } + return err +} + +func cachepath(path string, h v1.Hash) string { + var file string + if runtime.GOOS == "windows" { + file = fmt.Sprintf("%s-%s", h.Algorithm, h.Hex) + } else { + file = h.String() + } + return filepath.Join(path, file) +} diff --git a/pkg/v1/cache/fs_test.go b/pkg/v1/cache/fs_test.go new file mode 100644 index 0000000..2e05d29 --- /dev/null +++ b/pkg/v1/cache/fs_test.go @@ -0,0 +1,213 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "errors" + "io" + "os" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestFilesystemCache(t *testing.T) { + dir := t.TempDir() + + numLayers := 5 + img, err := random.Image(10, int64(numLayers)) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + c := NewFilesystemCache(dir) + img = Image(img, c) + + // Read all the (compressed) layers to populate the cache. + ls, err := img.Layers() + if err != nil { + t.Fatalf("Layers: %v", err) + } + for i, l := range ls { + rc, err := l.Compressed() + if err != nil { + t.Fatalf("layer[%d].Compressed: %v", i, err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Fatalf("Error reading contents: %v", err) + } + rc.Close() + } + + // Check that layers exist in the fs cache. + dirEntries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if got, want := len(dirEntries), numLayers; got != want { + t.Errorf("Got %d cached files, want %d", got, want) + } + for _, de := range dirEntries { + fi, err := de.Info() + if err != nil { + t.Fatal(err) + } + if fi.Size() == 0 { + t.Errorf("Cached file %q is empty", fi.Name()) + } + } + + // Read all (uncompressed) layers, those populate the cache too. + for i, l := range ls { + rc, err := l.Uncompressed() + if err != nil { + t.Fatalf("layer[%d].Compressed: %v", i, err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Fatalf("Error reading contents: %v", err) + } + rc.Close() + } + + // Check that double the layers are present now, both compressed and + // uncompressed. + dirEntries, err = os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if got, want := len(dirEntries), numLayers*2; got != want { + t.Errorf("Got %d cached files, want %d", got, want) + } + for _, de := range dirEntries { + fi, err := de.Info() + if err != nil { + t.Fatal(err) + } + if fi.Size() == 0 { + t.Errorf("Cached file %q is empty", fi.Name()) + } + } + + // Delete a cached layer, see it disappear. + l := ls[0] + h, err := l.Digest() + if err != nil { + t.Fatalf("layer.Digest: %v", err) + } + if err := c.Delete(h); err != nil { + t.Errorf("cache.Delete: %v", err) + } + dirEntries, err = os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if got, want := len(dirEntries), numLayers*2-1; got != want { + t.Errorf("Got %d cached files, want %d", got, want) + } + + // Read the image again, see the layer reappear. + for i, l := range ls { + rc, err := l.Compressed() + if err != nil { + t.Fatalf("layer[%d].Compressed: %v", i, err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Fatalf("Error reading contents: %v", err) + } + rc.Close() + } + + // Check that layers exist in the fs cache. + dirEntries, err = os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if got, want := len(dirEntries), numLayers*2; got != want { + t.Errorf("Got %d cached files, want %d", got, want) + } + for _, de := range dirEntries { + fi, err := de.Info() + if err != nil { + t.Fatal(err) + } + if fi.Size() == 0 { + t.Errorf("Cached file %q is empty", fi.Name()) + } + } +} + +func TestErrNotFound(t *testing.T) { + dir := t.TempDir() + + c := NewFilesystemCache(dir) + h := v1.Hash{Algorithm: "fake", Hex: "not-found"} + if _, err := c.Get(h); !errors.Is(err, ErrNotFound) { + t.Errorf("Get(%q): %v", h, err) + } + if err := c.Delete(h); !errors.Is(err, ErrNotFound) { + t.Errorf("Delete(%q): %v", h, err) + } +} + +func TestErrUnexpectedEOF(t *testing.T) { + dir := t.TempDir() + + // create a random layer + l, err := random.Layer(10, types.DockerLayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + rc, err := l.Compressed() + if err != nil { + t.Fatalf("layer.Compressed(): %v", err) + } + + h, err := l.Digest() + if err != nil { + t.Fatalf("layer.Digest(): %v", err) + } + p := cachepath(dir, h) + + // Write only the first segment of the compressed layer to produce an + // UnexpectedEOF error when reading it + buf := make([]byte, 10) + n, err := rc.Read(buf) + if err != nil { + t.Fatalf("Read(buf): %v", err) + } + if err := os.WriteFile(p, buf[:n], 0644); err != nil { + t.Fatalf("os.WriteFile(%s, buf[:%d]): %v", p, n, err) + } + + c := NewFilesystemCache(dir) + + // make sure LayerFromFile returns UnexpectedEOF + if _, err := tarball.LayerFromFile(p); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("tarball.LayerFromFile(%s): expected %v, got %v", p, io.ErrUnexpectedEOF, err) + } + + // Try to Get the layer + if _, err := c.Get(h); !errors.Is(err, ErrNotFound) { + t.Errorf("Get(%q): %v", h, err) + } + + // If we had an UnexpectedEOF and the cache deleted the broken layer no file + // should exist + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("os.Stat(%q): %v", p, err) + } +} diff --git a/pkg/v1/cache/ro.go b/pkg/v1/cache/ro.go new file mode 100644 index 0000000..028a612 --- /dev/null +++ b/pkg/v1/cache/ro.go @@ -0,0 +1,27 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// ReadOnly returns a read-only implementation of the given Cache. +// +// Put and Delete operations are a no-op. +func ReadOnly(c Cache) Cache { return &ro{Cache: c} } + +type ro struct{ Cache } + +func (ro) Put(l v1.Layer) (v1.Layer, error) { return l, nil } +func (ro) Delete(v1.Hash) error { return nil } diff --git a/pkg/v1/cache/ro_test.go b/pkg/v1/cache/ro_test.go new file mode 100644 index 0000000..4ec6b1c --- /dev/null +++ b/pkg/v1/cache/ro_test.go @@ -0,0 +1,79 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func TestReadOnly(t *testing.T) { + m := &memcache{map[v1.Hash]v1.Layer{}} + ro := ReadOnly(m) + + // Populate the cache. + img, err := random.Image(10, 1) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + img = Image(img, m) + ls, err := img.Layers() + if err != nil { + t.Fatalf("Layers: %v", err) + } + if got, want := len(ls), 1; got != want { + t.Fatalf("Layers returned %d layers, want %d", got, want) + } + h, err := ls[0].Digest() + if err != nil { + t.Fatalf("layer.Digest: %v", err) + } + m.m[h] = ls[0] + + // Layer can be read from original cache and RO cache. + if _, err := m.Get(h); err != nil { + t.Fatalf("m.Get: %v", err) + } + if _, err := ro.Get(h); err != nil { + t.Fatalf("ro.Get: %v", err) + } + ln := len(m.m) + + // RO Put is a no-op. + if _, err := ro.Put(ls[0]); err != nil { + t.Fatalf("ro.Put: %v", err) + } + if got, want := len(m.m), ln; got != want { + t.Errorf("After Put, got %v entries, want %v", got, want) + } + + // RO Delete is a no-op. + if err := ro.Delete(h); err != nil { + t.Fatalf("ro.Delete: %v", err) + } + if got, want := len(m.m), ln; got != want { + t.Errorf("After Delete, got %v entries, want %v", got, want) + } + + // Deleting from the underlying RW cache updates RO view. + if err := m.Delete(h); err != nil { + t.Fatalf("m.Delete: %v", err) + } + if got, want := len(m.m), 0; got != want { + t.Errorf("After RW Delete, got %v entries, want %v", got, want) + } +} diff --git a/pkg/v1/config.go b/pkg/v1/config.go new file mode 100644 index 0000000..960c93b --- /dev/null +++ b/pkg/v1/config.go @@ -0,0 +1,151 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "io" + "time" +) + +// ConfigFile is the configuration file that holds the metadata describing +// how to launch a container. See: +// https://github.com/opencontainers/image-spec/blob/master/config.md +// +// docker_version and os.version are not part of the spec but included +// for backwards compatibility. +type ConfigFile struct { + Architecture string `json:"architecture"` + Author string `json:"author,omitempty"` + Container string `json:"container,omitempty"` + Created Time `json:"created,omitempty"` + DockerVersion string `json:"docker_version,omitempty"` + History []History `json:"history,omitempty"` + OS string `json:"os"` + RootFS RootFS `json:"rootfs"` + Config Config `json:"config"` + OSVersion string `json:"os.version,omitempty"` + Variant string `json:"variant,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` +} + +// Platform attempts to generates a Platform from the ConfigFile fields. +func (cf *ConfigFile) Platform() *Platform { + if cf.OS == "" && cf.Architecture == "" && cf.OSVersion == "" && cf.Variant == "" && len(cf.OSFeatures) == 0 { + return nil + } + return &Platform{ + OS: cf.OS, + Architecture: cf.Architecture, + OSVersion: cf.OSVersion, + Variant: cf.Variant, + OSFeatures: cf.OSFeatures, + } +} + +// History is one entry of a list recording how this container image was built. +type History struct { + Author string `json:"author,omitempty"` + Created Time `json:"created,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Comment string `json:"comment,omitempty"` + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +// Time is a wrapper around time.Time to help with deep copying +type Time struct { + time.Time +} + +// DeepCopyInto creates a deep-copy of the Time value. The underlying time.Time +// type is effectively immutable in the time API, so it is safe to +// copy-by-assign, despite the presence of (unexported) Pointer fields. +func (t *Time) DeepCopyInto(out *Time) { + *out = *t +} + +// RootFS holds the ordered list of file system deltas that comprise the +// container image's root filesystem. +type RootFS struct { + Type string `json:"type"` + DiffIDs []Hash `json:"diff_ids"` +} + +// HealthConfig holds configuration settings for the HEALTHCHECK feature. +type HealthConfig struct { + // Test is the test to perform to check that the container is healthy. + // An empty slice means to inherit the default. + // The options are: + // {} : inherit healthcheck + // {"NONE"} : disable healthcheck + // {"CMD", args...} : exec arguments directly + // {"CMD-SHELL", command} : run command with system's default shell + Test []string `json:",omitempty"` + + // Zero means to inherit. Durations are expressed as integer nanoseconds. + Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. + Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + StartPeriod time.Duration `json:",omitempty"` // The start period for the container to initialize before the retries starts to count down. + + // Retries is the number of consecutive failures needed to consider a container as unhealthy. + // Zero means inherit. + Retries int `json:",omitempty"` +} + +// Config is a submessage of the config file described as: +// +// The execution parameters which SHOULD be used as a base when running +// a container using the image. +// +// The names of the fields in this message are chosen to reflect the JSON +// payload of the Config as defined here: +// https://git.io/vrAET +// and +// https://github.com/opencontainers/image-spec/blob/master/config.md +type Config struct { + AttachStderr bool `json:"AttachStderr,omitempty"` + AttachStdin bool `json:"AttachStdin,omitempty"` + AttachStdout bool `json:"AttachStdout,omitempty"` + Cmd []string `json:"Cmd,omitempty"` + Healthcheck *HealthConfig `json:"Healthcheck,omitempty"` + Domainname string `json:"Domainname,omitempty"` + Entrypoint []string `json:"Entrypoint,omitempty"` + Env []string `json:"Env,omitempty"` + Hostname string `json:"Hostname,omitempty"` + Image string `json:"Image,omitempty"` + Labels map[string]string `json:"Labels,omitempty"` + OnBuild []string `json:"OnBuild,omitempty"` + OpenStdin bool `json:"OpenStdin,omitempty"` + StdinOnce bool `json:"StdinOnce,omitempty"` + Tty bool `json:"Tty,omitempty"` + User string `json:"User,omitempty"` + Volumes map[string]struct{} `json:"Volumes,omitempty"` + WorkingDir string `json:"WorkingDir,omitempty"` + ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"` + ArgsEscaped bool `json:"ArgsEscaped,omitempty"` + NetworkDisabled bool `json:"NetworkDisabled,omitempty"` + MacAddress string `json:"MacAddress,omitempty"` + StopSignal string `json:"StopSignal,omitempty"` + Shell []string `json:"Shell,omitempty"` +} + +// ParseConfigFile parses the io.Reader's contents into a ConfigFile. +func ParseConfigFile(r io.Reader) (*ConfigFile, error) { + cf := ConfigFile{} + if err := json.NewDecoder(r).Decode(&cf); err != nil { + return nil, err + } + return &cf, nil +} diff --git a/pkg/v1/config_test.go b/pkg/v1/config_test.go new file mode 100644 index 0000000..6e190bf --- /dev/null +++ b/pkg/v1/config_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseConfig(t *testing.T) { + got, err := ParseConfigFile(strings.NewReader("{}")) + if err != nil { + t.Fatal(err) + } + want := ConfigFile{} + + if diff := cmp.Diff(want, *got); diff != "" { + t.Errorf("ParseConfigFile({}); (-want +got) %s", diff) + } + + if got, err := ParseConfigFile(strings.NewReader("{")); err == nil { + t.Errorf("expected error, got: %v", got) + } +} diff --git a/pkg/v1/daemon/README.md b/pkg/v1/daemon/README.md new file mode 100644 index 0000000..74fc3a8 --- /dev/null +++ b/pkg/v1/daemon/README.md @@ -0,0 +1,11 @@ +# `daemon` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon) + +The `daemon` package enables reading/writing images from/to the docker daemon. + +It is not fully fleshed out, but is useful for interoperability, see various issues: + +* https://github.com/google/go-containerregistry/issues/205 +* https://github.com/google/go-containerregistry/issues/552 +* https://github.com/google/go-containerregistry/issues/627 diff --git a/pkg/v1/daemon/doc.go b/pkg/v1/daemon/doc.go new file mode 100644 index 0000000..ac05d96 --- /dev/null +++ b/pkg/v1/daemon/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package daemon provides facilities for reading/writing v1.Image from/to +// a running daemon. +package daemon diff --git a/pkg/v1/daemon/image.go b/pkg/v1/daemon/image.go new file mode 100644 index 0000000..55ba833 --- /dev/null +++ b/pkg/v1/daemon/image.go @@ -0,0 +1,203 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "bytes" + "context" + "io" + "sync" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type image struct { + ref name.Reference + opener *imageOpener + tarballImage v1.Image + id *v1.Hash + + once sync.Once + err error +} + +type imageOpener struct { + ref name.Reference + ctx context.Context + + buffered bool + client Client + + once sync.Once + bytes []byte + err error +} + +func (i *imageOpener) saveImage() (io.ReadCloser, error) { + return i.client.ImageSave(i.ctx, []string{i.ref.Name()}) +} + +func (i *imageOpener) bufferedOpener() (io.ReadCloser, error) { + // Store the tarball in memory and return a new reader into the bytes each time we need to access something. + i.once.Do(func() { + i.bytes, i.err = func() ([]byte, error) { + rc, err := i.saveImage() + if err != nil { + return nil, err + } + defer rc.Close() + + return io.ReadAll(rc) + }() + }) + + // Wrap the bytes in a ReadCloser so it looks like an opened file. + return io.NopCloser(bytes.NewReader(i.bytes)), i.err +} + +func (i *imageOpener) opener() tarball.Opener { + if i.buffered { + return i.bufferedOpener + } + + // To avoid storing the tarball in memory, do a save every time we need to access something. + return i.saveImage +} + +// Image provides access to an image reference from the Docker daemon, +// applying functional options to the underlying imageOpener before +// resolving the reference into a v1.Image. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + o, err := makeOptions(options...) + if err != nil { + return nil, err + } + + i := &imageOpener{ + ref: ref, + buffered: o.buffered, + client: o.client, + ctx: o.ctx, + } + + img := &image{ + ref: ref, + opener: i, + } + + // Eagerly fetch Image ID to ensure it actually exists. + // https://github.com/google/go-containerregistry/issues/1186 + id, err := img.ConfigName() + if err != nil { + return nil, err + } + img.id = &id + + return img, nil +} + +func (i *image) initialize() error { + // Don't re-initialize tarball if already initialized. + if i.tarballImage == nil { + i.once.Do(func() { + i.tarballImage, i.err = tarball.Image(i.opener.opener(), nil) + }) + } + return i.err +} + +func (i *image) Layers() ([]v1.Layer, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.Layers() +} + +func (i *image) MediaType() (types.MediaType, error) { + if err := i.initialize(); err != nil { + return "", err + } + return i.tarballImage.MediaType() +} + +func (i *image) Size() (int64, error) { + if err := i.initialize(); err != nil { + return 0, err + } + return i.tarballImage.Size() +} + +func (i *image) ConfigName() (v1.Hash, error) { + if i.id != nil { + return *i.id, nil + } + res, _, err := i.opener.client.ImageInspectWithRaw(i.opener.ctx, i.ref.String()) + if err != nil { + return v1.Hash{}, err + } + return v1.NewHash(res.ID) +} + +func (i *image) ConfigFile() (*v1.ConfigFile, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.ConfigFile() +} + +func (i *image) RawConfigFile() ([]byte, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.RawConfigFile() +} + +func (i *image) Digest() (v1.Hash, error) { + if err := i.initialize(); err != nil { + return v1.Hash{}, err + } + return i.tarballImage.Digest() +} + +func (i *image) Manifest() (*v1.Manifest, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.Manifest() +} + +func (i *image) RawManifest() ([]byte, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.RawManifest() +} + +func (i *image) LayerByDigest(h v1.Hash) (v1.Layer, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.LayerByDigest(h) +} + +func (i *image) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + if err := i.initialize(); err != nil { + return nil, err + } + return i.tarballImage.LayerByDiffID(h) +} diff --git a/pkg/v1/daemon/image_test.go b/pkg/v1/daemon/image_test.go new file mode 100644 index 0000000..6456832 --- /dev/null +++ b/pkg/v1/daemon/image_test.go @@ -0,0 +1,159 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/docker/docker/api/types" + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +var imagePath = "../tarball/testdata/test_image_1.tar" + +type MockClient struct { + Client + path string + negotiated bool + + wantCtx context.Context + + loadErr error + loadBody io.ReadCloser + + saveErr error + saveBody io.ReadCloser +} + +func (m *MockClient) NegotiateAPIVersion(ctx context.Context) { + m.negotiated = true +} + +func (m *MockClient) ImageSave(_ context.Context, _ []string) (io.ReadCloser, error) { + if !m.negotiated { + return nil, errors.New("you forgot to call NegotiateAPIVersion before calling ImageSave") + } + + if m.path != "" { + return os.Open(m.path) + } + + return m.saveBody, m.saveErr +} + +func (m *MockClient) ImageInspectWithRaw(context.Context, string) (types.ImageInspect, []byte, error) { + return types.ImageInspect{ + ID: "sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e", + }, nil, nil +} + +func TestImage(t *testing.T) { + for _, tc := range []struct { + name string + buffered bool + client *MockClient + wantResponse string + wantErr string + }{{ + name: "success", + client: &MockClient{ + path: imagePath, + }, + }, { + name: "save err", + client: &MockClient{ + saveBody: io.NopCloser(strings.NewReader("Loaded")), + saveErr: fmt.Errorf("locked and loaded"), + }, + wantErr: "locked and loaded", + }, { + name: "read err", + client: &MockClient{ + saveBody: io.NopCloser(&errReader{fmt.Errorf("goodbye, world")}), + }, + wantErr: "goodbye, world", + }} { + run := func(t *testing.T) { + opts := []Option{WithClient(tc.client)} + if tc.buffered { + opts = append(opts, WithBufferedOpener()) + } else { + opts = append(opts, WithUnbufferedOpener()) + } + img, err := tarball.ImageFromPath(imagePath, nil) + if err != nil { + t.Fatalf("error loading test image: %s", err) + } + + tag, err := name.NewTag("unused", name.WeakValidation) + if err != nil { + t.Fatalf("error creating test name: %s", err) + } + + dmn, err := Image(tag, opts...) + if err != nil { + if tc.wantErr == "" { + t.Errorf("Error loading daemon image: %s", err) + } else if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("wanted %s to contain %s", err.Error(), tc.wantErr) + } + return + } + err = compare.Images(img, dmn) + if err != nil { + if tc.wantErr == "" { + t.Errorf("compare.Images: %v", err) + } else if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("wanted %s to contain %s", err.Error(), tc.wantErr) + } + } + + err = validate.Image(dmn) + if err != nil { + if tc.wantErr == "" { + t.Errorf("validate.Image: %v", err) + } else if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("wanted %s to contain %s", err.Error(), tc.wantErr) + } + } + } + + tc.buffered = true + t.Run(tc.name+" buffered", run) + + tc.buffered = false + t.Run(tc.name+" unbuffered", run) + } +} + +func TestImageDefaultClient(t *testing.T) { + wantErr := fmt.Errorf("bad client") + defaultClient = func() (Client, error) { + return nil, wantErr + } + + if _, err := Image(name.MustParseReference("unused")); !errors.Is(err, wantErr) { + t.Errorf("Image(): want %v; got %v", wantErr, err) + } +} diff --git a/pkg/v1/daemon/options.go b/pkg/v1/daemon/options.go new file mode 100644 index 0000000..e8a5a1e --- /dev/null +++ b/pkg/v1/daemon/options.go @@ -0,0 +1,103 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "io" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +// ImageOption is an alias for Option. +// Deprecated: Use Option instead. +type ImageOption Option + +// Option is a functional option for daemon operations. +type Option func(*options) + +type options struct { + ctx context.Context + client Client + buffered bool +} + +var defaultClient = func() (Client, error) { + return client.NewClientWithOpts(client.FromEnv) +} + +func makeOptions(opts ...Option) (*options, error) { + o := &options{ + buffered: true, + ctx: context.Background(), + } + for _, opt := range opts { + opt(o) + } + + if o.client == nil { + client, err := defaultClient() + if err != nil { + return nil, err + } + o.client = client + } + o.client.NegotiateAPIVersion(o.ctx) + + return o, nil +} + +// WithBufferedOpener buffers the image. +func WithBufferedOpener() Option { + return func(o *options) { + o.buffered = true + } +} + +// WithUnbufferedOpener streams the image to avoid buffering. +func WithUnbufferedOpener() Option { + return func(o *options) { + o.buffered = false + } +} + +// WithClient is a functional option to allow injecting a docker client. +// +// By default, github.com/docker/docker/client.FromEnv is used. +func WithClient(client Client) Option { + return func(o *options) { + o.client = client + } +} + +// WithContext is a functional option to pass through a context.Context. +// +// By default, context.Background() is used. +func WithContext(ctx context.Context) Option { + return func(o *options) { + o.ctx = ctx + } +} + +// Client represents the subset of a docker client that the daemon +// package uses. +type Client interface { + NegotiateAPIVersion(ctx context.Context) + ImageSave(context.Context, []string) (io.ReadCloser, error) + ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) + ImageTag(context.Context, string, string) error + ImageInspectWithRaw(context.Context, string) (types.ImageInspect, []byte, error) +} diff --git a/pkg/v1/daemon/write.go b/pkg/v1/daemon/write.go new file mode 100644 index 0000000..48186f6 --- /dev/null +++ b/pkg/v1/daemon/write.go @@ -0,0 +1,60 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "fmt" + "io" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Tag adds a tag to an already existent image. +func Tag(src, dest name.Tag, options ...Option) error { + o, err := makeOptions(options...) + if err != nil { + return err + } + + return o.client.ImageTag(o.ctx, src.String(), dest.String()) +} + +// Write saves the image into the daemon as the given tag. +func Write(tag name.Tag, img v1.Image, options ...Option) (string, error) { + o, err := makeOptions(options...) + if err != nil { + return "", err + } + + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(tarball.Write(tag, img, pw)) + }() + + // write the image in docker save format first, then load it + resp, err := o.client.ImageLoad(o.ctx, pr, false) + if err != nil { + return "", fmt.Errorf("error loading image: %w", err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + response := string(b) + if err != nil { + return response, fmt.Errorf("error reading load response body: %w", err) + } + return response, nil +} diff --git a/pkg/v1/daemon/write_test.go b/pkg/v1/daemon/write_test.go new file mode 100644 index 0000000..0e5495c --- /dev/null +++ b/pkg/v1/daemon/write_test.go @@ -0,0 +1,159 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/docker/docker/api/types" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type errReader struct { + err error +} + +func (r *errReader) Read(p []byte) (int, error) { + return 0, r.err +} + +func (m *MockClient) ImageLoad(ctx context.Context, r io.Reader, _ bool) (types.ImageLoadResponse, error) { + if !m.negotiated { + return types.ImageLoadResponse{}, errors.New("you forgot to call NegotiateAPIVersion before calling ImageLoad") + } + if m.wantCtx != nil && m.wantCtx != ctx { + return types.ImageLoadResponse{}, fmt.Errorf("ImageLoad: wrong context") + } + + _, _ = io.Copy(io.Discard, r) + return types.ImageLoadResponse{ + Body: m.loadBody, + }, m.loadErr +} + +func (m *MockClient) ImageTag(ctx context.Context, source, target string) error { + if !m.negotiated { + return errors.New("you forgot to call NegotiateAPIVersion before calling ImageTag") + } + if m.wantCtx != nil && m.wantCtx != ctx { + return fmt.Errorf("ImageTag: wrong context") + } + return nil +} + +func TestWriteImage(t *testing.T) { + for _, tc := range []struct { + name string + client *MockClient + wantResponse string + wantErr string + }{{ + name: "success", + client: &MockClient{ + loadBody: io.NopCloser(strings.NewReader("Loaded")), + }, + wantResponse: "Loaded", + }, { + name: "load err", + client: &MockClient{ + loadBody: io.NopCloser(strings.NewReader("Loaded")), + loadErr: fmt.Errorf("locked and loaded"), + }, + wantErr: "locked and loaded", + }, { + name: "read err", + client: &MockClient{ + loadBody: io.NopCloser(&errReader{fmt.Errorf("goodbye, world")}), + }, + wantErr: "goodbye, world", + }} { + t.Run(tc.name, func(t *testing.T) { + image, err := tarball.ImageFromPath("../tarball/testdata/test_image_1.tar", nil) + if err != nil { + t.Errorf("Error loading image: %v", err.Error()) + } + tag, err := name.NewTag("test_image_2:latest") + if err != nil { + t.Fatal(err) + } + response, err := Write(tag, image, WithClient(tc.client)) + if tc.wantErr == "" { + if err != nil { + t.Errorf("Error writing image tar: %s", err.Error()) + } + } else { + if err == nil { + t.Errorf("expected err") + } else if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("Error writing image tar: wanted %s to contain %s", err.Error(), tc.wantErr) + } + } + if !strings.Contains(response, tc.wantResponse) { + t.Errorf("Error loading image. Response: %s", response) + } + + dst, err := name.NewTag("hello:world") + if err != nil { + t.Fatal(err) + } + if err := Tag(tag, dst, WithClient(tc.client)); err != nil { + t.Errorf("Error tagging image: %v", err) + } + }) + } +} + +func TestWriteDefaultClient(t *testing.T) { + wantErr := fmt.Errorf("bad client") + defaultClient = func() (Client, error) { + return nil, wantErr + } + + tag, err := name.NewTag("test_image_2:latest") + if err != nil { + t.Fatal(err) + } + + if _, err := Write(tag, empty.Image); !errors.Is(err, wantErr) { + t.Errorf("Write(): want %v; got %v", wantErr, err) + } + + if err := Tag(tag, tag); !errors.Is(err, wantErr) { + t.Errorf("Tag(): want %v; got %v", wantErr, err) + } + + // Cover default client init and ctx use as well. + ctx := context.TODO() + defaultClient = func() (Client, error) { + return &MockClient{ + loadBody: io.NopCloser(strings.NewReader("Loaded")), + wantCtx: ctx, + }, nil + } + if err := Tag(tag, tag, WithContext(ctx)); err != nil { + t.Fatal(err) + } + if _, err := Write(tag, empty.Image, WithContext(ctx)); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/v1/doc.go b/pkg/v1/doc.go new file mode 100644 index 0000000..7a84736 --- /dev/null +++ b/pkg/v1/doc.go @@ -0,0 +1,18 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +k8s:deepcopy-gen=package + +// Package v1 defines structured types for OCI v1 images +package v1 diff --git a/pkg/v1/empty/README.md b/pkg/v1/empty/README.md new file mode 100644 index 0000000..8663a83 --- /dev/null +++ b/pkg/v1/empty/README.md @@ -0,0 +1,8 @@ +# `empty` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/empty?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/empty) + +The empty packages provides an empty base for constructing a `v1.Image` or `v1.ImageIndex`. +This is especially useful when paired with the [`mutate`](/pkg/v1/mutate) package, +see [`mutate.Append`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/mutate#Append) +and [`mutate.AppendManifests`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/mutate#AppendManifests). diff --git a/pkg/v1/empty/doc.go b/pkg/v1/empty/doc.go new file mode 100644 index 0000000..1a521e9 --- /dev/null +++ b/pkg/v1/empty/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package empty provides an implementation of v1.Image equivalent to "FROM scratch". +package empty diff --git a/pkg/v1/empty/image.go b/pkg/v1/empty/image.go new file mode 100644 index 0000000..c58a06c --- /dev/null +++ b/pkg/v1/empty/image.go @@ -0,0 +1,52 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package empty + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Image is a singleton empty image, think: FROM scratch. +var Image, _ = partial.UncompressedToImage(emptyImage{}) + +type emptyImage struct{} + +// MediaType implements partial.UncompressedImageCore. +func (i emptyImage) MediaType() (types.MediaType, error) { + return types.DockerManifestSchema2, nil +} + +// RawConfigFile implements partial.UncompressedImageCore. +func (i emptyImage) RawConfigFile() ([]byte, error) { + return partial.RawConfigFile(i) +} + +// ConfigFile implements v1.Image. +func (i emptyImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{ + RootFS: v1.RootFS{ + // Some clients check this. + Type: "layers", + }, + }, nil +} + +func (i emptyImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { + return nil, fmt.Errorf("LayerByDiffID(%s): empty image", h) +} diff --git a/pkg/v1/empty/image_test.go b/pkg/v1/empty/image_test.go new file mode 100644 index 0000000..c9204d9 --- /dev/null +++ b/pkg/v1/empty/image_test.go @@ -0,0 +1,48 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package empty + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestImage(t *testing.T) { + if err := validate.Image(Image); err != nil { + t.Fatalf("validate.Image(empty.Image) = %v", err) + } +} + +func TestManifestAndConfig(t *testing.T) { + manifest, err := Image.Manifest() + if err != nil { + t.Fatalf("Error loading manifest: %v", err) + } + if got, want := len(manifest.Layers), 0; got != want { + t.Fatalf("num layers; got %v, want %v", got, want) + } + + config, err := Image.ConfigFile() + if err != nil { + t.Fatalf("Error loading config file: %v", err) + } + if got, want := len(config.RootFS.DiffIDs), 0; got != want { + t.Fatalf("num diff ids; got %v, want %v", got, want) + } + if got, want := config.RootFS.Type, "layers"; got != want { + t.Fatalf("rootfs type; got %v, want %v", got, want) + } +} diff --git a/pkg/v1/empty/index.go b/pkg/v1/empty/index.go new file mode 100644 index 0000000..1066535 --- /dev/null +++ b/pkg/v1/empty/index.go @@ -0,0 +1,64 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package empty + +import ( + "encoding/json" + "errors" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Index is a singleton empty index, think: FROM scratch. +var Index = emptyIndex{} + +type emptyIndex struct{} + +func (i emptyIndex) MediaType() (types.MediaType, error) { + return types.OCIImageIndex, nil +} + +func (i emptyIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i emptyIndex) Size() (int64, error) { + return partial.Size(i) +} + +func (i emptyIndex) IndexManifest() (*v1.IndexManifest, error) { + return base(), nil +} + +func (i emptyIndex) RawManifest() ([]byte, error) { + return json.Marshal(base()) +} + +func (i emptyIndex) Image(v1.Hash) (v1.Image, error) { + return nil, errors.New("empty index") +} + +func (i emptyIndex) ImageIndex(v1.Hash) (v1.ImageIndex, error) { + return nil, errors.New("empty index") +} + +func base() *v1.IndexManifest { + return &v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + } +} diff --git a/pkg/v1/empty/index_test.go b/pkg/v1/empty/index_test.go new file mode 100644 index 0000000..dd99e10 --- /dev/null +++ b/pkg/v1/empty/index_test.go @@ -0,0 +1,40 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package empty + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestIndex(t *testing.T) { + if err := validate.Index(Index); err != nil { + t.Fatalf("validate.Index(empty.Index) = %v", err) + } + + if mt, err := Index.MediaType(); err != nil || mt != types.OCIImageIndex { + t.Errorf("empty.Index.MediaType() = %v, %v", mt, err) + } + + if _, err := Index.Image(v1.Hash{}); err == nil { + t.Errorf("empty.Index.Image() should always fail") + } + if _, err := Index.ImageIndex(v1.Hash{}); err == nil { + t.Errorf("empty.Index.ImageIndex() should always fail") + } +} diff --git a/pkg/v1/fake/image.go b/pkg/v1/fake/image.go new file mode 100644 index 0000000..f95ac61 --- /dev/null +++ b/pkg/v1/fake/image.go @@ -0,0 +1,826 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package fake + +import ( + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type FakeImage struct { + ConfigFileStub func() (*v1.ConfigFile, error) + configFileMutex sync.RWMutex + configFileArgsForCall []struct { + } + configFileReturns struct { + result1 *v1.ConfigFile + result2 error + } + configFileReturnsOnCall map[int]struct { + result1 *v1.ConfigFile + result2 error + } + ConfigNameStub func() (v1.Hash, error) + configNameMutex sync.RWMutex + configNameArgsForCall []struct { + } + configNameReturns struct { + result1 v1.Hash + result2 error + } + configNameReturnsOnCall map[int]struct { + result1 v1.Hash + result2 error + } + DigestStub func() (v1.Hash, error) + digestMutex sync.RWMutex + digestArgsForCall []struct { + } + digestReturns struct { + result1 v1.Hash + result2 error + } + digestReturnsOnCall map[int]struct { + result1 v1.Hash + result2 error + } + LayerByDiffIDStub func(v1.Hash) (v1.Layer, error) + layerByDiffIDMutex sync.RWMutex + layerByDiffIDArgsForCall []struct { + arg1 v1.Hash + } + layerByDiffIDReturns struct { + result1 v1.Layer + result2 error + } + layerByDiffIDReturnsOnCall map[int]struct { + result1 v1.Layer + result2 error + } + LayerByDigestStub func(v1.Hash) (v1.Layer, error) + layerByDigestMutex sync.RWMutex + layerByDigestArgsForCall []struct { + arg1 v1.Hash + } + layerByDigestReturns struct { + result1 v1.Layer + result2 error + } + layerByDigestReturnsOnCall map[int]struct { + result1 v1.Layer + result2 error + } + LayersStub func() ([]v1.Layer, error) + layersMutex sync.RWMutex + layersArgsForCall []struct { + } + layersReturns struct { + result1 []v1.Layer + result2 error + } + layersReturnsOnCall map[int]struct { + result1 []v1.Layer + result2 error + } + ManifestStub func() (*v1.Manifest, error) + manifestMutex sync.RWMutex + manifestArgsForCall []struct { + } + manifestReturns struct { + result1 *v1.Manifest + result2 error + } + manifestReturnsOnCall map[int]struct { + result1 *v1.Manifest + result2 error + } + MediaTypeStub func() (types.MediaType, error) + mediaTypeMutex sync.RWMutex + mediaTypeArgsForCall []struct { + } + mediaTypeReturns struct { + result1 types.MediaType + result2 error + } + mediaTypeReturnsOnCall map[int]struct { + result1 types.MediaType + result2 error + } + RawConfigFileStub func() ([]byte, error) + rawConfigFileMutex sync.RWMutex + rawConfigFileArgsForCall []struct { + } + rawConfigFileReturns struct { + result1 []byte + result2 error + } + rawConfigFileReturnsOnCall map[int]struct { + result1 []byte + result2 error + } + RawManifestStub func() ([]byte, error) + rawManifestMutex sync.RWMutex + rawManifestArgsForCall []struct { + } + rawManifestReturns struct { + result1 []byte + result2 error + } + rawManifestReturnsOnCall map[int]struct { + result1 []byte + result2 error + } + SizeStub func() (int64, error) + sizeMutex sync.RWMutex + sizeArgsForCall []struct { + } + sizeReturns struct { + result1 int64 + result2 error + } + sizeReturnsOnCall map[int]struct { + result1 int64 + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeImage) ConfigFile() (*v1.ConfigFile, error) { + fake.configFileMutex.Lock() + ret, specificReturn := fake.configFileReturnsOnCall[len(fake.configFileArgsForCall)] + fake.configFileArgsForCall = append(fake.configFileArgsForCall, struct { + }{}) + stub := fake.ConfigFileStub + fakeReturns := fake.configFileReturns + fake.recordInvocation("ConfigFile", []interface{}{}) + fake.configFileMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) ConfigFileCallCount() int { + fake.configFileMutex.RLock() + defer fake.configFileMutex.RUnlock() + return len(fake.configFileArgsForCall) +} + +func (fake *FakeImage) ConfigFileCalls(stub func() (*v1.ConfigFile, error)) { + fake.configFileMutex.Lock() + defer fake.configFileMutex.Unlock() + fake.ConfigFileStub = stub +} + +func (fake *FakeImage) ConfigFileReturns(result1 *v1.ConfigFile, result2 error) { + fake.configFileMutex.Lock() + defer fake.configFileMutex.Unlock() + fake.ConfigFileStub = nil + fake.configFileReturns = struct { + result1 *v1.ConfigFile + result2 error + }{result1, result2} +} + +func (fake *FakeImage) ConfigFileReturnsOnCall(i int, result1 *v1.ConfigFile, result2 error) { + fake.configFileMutex.Lock() + defer fake.configFileMutex.Unlock() + fake.ConfigFileStub = nil + if fake.configFileReturnsOnCall == nil { + fake.configFileReturnsOnCall = make(map[int]struct { + result1 *v1.ConfigFile + result2 error + }) + } + fake.configFileReturnsOnCall[i] = struct { + result1 *v1.ConfigFile + result2 error + }{result1, result2} +} + +func (fake *FakeImage) ConfigName() (v1.Hash, error) { + fake.configNameMutex.Lock() + ret, specificReturn := fake.configNameReturnsOnCall[len(fake.configNameArgsForCall)] + fake.configNameArgsForCall = append(fake.configNameArgsForCall, struct { + }{}) + stub := fake.ConfigNameStub + fakeReturns := fake.configNameReturns + fake.recordInvocation("ConfigName", []interface{}{}) + fake.configNameMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) ConfigNameCallCount() int { + fake.configNameMutex.RLock() + defer fake.configNameMutex.RUnlock() + return len(fake.configNameArgsForCall) +} + +func (fake *FakeImage) ConfigNameCalls(stub func() (v1.Hash, error)) { + fake.configNameMutex.Lock() + defer fake.configNameMutex.Unlock() + fake.ConfigNameStub = stub +} + +func (fake *FakeImage) ConfigNameReturns(result1 v1.Hash, result2 error) { + fake.configNameMutex.Lock() + defer fake.configNameMutex.Unlock() + fake.ConfigNameStub = nil + fake.configNameReturns = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImage) ConfigNameReturnsOnCall(i int, result1 v1.Hash, result2 error) { + fake.configNameMutex.Lock() + defer fake.configNameMutex.Unlock() + fake.ConfigNameStub = nil + if fake.configNameReturnsOnCall == nil { + fake.configNameReturnsOnCall = make(map[int]struct { + result1 v1.Hash + result2 error + }) + } + fake.configNameReturnsOnCall[i] = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImage) Digest() (v1.Hash, error) { + fake.digestMutex.Lock() + ret, specificReturn := fake.digestReturnsOnCall[len(fake.digestArgsForCall)] + fake.digestArgsForCall = append(fake.digestArgsForCall, struct { + }{}) + stub := fake.DigestStub + fakeReturns := fake.digestReturns + fake.recordInvocation("Digest", []interface{}{}) + fake.digestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) DigestCallCount() int { + fake.digestMutex.RLock() + defer fake.digestMutex.RUnlock() + return len(fake.digestArgsForCall) +} + +func (fake *FakeImage) DigestCalls(stub func() (v1.Hash, error)) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = stub +} + +func (fake *FakeImage) DigestReturns(result1 v1.Hash, result2 error) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = nil + fake.digestReturns = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImage) DigestReturnsOnCall(i int, result1 v1.Hash, result2 error) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = nil + if fake.digestReturnsOnCall == nil { + fake.digestReturnsOnCall = make(map[int]struct { + result1 v1.Hash + result2 error + }) + } + fake.digestReturnsOnCall[i] = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImage) LayerByDiffID(arg1 v1.Hash) (v1.Layer, error) { + fake.layerByDiffIDMutex.Lock() + ret, specificReturn := fake.layerByDiffIDReturnsOnCall[len(fake.layerByDiffIDArgsForCall)] + fake.layerByDiffIDArgsForCall = append(fake.layerByDiffIDArgsForCall, struct { + arg1 v1.Hash + }{arg1}) + stub := fake.LayerByDiffIDStub + fakeReturns := fake.layerByDiffIDReturns + fake.recordInvocation("LayerByDiffID", []interface{}{arg1}) + fake.layerByDiffIDMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) LayerByDiffIDCallCount() int { + fake.layerByDiffIDMutex.RLock() + defer fake.layerByDiffIDMutex.RUnlock() + return len(fake.layerByDiffIDArgsForCall) +} + +func (fake *FakeImage) LayerByDiffIDCalls(stub func(v1.Hash) (v1.Layer, error)) { + fake.layerByDiffIDMutex.Lock() + defer fake.layerByDiffIDMutex.Unlock() + fake.LayerByDiffIDStub = stub +} + +func (fake *FakeImage) LayerByDiffIDArgsForCall(i int) v1.Hash { + fake.layerByDiffIDMutex.RLock() + defer fake.layerByDiffIDMutex.RUnlock() + argsForCall := fake.layerByDiffIDArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeImage) LayerByDiffIDReturns(result1 v1.Layer, result2 error) { + fake.layerByDiffIDMutex.Lock() + defer fake.layerByDiffIDMutex.Unlock() + fake.LayerByDiffIDStub = nil + fake.layerByDiffIDReturns = struct { + result1 v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) LayerByDiffIDReturnsOnCall(i int, result1 v1.Layer, result2 error) { + fake.layerByDiffIDMutex.Lock() + defer fake.layerByDiffIDMutex.Unlock() + fake.LayerByDiffIDStub = nil + if fake.layerByDiffIDReturnsOnCall == nil { + fake.layerByDiffIDReturnsOnCall = make(map[int]struct { + result1 v1.Layer + result2 error + }) + } + fake.layerByDiffIDReturnsOnCall[i] = struct { + result1 v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) LayerByDigest(arg1 v1.Hash) (v1.Layer, error) { + fake.layerByDigestMutex.Lock() + ret, specificReturn := fake.layerByDigestReturnsOnCall[len(fake.layerByDigestArgsForCall)] + fake.layerByDigestArgsForCall = append(fake.layerByDigestArgsForCall, struct { + arg1 v1.Hash + }{arg1}) + stub := fake.LayerByDigestStub + fakeReturns := fake.layerByDigestReturns + fake.recordInvocation("LayerByDigest", []interface{}{arg1}) + fake.layerByDigestMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) LayerByDigestCallCount() int { + fake.layerByDigestMutex.RLock() + defer fake.layerByDigestMutex.RUnlock() + return len(fake.layerByDigestArgsForCall) +} + +func (fake *FakeImage) LayerByDigestCalls(stub func(v1.Hash) (v1.Layer, error)) { + fake.layerByDigestMutex.Lock() + defer fake.layerByDigestMutex.Unlock() + fake.LayerByDigestStub = stub +} + +func (fake *FakeImage) LayerByDigestArgsForCall(i int) v1.Hash { + fake.layerByDigestMutex.RLock() + defer fake.layerByDigestMutex.RUnlock() + argsForCall := fake.layerByDigestArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeImage) LayerByDigestReturns(result1 v1.Layer, result2 error) { + fake.layerByDigestMutex.Lock() + defer fake.layerByDigestMutex.Unlock() + fake.LayerByDigestStub = nil + fake.layerByDigestReturns = struct { + result1 v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) LayerByDigestReturnsOnCall(i int, result1 v1.Layer, result2 error) { + fake.layerByDigestMutex.Lock() + defer fake.layerByDigestMutex.Unlock() + fake.LayerByDigestStub = nil + if fake.layerByDigestReturnsOnCall == nil { + fake.layerByDigestReturnsOnCall = make(map[int]struct { + result1 v1.Layer + result2 error + }) + } + fake.layerByDigestReturnsOnCall[i] = struct { + result1 v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) Layers() ([]v1.Layer, error) { + fake.layersMutex.Lock() + ret, specificReturn := fake.layersReturnsOnCall[len(fake.layersArgsForCall)] + fake.layersArgsForCall = append(fake.layersArgsForCall, struct { + }{}) + stub := fake.LayersStub + fakeReturns := fake.layersReturns + fake.recordInvocation("Layers", []interface{}{}) + fake.layersMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) LayersCallCount() int { + fake.layersMutex.RLock() + defer fake.layersMutex.RUnlock() + return len(fake.layersArgsForCall) +} + +func (fake *FakeImage) LayersCalls(stub func() ([]v1.Layer, error)) { + fake.layersMutex.Lock() + defer fake.layersMutex.Unlock() + fake.LayersStub = stub +} + +func (fake *FakeImage) LayersReturns(result1 []v1.Layer, result2 error) { + fake.layersMutex.Lock() + defer fake.layersMutex.Unlock() + fake.LayersStub = nil + fake.layersReturns = struct { + result1 []v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) LayersReturnsOnCall(i int, result1 []v1.Layer, result2 error) { + fake.layersMutex.Lock() + defer fake.layersMutex.Unlock() + fake.LayersStub = nil + if fake.layersReturnsOnCall == nil { + fake.layersReturnsOnCall = make(map[int]struct { + result1 []v1.Layer + result2 error + }) + } + fake.layersReturnsOnCall[i] = struct { + result1 []v1.Layer + result2 error + }{result1, result2} +} + +func (fake *FakeImage) Manifest() (*v1.Manifest, error) { + fake.manifestMutex.Lock() + ret, specificReturn := fake.manifestReturnsOnCall[len(fake.manifestArgsForCall)] + fake.manifestArgsForCall = append(fake.manifestArgsForCall, struct { + }{}) + stub := fake.ManifestStub + fakeReturns := fake.manifestReturns + fake.recordInvocation("Manifest", []interface{}{}) + fake.manifestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) ManifestCallCount() int { + fake.manifestMutex.RLock() + defer fake.manifestMutex.RUnlock() + return len(fake.manifestArgsForCall) +} + +func (fake *FakeImage) ManifestCalls(stub func() (*v1.Manifest, error)) { + fake.manifestMutex.Lock() + defer fake.manifestMutex.Unlock() + fake.ManifestStub = stub +} + +func (fake *FakeImage) ManifestReturns(result1 *v1.Manifest, result2 error) { + fake.manifestMutex.Lock() + defer fake.manifestMutex.Unlock() + fake.ManifestStub = nil + fake.manifestReturns = struct { + result1 *v1.Manifest + result2 error + }{result1, result2} +} + +func (fake *FakeImage) ManifestReturnsOnCall(i int, result1 *v1.Manifest, result2 error) { + fake.manifestMutex.Lock() + defer fake.manifestMutex.Unlock() + fake.ManifestStub = nil + if fake.manifestReturnsOnCall == nil { + fake.manifestReturnsOnCall = make(map[int]struct { + result1 *v1.Manifest + result2 error + }) + } + fake.manifestReturnsOnCall[i] = struct { + result1 *v1.Manifest + result2 error + }{result1, result2} +} + +func (fake *FakeImage) MediaType() (types.MediaType, error) { + fake.mediaTypeMutex.Lock() + ret, specificReturn := fake.mediaTypeReturnsOnCall[len(fake.mediaTypeArgsForCall)] + fake.mediaTypeArgsForCall = append(fake.mediaTypeArgsForCall, struct { + }{}) + stub := fake.MediaTypeStub + fakeReturns := fake.mediaTypeReturns + fake.recordInvocation("MediaType", []interface{}{}) + fake.mediaTypeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) MediaTypeCallCount() int { + fake.mediaTypeMutex.RLock() + defer fake.mediaTypeMutex.RUnlock() + return len(fake.mediaTypeArgsForCall) +} + +func (fake *FakeImage) MediaTypeCalls(stub func() (types.MediaType, error)) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = stub +} + +func (fake *FakeImage) MediaTypeReturns(result1 types.MediaType, result2 error) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = nil + fake.mediaTypeReturns = struct { + result1 types.MediaType + result2 error + }{result1, result2} +} + +func (fake *FakeImage) MediaTypeReturnsOnCall(i int, result1 types.MediaType, result2 error) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = nil + if fake.mediaTypeReturnsOnCall == nil { + fake.mediaTypeReturnsOnCall = make(map[int]struct { + result1 types.MediaType + result2 error + }) + } + fake.mediaTypeReturnsOnCall[i] = struct { + result1 types.MediaType + result2 error + }{result1, result2} +} + +func (fake *FakeImage) RawConfigFile() ([]byte, error) { + fake.rawConfigFileMutex.Lock() + ret, specificReturn := fake.rawConfigFileReturnsOnCall[len(fake.rawConfigFileArgsForCall)] + fake.rawConfigFileArgsForCall = append(fake.rawConfigFileArgsForCall, struct { + }{}) + stub := fake.RawConfigFileStub + fakeReturns := fake.rawConfigFileReturns + fake.recordInvocation("RawConfigFile", []interface{}{}) + fake.rawConfigFileMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) RawConfigFileCallCount() int { + fake.rawConfigFileMutex.RLock() + defer fake.rawConfigFileMutex.RUnlock() + return len(fake.rawConfigFileArgsForCall) +} + +func (fake *FakeImage) RawConfigFileCalls(stub func() ([]byte, error)) { + fake.rawConfigFileMutex.Lock() + defer fake.rawConfigFileMutex.Unlock() + fake.RawConfigFileStub = stub +} + +func (fake *FakeImage) RawConfigFileReturns(result1 []byte, result2 error) { + fake.rawConfigFileMutex.Lock() + defer fake.rawConfigFileMutex.Unlock() + fake.RawConfigFileStub = nil + fake.rawConfigFileReturns = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImage) RawConfigFileReturnsOnCall(i int, result1 []byte, result2 error) { + fake.rawConfigFileMutex.Lock() + defer fake.rawConfigFileMutex.Unlock() + fake.RawConfigFileStub = nil + if fake.rawConfigFileReturnsOnCall == nil { + fake.rawConfigFileReturnsOnCall = make(map[int]struct { + result1 []byte + result2 error + }) + } + fake.rawConfigFileReturnsOnCall[i] = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImage) RawManifest() ([]byte, error) { + fake.rawManifestMutex.Lock() + ret, specificReturn := fake.rawManifestReturnsOnCall[len(fake.rawManifestArgsForCall)] + fake.rawManifestArgsForCall = append(fake.rawManifestArgsForCall, struct { + }{}) + stub := fake.RawManifestStub + fakeReturns := fake.rawManifestReturns + fake.recordInvocation("RawManifest", []interface{}{}) + fake.rawManifestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) RawManifestCallCount() int { + fake.rawManifestMutex.RLock() + defer fake.rawManifestMutex.RUnlock() + return len(fake.rawManifestArgsForCall) +} + +func (fake *FakeImage) RawManifestCalls(stub func() ([]byte, error)) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = stub +} + +func (fake *FakeImage) RawManifestReturns(result1 []byte, result2 error) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = nil + fake.rawManifestReturns = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImage) RawManifestReturnsOnCall(i int, result1 []byte, result2 error) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = nil + if fake.rawManifestReturnsOnCall == nil { + fake.rawManifestReturnsOnCall = make(map[int]struct { + result1 []byte + result2 error + }) + } + fake.rawManifestReturnsOnCall[i] = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImage) Size() (int64, error) { + fake.sizeMutex.Lock() + ret, specificReturn := fake.sizeReturnsOnCall[len(fake.sizeArgsForCall)] + fake.sizeArgsForCall = append(fake.sizeArgsForCall, struct { + }{}) + stub := fake.SizeStub + fakeReturns := fake.sizeReturns + fake.recordInvocation("Size", []interface{}{}) + fake.sizeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImage) SizeCallCount() int { + fake.sizeMutex.RLock() + defer fake.sizeMutex.RUnlock() + return len(fake.sizeArgsForCall) +} + +func (fake *FakeImage) SizeCalls(stub func() (int64, error)) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = stub +} + +func (fake *FakeImage) SizeReturns(result1 int64, result2 error) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = nil + fake.sizeReturns = struct { + result1 int64 + result2 error + }{result1, result2} +} + +func (fake *FakeImage) SizeReturnsOnCall(i int, result1 int64, result2 error) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = nil + if fake.sizeReturnsOnCall == nil { + fake.sizeReturnsOnCall = make(map[int]struct { + result1 int64 + result2 error + }) + } + fake.sizeReturnsOnCall[i] = struct { + result1 int64 + result2 error + }{result1, result2} +} + +func (fake *FakeImage) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.configFileMutex.RLock() + defer fake.configFileMutex.RUnlock() + fake.configNameMutex.RLock() + defer fake.configNameMutex.RUnlock() + fake.digestMutex.RLock() + defer fake.digestMutex.RUnlock() + fake.layerByDiffIDMutex.RLock() + defer fake.layerByDiffIDMutex.RUnlock() + fake.layerByDigestMutex.RLock() + defer fake.layerByDigestMutex.RUnlock() + fake.layersMutex.RLock() + defer fake.layersMutex.RUnlock() + fake.manifestMutex.RLock() + defer fake.manifestMutex.RUnlock() + fake.mediaTypeMutex.RLock() + defer fake.mediaTypeMutex.RUnlock() + fake.rawConfigFileMutex.RLock() + defer fake.rawConfigFileMutex.RUnlock() + fake.rawManifestMutex.RLock() + defer fake.rawManifestMutex.RUnlock() + fake.sizeMutex.RLock() + defer fake.sizeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeImage) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ v1.Image = new(FakeImage) diff --git a/pkg/v1/fake/index.go b/pkg/v1/fake/index.go new file mode 100644 index 0000000..8c66a98 --- /dev/null +++ b/pkg/v1/fake/index.go @@ -0,0 +1,546 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package fake + +import ( + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type FakeImageIndex struct { + DigestStub func() (v1.Hash, error) + digestMutex sync.RWMutex + digestArgsForCall []struct { + } + digestReturns struct { + result1 v1.Hash + result2 error + } + digestReturnsOnCall map[int]struct { + result1 v1.Hash + result2 error + } + ImageStub func(v1.Hash) (v1.Image, error) + imageMutex sync.RWMutex + imageArgsForCall []struct { + arg1 v1.Hash + } + imageReturns struct { + result1 v1.Image + result2 error + } + imageReturnsOnCall map[int]struct { + result1 v1.Image + result2 error + } + ImageIndexStub func(v1.Hash) (v1.ImageIndex, error) + imageIndexMutex sync.RWMutex + imageIndexArgsForCall []struct { + arg1 v1.Hash + } + imageIndexReturns struct { + result1 v1.ImageIndex + result2 error + } + imageIndexReturnsOnCall map[int]struct { + result1 v1.ImageIndex + result2 error + } + IndexManifestStub func() (*v1.IndexManifest, error) + indexManifestMutex sync.RWMutex + indexManifestArgsForCall []struct { + } + indexManifestReturns struct { + result1 *v1.IndexManifest + result2 error + } + indexManifestReturnsOnCall map[int]struct { + result1 *v1.IndexManifest + result2 error + } + MediaTypeStub func() (types.MediaType, error) + mediaTypeMutex sync.RWMutex + mediaTypeArgsForCall []struct { + } + mediaTypeReturns struct { + result1 types.MediaType + result2 error + } + mediaTypeReturnsOnCall map[int]struct { + result1 types.MediaType + result2 error + } + RawManifestStub func() ([]byte, error) + rawManifestMutex sync.RWMutex + rawManifestArgsForCall []struct { + } + rawManifestReturns struct { + result1 []byte + result2 error + } + rawManifestReturnsOnCall map[int]struct { + result1 []byte + result2 error + } + SizeStub func() (int64, error) + sizeMutex sync.RWMutex + sizeArgsForCall []struct { + } + sizeReturns struct { + result1 int64 + result2 error + } + sizeReturnsOnCall map[int]struct { + result1 int64 + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeImageIndex) Digest() (v1.Hash, error) { + fake.digestMutex.Lock() + ret, specificReturn := fake.digestReturnsOnCall[len(fake.digestArgsForCall)] + fake.digestArgsForCall = append(fake.digestArgsForCall, struct { + }{}) + stub := fake.DigestStub + fakeReturns := fake.digestReturns + fake.recordInvocation("Digest", []interface{}{}) + fake.digestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) DigestCallCount() int { + fake.digestMutex.RLock() + defer fake.digestMutex.RUnlock() + return len(fake.digestArgsForCall) +} + +func (fake *FakeImageIndex) DigestCalls(stub func() (v1.Hash, error)) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = stub +} + +func (fake *FakeImageIndex) DigestReturns(result1 v1.Hash, result2 error) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = nil + fake.digestReturns = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) DigestReturnsOnCall(i int, result1 v1.Hash, result2 error) { + fake.digestMutex.Lock() + defer fake.digestMutex.Unlock() + fake.DigestStub = nil + if fake.digestReturnsOnCall == nil { + fake.digestReturnsOnCall = make(map[int]struct { + result1 v1.Hash + result2 error + }) + } + fake.digestReturnsOnCall[i] = struct { + result1 v1.Hash + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) Image(arg1 v1.Hash) (v1.Image, error) { + fake.imageMutex.Lock() + ret, specificReturn := fake.imageReturnsOnCall[len(fake.imageArgsForCall)] + fake.imageArgsForCall = append(fake.imageArgsForCall, struct { + arg1 v1.Hash + }{arg1}) + stub := fake.ImageStub + fakeReturns := fake.imageReturns + fake.recordInvocation("Image", []interface{}{arg1}) + fake.imageMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) ImageCallCount() int { + fake.imageMutex.RLock() + defer fake.imageMutex.RUnlock() + return len(fake.imageArgsForCall) +} + +func (fake *FakeImageIndex) ImageCalls(stub func(v1.Hash) (v1.Image, error)) { + fake.imageMutex.Lock() + defer fake.imageMutex.Unlock() + fake.ImageStub = stub +} + +func (fake *FakeImageIndex) ImageArgsForCall(i int) v1.Hash { + fake.imageMutex.RLock() + defer fake.imageMutex.RUnlock() + argsForCall := fake.imageArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeImageIndex) ImageReturns(result1 v1.Image, result2 error) { + fake.imageMutex.Lock() + defer fake.imageMutex.Unlock() + fake.ImageStub = nil + fake.imageReturns = struct { + result1 v1.Image + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) ImageReturnsOnCall(i int, result1 v1.Image, result2 error) { + fake.imageMutex.Lock() + defer fake.imageMutex.Unlock() + fake.ImageStub = nil + if fake.imageReturnsOnCall == nil { + fake.imageReturnsOnCall = make(map[int]struct { + result1 v1.Image + result2 error + }) + } + fake.imageReturnsOnCall[i] = struct { + result1 v1.Image + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) ImageIndex(arg1 v1.Hash) (v1.ImageIndex, error) { + fake.imageIndexMutex.Lock() + ret, specificReturn := fake.imageIndexReturnsOnCall[len(fake.imageIndexArgsForCall)] + fake.imageIndexArgsForCall = append(fake.imageIndexArgsForCall, struct { + arg1 v1.Hash + }{arg1}) + stub := fake.ImageIndexStub + fakeReturns := fake.imageIndexReturns + fake.recordInvocation("ImageIndex", []interface{}{arg1}) + fake.imageIndexMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) ImageIndexCallCount() int { + fake.imageIndexMutex.RLock() + defer fake.imageIndexMutex.RUnlock() + return len(fake.imageIndexArgsForCall) +} + +func (fake *FakeImageIndex) ImageIndexCalls(stub func(v1.Hash) (v1.ImageIndex, error)) { + fake.imageIndexMutex.Lock() + defer fake.imageIndexMutex.Unlock() + fake.ImageIndexStub = stub +} + +func (fake *FakeImageIndex) ImageIndexArgsForCall(i int) v1.Hash { + fake.imageIndexMutex.RLock() + defer fake.imageIndexMutex.RUnlock() + argsForCall := fake.imageIndexArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeImageIndex) ImageIndexReturns(result1 v1.ImageIndex, result2 error) { + fake.imageIndexMutex.Lock() + defer fake.imageIndexMutex.Unlock() + fake.ImageIndexStub = nil + fake.imageIndexReturns = struct { + result1 v1.ImageIndex + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) ImageIndexReturnsOnCall(i int, result1 v1.ImageIndex, result2 error) { + fake.imageIndexMutex.Lock() + defer fake.imageIndexMutex.Unlock() + fake.ImageIndexStub = nil + if fake.imageIndexReturnsOnCall == nil { + fake.imageIndexReturnsOnCall = make(map[int]struct { + result1 v1.ImageIndex + result2 error + }) + } + fake.imageIndexReturnsOnCall[i] = struct { + result1 v1.ImageIndex + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) IndexManifest() (*v1.IndexManifest, error) { + fake.indexManifestMutex.Lock() + ret, specificReturn := fake.indexManifestReturnsOnCall[len(fake.indexManifestArgsForCall)] + fake.indexManifestArgsForCall = append(fake.indexManifestArgsForCall, struct { + }{}) + stub := fake.IndexManifestStub + fakeReturns := fake.indexManifestReturns + fake.recordInvocation("IndexManifest", []interface{}{}) + fake.indexManifestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) IndexManifestCallCount() int { + fake.indexManifestMutex.RLock() + defer fake.indexManifestMutex.RUnlock() + return len(fake.indexManifestArgsForCall) +} + +func (fake *FakeImageIndex) IndexManifestCalls(stub func() (*v1.IndexManifest, error)) { + fake.indexManifestMutex.Lock() + defer fake.indexManifestMutex.Unlock() + fake.IndexManifestStub = stub +} + +func (fake *FakeImageIndex) IndexManifestReturns(result1 *v1.IndexManifest, result2 error) { + fake.indexManifestMutex.Lock() + defer fake.indexManifestMutex.Unlock() + fake.IndexManifestStub = nil + fake.indexManifestReturns = struct { + result1 *v1.IndexManifest + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) IndexManifestReturnsOnCall(i int, result1 *v1.IndexManifest, result2 error) { + fake.indexManifestMutex.Lock() + defer fake.indexManifestMutex.Unlock() + fake.IndexManifestStub = nil + if fake.indexManifestReturnsOnCall == nil { + fake.indexManifestReturnsOnCall = make(map[int]struct { + result1 *v1.IndexManifest + result2 error + }) + } + fake.indexManifestReturnsOnCall[i] = struct { + result1 *v1.IndexManifest + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) MediaType() (types.MediaType, error) { + fake.mediaTypeMutex.Lock() + ret, specificReturn := fake.mediaTypeReturnsOnCall[len(fake.mediaTypeArgsForCall)] + fake.mediaTypeArgsForCall = append(fake.mediaTypeArgsForCall, struct { + }{}) + stub := fake.MediaTypeStub + fakeReturns := fake.mediaTypeReturns + fake.recordInvocation("MediaType", []interface{}{}) + fake.mediaTypeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) MediaTypeCallCount() int { + fake.mediaTypeMutex.RLock() + defer fake.mediaTypeMutex.RUnlock() + return len(fake.mediaTypeArgsForCall) +} + +func (fake *FakeImageIndex) MediaTypeCalls(stub func() (types.MediaType, error)) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = stub +} + +func (fake *FakeImageIndex) MediaTypeReturns(result1 types.MediaType, result2 error) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = nil + fake.mediaTypeReturns = struct { + result1 types.MediaType + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) MediaTypeReturnsOnCall(i int, result1 types.MediaType, result2 error) { + fake.mediaTypeMutex.Lock() + defer fake.mediaTypeMutex.Unlock() + fake.MediaTypeStub = nil + if fake.mediaTypeReturnsOnCall == nil { + fake.mediaTypeReturnsOnCall = make(map[int]struct { + result1 types.MediaType + result2 error + }) + } + fake.mediaTypeReturnsOnCall[i] = struct { + result1 types.MediaType + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) RawManifest() ([]byte, error) { + fake.rawManifestMutex.Lock() + ret, specificReturn := fake.rawManifestReturnsOnCall[len(fake.rawManifestArgsForCall)] + fake.rawManifestArgsForCall = append(fake.rawManifestArgsForCall, struct { + }{}) + stub := fake.RawManifestStub + fakeReturns := fake.rawManifestReturns + fake.recordInvocation("RawManifest", []interface{}{}) + fake.rawManifestMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) RawManifestCallCount() int { + fake.rawManifestMutex.RLock() + defer fake.rawManifestMutex.RUnlock() + return len(fake.rawManifestArgsForCall) +} + +func (fake *FakeImageIndex) RawManifestCalls(stub func() ([]byte, error)) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = stub +} + +func (fake *FakeImageIndex) RawManifestReturns(result1 []byte, result2 error) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = nil + fake.rawManifestReturns = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) RawManifestReturnsOnCall(i int, result1 []byte, result2 error) { + fake.rawManifestMutex.Lock() + defer fake.rawManifestMutex.Unlock() + fake.RawManifestStub = nil + if fake.rawManifestReturnsOnCall == nil { + fake.rawManifestReturnsOnCall = make(map[int]struct { + result1 []byte + result2 error + }) + } + fake.rawManifestReturnsOnCall[i] = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) Size() (int64, error) { + fake.sizeMutex.Lock() + ret, specificReturn := fake.sizeReturnsOnCall[len(fake.sizeArgsForCall)] + fake.sizeArgsForCall = append(fake.sizeArgsForCall, struct { + }{}) + stub := fake.SizeStub + fakeReturns := fake.sizeReturns + fake.recordInvocation("Size", []interface{}{}) + fake.sizeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeImageIndex) SizeCallCount() int { + fake.sizeMutex.RLock() + defer fake.sizeMutex.RUnlock() + return len(fake.sizeArgsForCall) +} + +func (fake *FakeImageIndex) SizeCalls(stub func() (int64, error)) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = stub +} + +func (fake *FakeImageIndex) SizeReturns(result1 int64, result2 error) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = nil + fake.sizeReturns = struct { + result1 int64 + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) SizeReturnsOnCall(i int, result1 int64, result2 error) { + fake.sizeMutex.Lock() + defer fake.sizeMutex.Unlock() + fake.SizeStub = nil + if fake.sizeReturnsOnCall == nil { + fake.sizeReturnsOnCall = make(map[int]struct { + result1 int64 + result2 error + }) + } + fake.sizeReturnsOnCall[i] = struct { + result1 int64 + result2 error + }{result1, result2} +} + +func (fake *FakeImageIndex) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.digestMutex.RLock() + defer fake.digestMutex.RUnlock() + fake.imageMutex.RLock() + defer fake.imageMutex.RUnlock() + fake.imageIndexMutex.RLock() + defer fake.imageIndexMutex.RUnlock() + fake.indexManifestMutex.RLock() + defer fake.indexManifestMutex.RUnlock() + fake.mediaTypeMutex.RLock() + defer fake.mediaTypeMutex.RUnlock() + fake.rawManifestMutex.RLock() + defer fake.rawManifestMutex.RUnlock() + fake.sizeMutex.RLock() + defer fake.sizeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeImageIndex) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ v1.ImageIndex = new(FakeImageIndex) diff --git a/pkg/v1/google/README.md b/pkg/v1/google/README.md new file mode 100644 index 0000000..7cd8971 --- /dev/null +++ b/pkg/v1/google/README.md @@ -0,0 +1,7 @@ +# `google` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/google?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/google) + +The `google` package provides: +* Some google-specific authentication methods. +* Some [GCR](gcr.io)-specific listing methods. diff --git a/pkg/v1/google/auth.go b/pkg/v1/google/auth.go new file mode 100644 index 0000000..11ae397 --- /dev/null +++ b/pkg/v1/google/auth.go @@ -0,0 +1,179 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "golang.org/x/oauth2" + googauth "golang.org/x/oauth2/google" +) + +const cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" + +// GetGcloudCmd is exposed so we can test this. +var GetGcloudCmd = func() *exec.Cmd { + // This is odd, but basically what docker-credential-gcr does. + // + // config-helper is undocumented, but it's purportedly the only supported way + // of accessing tokens (`gcloud auth print-access-token` is discouraged). + // + // --force-auth-refresh means we are getting a token that is valid for about + // an hour (we reuse it until it's expired). + return exec.Command("gcloud", "config", "config-helper", "--force-auth-refresh", "--format=json(credential)") +} + +// NewEnvAuthenticator returns an authn.Authenticator that generates access +// tokens from the environment we're running in. +// +// See: https://godoc.org/golang.org/x/oauth2/google#FindDefaultCredentials +func NewEnvAuthenticator() (authn.Authenticator, error) { + ts, err := googauth.DefaultTokenSource(context.Background(), cloudPlatformScope) + if err != nil { + return nil, err + } + + token, err := ts.Token() + if err != nil { + return nil, err + } + + return &tokenSourceAuth{oauth2.ReuseTokenSource(token, ts)}, nil +} + +// NewGcloudAuthenticator returns an oauth2.TokenSource that generates access +// tokens by shelling out to the gcloud sdk. +func NewGcloudAuthenticator() (authn.Authenticator, error) { + if _, err := exec.LookPath("gcloud"); err != nil { + // gcloud is not available, fall back to anonymous + logs.Warn.Println("gcloud binary not found") + return authn.Anonymous, nil + } + + ts := gcloudSource{GetGcloudCmd} + + // Attempt to fetch a token to ensure gcloud is installed and we can run it. + token, err := ts.Token() + if err != nil { + return nil, err + } + + return &tokenSourceAuth{oauth2.ReuseTokenSource(token, ts)}, nil +} + +// NewJSONKeyAuthenticator returns a Basic authenticator which uses Service Account +// as a way of authenticating with Google Container Registry. +// More information: https://cloud.google.com/container-registry/docs/advanced-authentication#json_key_file +func NewJSONKeyAuthenticator(serviceAccountJSON string) authn.Authenticator { + return &authn.Basic{ + Username: "_json_key", + Password: serviceAccountJSON, + } +} + +// NewTokenAuthenticator returns an oauth2.TokenSource that generates access +// tokens by using the Google SDK to produce JWT tokens from a Service Account. +// More information: https://godoc.org/golang.org/x/oauth2/google#JWTAccessTokenSourceFromJSON +func NewTokenAuthenticator(serviceAccountJSON string, scope string) (authn.Authenticator, error) { + ts, err := googauth.JWTAccessTokenSourceFromJSON([]byte(serviceAccountJSON), scope) + if err != nil { + return nil, err + } + + return &tokenSourceAuth{oauth2.ReuseTokenSource(nil, ts)}, nil +} + +// NewTokenSourceAuthenticator converts an oauth2.TokenSource into an authn.Authenticator. +func NewTokenSourceAuthenticator(ts oauth2.TokenSource) authn.Authenticator { + return &tokenSourceAuth{ts} +} + +// tokenSourceAuth turns an oauth2.TokenSource into an authn.Authenticator. +type tokenSourceAuth struct { + oauth2.TokenSource +} + +// Authorization implements authn.Authenticator. +func (tsa *tokenSourceAuth) Authorization() (*authn.AuthConfig, error) { + token, err := tsa.Token() + if err != nil { + return nil, err + } + + return &authn.AuthConfig{ + Username: "_token", + Password: token.AccessToken, + }, nil +} + +// gcloudOutput represents the output of the gcloud command we invoke. +// +// `gcloud config config-helper --format=json(credential)` looks something like: +// +// { +// "credential": { +// "access_token": "supersecretaccesstoken", +// "token_expiry": "2018-12-02T04:08:13Z" +// } +// } +type gcloudOutput struct { + Credential struct { + AccessToken string `json:"access_token"` + TokenExpiry string `json:"token_expiry"` + } `json:"credential"` +} + +type gcloudSource struct { + // This is passed in so that we mock out gcloud and test Token. + exec func() *exec.Cmd +} + +// Token implements oauath2.TokenSource. +func (gs gcloudSource) Token() (*oauth2.Token, error) { + cmd := gs.exec() + var out bytes.Buffer + cmd.Stdout = &out + + // Don't attempt to interpret stderr, just pass it through. + cmd.Stderr = logs.Warn.Writer() + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("error executing `gcloud config config-helper`: %w", err) + } + + creds := gcloudOutput{} + if err := json.Unmarshal(out.Bytes(), &creds); err != nil { + return nil, fmt.Errorf("failed to parse `gcloud config config-helper` output: %w", err) + } + + expiry, err := time.Parse(time.RFC3339, creds.Credential.TokenExpiry) + if err != nil { + return nil, fmt.Errorf("failed to parse gcloud token expiry: %w", err) + } + + token := oauth2.Token{ + AccessToken: creds.Credential.AccessToken, + Expiry: expiry, + } + + return &token, nil +} diff --git a/pkg/v1/google/auth_test.go b/pkg/v1/google/auth_test.go new file mode 100644 index 0000000..d2974ff --- /dev/null +++ b/pkg/v1/google/auth_test.go @@ -0,0 +1,270 @@ +//go:build !arm64 +// +build !arm64 + +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "golang.org/x/oauth2" +) + +const ( + // Fails to parse as JSON at all. + badoutput = "" + + // Fails to parse token_expiry format. + badexpiry = ` +{ + "credential": { + "access_token": "mytoken", + "token_expiry": "most-definitely-not-a-date" + } +}` + + // Expires in 6,000 years. Hopefully nobody is using software then. + success = ` +{ + "credential": { + "access_token": "mytoken", + "token_expiry": "8018-12-02T04:08:13Z" + } +}` +) + +// We'll invoke ourselves with a special environment variable in order to mock +// out the gcloud dependency of gcloudSource. The exec package does this, too. +// +// See: https://www.joeshaw.org/testing-with-os-exec-and-testmain/ +// +// TODO(#908): This doesn't work on arm64 or darwin for some reason. +func TestMain(m *testing.M) { + switch os.Getenv("GO_TEST_MODE") { + case "": + // Normal test mode + os.Exit(m.Run()) + + case "error": + // Makes cmd.Run() return an error. + os.Exit(2) + + case "badoutput": + // Makes the gcloudOutput Unmarshaler fail. + fmt.Println(badoutput) + + case "badexpiry": + // Makes the token_expiry time parser fail. + fmt.Println(badexpiry) + + case "success": + // Returns a seemingly valid token. + fmt.Println(success) + } +} + +func newGcloudCmdMock(env string) func() *exec.Cmd { + return func() *exec.Cmd { + cmd := exec.Command(os.Args[0]) + cmd.Env = []string{fmt.Sprintf("GO_TEST_MODE=%s", env)} + return cmd + } +} + +func TestGcloudErrors(t *testing.T) { + cases := []struct { + env string + + // Just look for the prefix because we can't control other packages' errors. + wantPrefix string + }{{ + env: "error", + wantPrefix: "error executing `gcloud config config-helper`:", + }, { + env: "badoutput", + wantPrefix: "failed to parse `gcloud config config-helper` output:", + }, { + env: "badexpiry", + wantPrefix: "failed to parse gcloud token expiry:", + }} + + for _, tc := range cases { + t.Run(tc.env, func(t *testing.T) { + GetGcloudCmd = newGcloudCmdMock(tc.env) + + if _, err := NewGcloudAuthenticator(); err == nil { + t.Errorf("wanted error, got nil") + } else if got := err.Error(); !strings.HasPrefix(got, tc.wantPrefix) { + t.Errorf("wanted error prefix %q, got %q", tc.wantPrefix, got) + } + }) + } +} + +func TestGcloudSuccess(t *testing.T) { + // Stupid coverage to make sure it doesn't panic. + var b bytes.Buffer + logs.Debug.SetOutput(&b) + + GetGcloudCmd = newGcloudCmdMock("success") + + auth, err := NewGcloudAuthenticator() + if err != nil { + t.Fatalf("NewGcloudAuthenticator got error %v", err) + } + + token, err := auth.Authorization() + if err != nil { + t.Fatalf("Authorization got error %v", err) + } + + if got, want := token.Password, "mytoken"; got != want { + t.Errorf("wanted token %q, got %q", want, got) + } +} + +// +// Keychain tests are in here so we can reuse the fake gcloud stuff. +// + +func mustRegistry(r string) name.Registry { + reg, err := name.NewRegistry(r, name.StrictValidation) + if err != nil { + panic(err) + } + return reg +} + +func TestKeychainDockerHub(t *testing.T) { + if auth, err := Keychain.Resolve(mustRegistry("index.docker.io")); err != nil { + t.Errorf("expected success, got: %v", err) + } else if auth != authn.Anonymous { + t.Errorf("expected anonymous, got: %v", auth) + } +} + +func TestKeychainGCRandAR(t *testing.T) { + cases := []struct { + host string + expectAuth bool + }{ + // GCR hosts + {"gcr.io", true}, + {"us.gcr.io", true}, + {"eu.gcr.io", true}, + {"asia.gcr.io", true}, + {"staging-k8s.gcr.io", true}, + {"global.gcr.io", true}, + {"notgcr.io", false}, + {"fake-gcr.io", false}, + {"alsonot.gcr.iot", false}, + // AR hosts + {"us-docker.pkg.dev", true}, + {"asia-docker.pkg.dev", true}, + {"europe-docker.pkg.dev", true}, + {"us-central1-docker.pkg.dev", true}, + {"us-docker-pkg.dev", false}, + {"someotherpkg.dev", false}, + {"looks-like-pkg.dev", false}, + {"closeto.pkg.devops", false}, + } + + // Env should fail. + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/dev/null"); err != nil { + t.Fatalf("unexpected err os.Setenv: %v", err) + } + + for i, tc := range cases { + t.Run(fmt.Sprintf("cases[%d]", i), func(t *testing.T) { + // Reset the keychain to ensure we don't cache earlier results. + Keychain = &googleKeychain{} + + // Gcloud should succeed. + GetGcloudCmd = newGcloudCmdMock("success") + + if auth, err := Keychain.Resolve(mustRegistry(tc.host)); err != nil { + t.Errorf("expected success for %v, got: %v", tc.host, err) + } else if tc.expectAuth && auth == authn.Anonymous { + t.Errorf("expected not anonymous auth for %v, got: %v", tc, auth) + } else if !tc.expectAuth && auth != authn.Anonymous { + t.Errorf("expected anonymous auth for %v, got: %v", tc, auth) + } + + // Make gcloud fail to test that caching works. + GetGcloudCmd = newGcloudCmdMock("badoutput") + + if auth, err := Keychain.Resolve(mustRegistry(tc.host)); err != nil { + t.Errorf("expected success for %v, got: %v", tc.host, err) + } else if tc.expectAuth && auth == authn.Anonymous { + t.Errorf("expected not anonymous auth for %v, got: %v", tc, auth) + } else if !tc.expectAuth && auth != authn.Anonymous { + t.Errorf("expected anonymous auth for %v, got: %v", tc, auth) + } + }) + } +} + +func TestKeychainError(t *testing.T) { + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/dev/null"); err != nil { + t.Fatalf("unexpected err os.Setenv: %v", err) + } + + GetGcloudCmd = newGcloudCmdMock("badoutput") + + // Reset the keychain to ensure we don't cache earlier results. + Keychain = &googleKeychain{} + if auth, err := Keychain.Resolve(mustRegistry("gcr.io")); err != nil { + t.Fatalf("got error: %v", err) + } else if auth != authn.Anonymous { + t.Fatalf("wanted Anonymous, got %v", auth) + } +} + +type badSource struct{} + +func (bs badSource) Token() (*oauth2.Token, error) { + return nil, fmt.Errorf("oops") +} + +// This test is silly, but coverage. +func TestTokenSourceAuthError(t *testing.T) { + auth := tokenSourceAuth{badSource{}} + + _, err := auth.Authorization() + if err == nil { + t.Errorf("expected err, got nil") + } +} + +func TestNewEnvAuthenticatorFailure(t *testing.T) { + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/dev/null"); err != nil { + t.Fatalf("unexpected err os.Setenv: %v", err) + } + + // Expect error. + _, err := NewEnvAuthenticator() + if err == nil { + t.Errorf("expected err, got nil") + } +} diff --git a/pkg/v1/google/doc.go b/pkg/v1/google/doc.go new file mode 100644 index 0000000..b6a67df --- /dev/null +++ b/pkg/v1/google/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package google provides facilities for listing images in gcr.io. +package google diff --git a/pkg/v1/google/keychain.go b/pkg/v1/google/keychain.go new file mode 100644 index 0000000..6dc7a50 --- /dev/null +++ b/pkg/v1/google/keychain.go @@ -0,0 +1,92 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "strings" + "sync" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" +) + +// Keychain exports an instance of the google Keychain. +var Keychain authn.Keychain = &googleKeychain{} + +type googleKeychain struct { + once sync.Once + auth authn.Authenticator +} + +// Resolve implements authn.Keychain a la docker-credential-gcr. +// +// This behaves similarly to the GCR credential helper, but reuses tokens until +// they expire. +// +// We can't easily add this behavior to our credential helper implementation +// of authn.Authenticator because the credential helper protocol doesn't include +// expiration information, see here: +// https://godoc.org/github.com/docker/docker-credential-helpers/credentials#Credentials +// +// In addition to being a performance optimization, the reuse of these access +// tokens works around a bug in gcloud. It appears that attempting to invoke +// `gcloud config config-helper` multiple times too quickly will fail: +// https://github.com/GoogleCloudPlatform/docker-credential-gcr/issues/54 +// +// We could upstream this behavior into docker-credential-gcr by parsing +// gcloud's output and persisting its tokens across invocations, but then +// we have to deal with invalidating caches across multiple runs (no fun). +// +// In general, we don't worry about that here because we expect to use the same +// gcloud configuration in the scope of this one process. +func (gk *googleKeychain) Resolve(target authn.Resource) (authn.Authenticator, error) { + // Only authenticate GCR and AR so it works with authn.NewMultiKeychain to fallback. + host := target.RegistryStr() + if host != "gcr.io" && + !strings.HasSuffix(host, ".gcr.io") && + !strings.HasSuffix(host, ".pkg.dev") && + !strings.HasSuffix(host, ".google.com") { + return authn.Anonymous, nil + } + + gk.once.Do(func() { + gk.auth = resolve() + }) + + return gk.auth, nil +} + +func resolve() authn.Authenticator { + auth, envErr := NewEnvAuthenticator() + if envErr == nil && auth != authn.Anonymous { + logs.Debug.Println("google.Keychain: using Application Default Credentials") + return auth + } + + auth, gErr := NewGcloudAuthenticator() + if gErr == nil && auth != authn.Anonymous { + logs.Debug.Println("google.Keychain: using gcloud fallback") + return auth + } + + logs.Debug.Println("Failed to get any Google credentials, falling back to Anonymous") + if envErr != nil { + logs.Debug.Printf("Google env error: %v", envErr) + } + if gErr != nil { + logs.Debug.Printf("gcloud error: %v", gErr) + } + return authn.Anonymous +} diff --git a/pkg/v1/google/list.go b/pkg/v1/google/list.go new file mode 100644 index 0000000..a70bb27 --- /dev/null +++ b/pkg/v1/google/list.go @@ -0,0 +1,331 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// Option is a functional option for List and Walk. +// TODO: Can we somehow reuse the remote options here? +type Option func(*lister) error + +type lister struct { + auth authn.Authenticator + transport http.RoundTripper + repo name.Repository + client *http.Client + ctx context.Context + userAgent string +} + +func newLister(repo name.Repository, options ...Option) (*lister, error) { + l := &lister{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + repo: repo, + ctx: context.Background(), + } + + for _, option := range options { + if err := option(l); err != nil { + return nil, err + } + } + + // transport.Wrapper is a signal that consumers are opt-ing into providing their own transport without any additional wrapping. + // This is to allow consumers full control over the transports logic, such as providing retry logic. + if _, ok := l.transport.(*transport.Wrapper); !ok { + // Wrap the transport in something that logs requests and responses. + // It's expensive to generate the dumps, so skip it if we're writing + // to nothing. + if logs.Enabled(logs.Debug) { + l.transport = transport.NewLogger(l.transport) + } + + // Wrap the transport in something that can retry network flakes. + l.transport = transport.NewRetry(l.transport) + + // Wrap this last to prevent transport.New from double-wrapping. + if l.userAgent != "" { + l.transport = transport.NewUserAgent(l.transport, l.userAgent) + } + } + + scopes := []string{repo.Scope(transport.PullScope)} + tr, err := transport.NewWithContext(l.ctx, repo.Registry, l.auth, l.transport, scopes) + if err != nil { + return nil, err + } + + l.client = &http.Client{Transport: tr} + + return l, nil +} + +func (l *lister) list(repo name.Repository) (*Tags, error) { + uri := &url.URL{ + Scheme: repo.Registry.Scheme(), + Host: repo.Registry.RegistryStr(), + Path: fmt.Sprintf("/v2/%s/tags/list", repo.RepositoryStr()), + // ECR returns an error if n > 1000: + // https://github.com/google/go-containerregistry/issues/681 + RawQuery: "n=1000", + } + + tags := Tags{} + + // get responses until there is no next page + for { + select { + case <-l.ctx.Done(): + return nil, l.ctx.Err() + default: + } + + req, err := http.NewRequest("GET", uri.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(l.ctx) + + resp, err := l.client.Do(req) + if err != nil { + return nil, err + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, err + } + + parsed := Tags{} + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, err + } + + if err := resp.Body.Close(); err != nil { + return nil, err + } + + if len(parsed.Manifests) != 0 || len(parsed.Children) != 0 { + // We're dealing with GCR, just return directly. + return &parsed, nil + } + + // This isn't GCR, just append the tags and keep paginating. + tags.Tags = append(tags.Tags, parsed.Tags...) + + uri, err = getNextPageURL(resp) + if err != nil { + return nil, err + } + // no next page + if uri == nil { + break + } + logs.Warn.Printf("saw non-google tag listing response, falling back to pagination") + } + + return &tags, nil +} + +// getNextPageURL checks if there is a Link header in a http.Response which +// contains a link to the next page. If yes it returns the url.URL of the next +// page otherwise it returns nil. +func getNextPageURL(resp *http.Response) (*url.URL, error) { + link := resp.Header.Get("Link") + if link == "" { + return nil, nil + } + + if link[0] != '<' { + return nil, fmt.Errorf("failed to parse link header: missing '<' in: %s", link) + } + + end := strings.Index(link, ">") + if end == -1 { + return nil, fmt.Errorf("failed to parse link header: missing '>' in: %s", link) + } + link = link[1:end] + + linkURL, err := url.Parse(link) + if err != nil { + return nil, err + } + if resp.Request == nil || resp.Request.URL == nil { + return nil, nil + } + linkURL = resp.Request.URL.ResolveReference(linkURL) + return linkURL, nil +} + +type rawManifestInfo struct { + Size string `json:"imageSizeBytes"` + MediaType string `json:"mediaType"` + Created string `json:"timeCreatedMs"` + Uploaded string `json:"timeUploadedMs"` + Tags []string `json:"tag"` +} + +// ManifestInfo is a Manifests entry is the output of List and Walk. +type ManifestInfo struct { + Size uint64 `json:"imageSizeBytes"` + MediaType string `json:"mediaType"` + Created time.Time `json:"timeCreatedMs"` + Uploaded time.Time `json:"timeUploadedMs"` + Tags []string `json:"tag"` +} + +func fromUnixMs(ms int64) time.Time { + sec := ms / 1000 + ns := (ms % 1000) * 1000000 + return time.Unix(sec, ns) +} + +func toUnixMs(t time.Time) string { + return strconv.FormatInt(t.UnixNano()/1000000, 10) +} + +// MarshalJSON implements json.Marshaler +func (m ManifestInfo) MarshalJSON() ([]byte, error) { + return json.Marshal(rawManifestInfo{ + Size: strconv.FormatUint(m.Size, 10), + MediaType: m.MediaType, + Created: toUnixMs(m.Created), + Uploaded: toUnixMs(m.Uploaded), + Tags: m.Tags, + }) +} + +// UnmarshalJSON implements json.Unmarshaler +func (m *ManifestInfo) UnmarshalJSON(data []byte) error { + raw := rawManifestInfo{} + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + if raw.Size != "" { + size, err := strconv.ParseUint(raw.Size, 10, 64) + if err != nil { + return err + } + m.Size = size + } + + if raw.Created != "" { + created, err := strconv.ParseInt(raw.Created, 10, 64) + if err != nil { + return err + } + m.Created = fromUnixMs(created) + } + + if raw.Uploaded != "" { + uploaded, err := strconv.ParseInt(raw.Uploaded, 10, 64) + if err != nil { + return err + } + m.Uploaded = fromUnixMs(uploaded) + } + + m.MediaType = raw.MediaType + m.Tags = raw.Tags + + return nil +} + +// Tags is the result of List and Walk. +type Tags struct { + Children []string `json:"child"` + Manifests map[string]ManifestInfo `json:"manifest"` + Name string `json:"name"` + Tags []string `json:"tags"` +} + +// List calls /tags/list for the given repository. +func List(repo name.Repository, options ...Option) (*Tags, error) { + l, err := newLister(repo, options...) + if err != nil { + return nil, err + } + + return l.list(repo) +} + +// WalkFunc is the type of the function called for each repository visited by +// Walk. This implements a similar API to filepath.Walk. +// +// The repo argument contains the argument to Walk as a prefix; that is, if Walk +// is called with "gcr.io/foo", which is a repository containing the repository +// "bar", the walk function will be called with argument "gcr.io/foo/bar". +// The tags and error arguments are the result of calling List on repo. +// +// TODO: Do we want a SkipDir error, as in filepath.WalkFunc? +type WalkFunc func(repo name.Repository, tags *Tags, err error) error + +func walk(repo name.Repository, tags *Tags, walkFn WalkFunc, options ...Option) error { + if tags == nil { + // This shouldn't happen. + return fmt.Errorf("tags nil for %q", repo) + } + + if err := walkFn(repo, tags, nil); err != nil { + return err + } + + for _, path := range tags.Children { + child, err := name.NewRepository(fmt.Sprintf("%s/%s", repo, path), name.StrictValidation) + if err != nil { + // We don't expect this ever, so don't pass it through to walkFn. + return fmt.Errorf("unexpected path failure: %w", err) + } + + childTags, err := List(child, options...) + if err != nil { + if err := walkFn(child, nil, err); err != nil { + return err + } + } else { + if err := walk(child, childTags, walkFn, options...); err != nil { + return err + } + } + } + + // We made it! + return nil +} + +// Walk recursively descends repositories, calling walkFn. +func Walk(root name.Repository, walkFn WalkFunc, options ...Option) error { + tags, err := List(root, options...) + if err != nil { + return walkFn(root, nil, err) + } + + return walk(root, tags, walkFn, options...) +} diff --git a/pkg/v1/google/list_test.go b/pkg/v1/google/list_test.go new file mode 100644 index 0000000..5718526 --- /dev/null +++ b/pkg/v1/google/list_test.go @@ -0,0 +1,339 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" +) + +func mustParseDuration(t *testing.T, d string) time.Duration { + dur, err := time.ParseDuration(d) + if err != nil { + t.Fatal(err) + } + return dur +} + +func TestRoundtrip(t *testing.T) { + raw := rawManifestInfo{ + Size: "100", + MediaType: "hi", + Created: "12345678", + Uploaded: "23456789", + Tags: []string{"latest"}, + } + + og, err := json.Marshal(raw) + if err != nil { + t.Fatal(err) + } + + parsed := ManifestInfo{} + if err := json.Unmarshal(og, &parsed); err != nil { + t.Fatal(err) + } + + roundtripped, err := json.Marshal(parsed) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(og, roundtripped); diff != "" { + t.Errorf("ManifestInfo can't roundtrip: (-want +got) = %s", diff) + } +} + +func TestList(t *testing.T) { + cases := []struct { + name string + responseBody []byte + wantErr bool + wantTags *Tags + }{{ + name: "success", + responseBody: []byte(`{"tags":["foo","bar"]}`), + wantErr: false, + wantTags: &Tags{Tags: []string{"foo", "bar"}}, + }, { + name: "gcr success", + responseBody: []byte(`{"child":["hello", "world"],"manifest":{"digest1":{"imageSizeBytes":"1","mediaType":"mainstream","timeCreatedms":"1","timeUploadedMs":"2","tag":["foo"]},"digest2":{"imageSizeBytes":"2","mediaType":"indie","timeCreatedMs":"3","timeUploadedMs":"4","tag":["bar","baz"]}},"tags":["foo","bar","baz"]}`), + wantErr: false, + wantTags: &Tags{ + Children: []string{"hello", "world"}, + Manifests: map[string]ManifestInfo{ + "digest1": { + Size: 1, + MediaType: "mainstream", + Created: time.Unix(0, 0).Add(mustParseDuration(t, "1ms")), + Uploaded: time.Unix(0, 0).Add(mustParseDuration(t, "2ms")), + Tags: []string{"foo"}, + }, + "digest2": { + Size: 2, + MediaType: "indie", + Created: time.Unix(0, 0).Add(mustParseDuration(t, "3ms")), + Uploaded: time.Unix(0, 0).Add(mustParseDuration(t, "4ms")), + Tags: []string{"bar", "baz"}, + }, + }, + Tags: []string{"foo", "bar", "baz"}, + }, + }, { + name: "just children", + responseBody: []byte(`{"child":["hello", "world"]}`), + wantErr: false, + wantTags: &Tags{ + Children: []string{"hello", "world"}, + }, + }, { + name: "not json", + responseBody: []byte("notjson"), + wantErr: true, + }} + + repoName := "ubuntu" + // To test WithUserAgent + uaSentinel := "this-is-the-user-agent" + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tagsPath := fmt.Sprintf("/v2/%s/tags/list", repoName) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("User-Agent"), uaSentinel; !strings.Contains(got, want) { + t.Errorf("request did not container useragent, got %q want Contains(%q)", got, want) + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case tagsPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Write(tc.responseBody) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + repo, err := name.NewRepository(fmt.Sprintf("%s/%s", u.Host, repoName), name.WeakValidation) + if err != nil { + t.Fatalf("name.NewRepository(%v) = %v", repoName, err) + } + + tags, err := List(repo, WithAuthFromKeychain(authn.DefaultKeychain), WithTransport(http.DefaultTransport), WithUserAgent(uaSentinel), WithContext(context.Background())) + if (err != nil) != tc.wantErr { + t.Errorf("List() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + + if diff := cmp.Diff(tc.wantTags, tags); diff != "" { + t.Errorf("List() wrong tags (-want +got) = %s", diff) + } + }) + } +} + +type recorder struct { + Tags []*Tags + Errs []error +} + +func (r *recorder) walk(repo name.Repository, tags *Tags, err error) error { + r.Tags = append(r.Tags, tags) + r.Errs = append(r.Errs, err) + + return nil +} + +func TestWalk(t *testing.T) { + // Stupid coverage to make sure it doesn't panic. + var b bytes.Buffer + logs.Debug.SetOutput(&b) + + cases := []struct { + name string + responseBody []byte + wantResult recorder + }{{ + name: "gcr success", + responseBody: []byte(`{"child":["hello", "world"],"manifest":{"digest1":{"imageSizeBytes":"1","mediaType":"mainstream","timeCreatedms":"1","timeUploadedMs":"2","tag":["foo"]},"digest2":{"imageSizeBytes":"2","mediaType":"indie","timeCreatedMs":"3","timeUploadedMs":"4","tag":["bar","baz"]}},"tags":["foo","bar","baz"]}`), + wantResult: recorder{ + Tags: []*Tags{{ + Children: []string{"hello", "world"}, + Manifests: map[string]ManifestInfo{ + "digest1": { + Size: 1, + MediaType: "mainstream", + Created: time.Unix(0, 0).Add(mustParseDuration(t, "1ms")), + Uploaded: time.Unix(0, 0).Add(mustParseDuration(t, "2ms")), + Tags: []string{"foo"}, + }, + "digest2": { + Size: 2, + MediaType: "indie", + Created: time.Unix(0, 0).Add(mustParseDuration(t, "3ms")), + Uploaded: time.Unix(0, 0).Add(mustParseDuration(t, "4ms")), + Tags: []string{"bar", "baz"}, + }, + }, + Tags: []string{"foo", "bar", "baz"}, + }, { + Tags: []string{"hello"}, + }, { + Tags: []string{"world"}, + }}, + Errs: []error{nil, nil, nil}, + }, + }} + + repoName := "ubuntu" + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rootPath := fmt.Sprintf("/v2/%s/tags/list", repoName) + helloPath := fmt.Sprintf("/v2/%s/hello/tags/list", repoName) + worldPath := fmt.Sprintf("/v2/%s/world/tags/list", repoName) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case rootPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Write(tc.responseBody) + case helloPath: + w.Write([]byte(`{"tags":["hello"]}`)) + case worldPath: + w.Write([]byte(`{"tags":["world"]}`)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + repo, err := name.NewRepository(fmt.Sprintf("%s/%s", u.Host, repoName), name.WeakValidation) + if err != nil { + t.Fatalf("name.NewRepository(%v) = %v", repoName, err) + } + + r := recorder{} + if err := Walk(repo, r.walk, WithAuth(authn.Anonymous)); err != nil { + t.Errorf("unexpected err: %v", err) + } + + if diff := cmp.Diff(tc.wantResult, r); diff != "" { + t.Errorf("Walk() wrong tags (-want +got) = %s", diff) + } + }) + } +} + +// Copied shamelessly from remote. +func TestCancelledList(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + repoName := "doesnotmatter" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + repo, err := name.NewRepository(fmt.Sprintf("%s/%s", u.Host, repoName), name.WeakValidation) + if err != nil { + t.Fatalf("name.NewRepository(%v) = %v", repoName, err) + } + + _, err = List(repo, WithContext(ctx)) + if !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Errorf("wanted %q to contain %q", err.Error(), context.Canceled.Error()) + } +} + +func makeResp(hdr string) *http.Response { + return &http.Response{ + Header: http.Header{ + "Link": []string{hdr}, + }, + } +} + +func TestGetNextPageURL(t *testing.T) { + for _, hdr := range []string{ + "", + "<", + "><", + "<>", + fmt.Sprintf("<%c>", 0x7f), // makes url.Parse fail + } { + u, err := getNextPageURL(makeResp(hdr)) + if err == nil && u != nil { + t.Errorf("Expected err, got %+v", u) + } + } + + good := &http.Response{ + Header: http.Header{ + "Link": []string{""}, + }, + Request: &http.Request{ + URL: &url.URL{ + Scheme: "https", + }, + }, + } + u, err := getNextPageURL(good) + if err != nil { + t.Fatal(err) + } + + if u.Scheme != "https" { + t.Errorf("expected scheme to match request, got %s", u.Scheme) + } +} diff --git a/pkg/v1/google/options.go b/pkg/v1/google/options.go new file mode 100644 index 0000000..604808c --- /dev/null +++ b/pkg/v1/google/options.go @@ -0,0 +1,73 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "context" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// WithTransport is a functional option for overriding the default transport +// on a remote image +func WithTransport(t http.RoundTripper) Option { + return func(l *lister) error { + l.transport = t + return nil + } +} + +// WithAuth is a functional option for overriding the default authenticator +// on a remote image +func WithAuth(auth authn.Authenticator) Option { + return func(l *lister) error { + l.auth = auth + return nil + } +} + +// WithAuthFromKeychain is a functional option for overriding the default +// authenticator on a remote image using an authn.Keychain +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(l *lister) error { + auth, err := keys.Resolve(l.repo.Registry) + if err != nil { + return err + } + l.auth = auth + return nil + } +} + +// WithContext is a functional option for overriding the default +// context.Context for HTTP request to list remote images +func WithContext(ctx context.Context) Option { + return func(l *lister) error { + l.ctx = ctx + return nil + } +} + +// WithUserAgent adds the given string to the User-Agent header for any HTTP +// requests. This header will also include "go-containerregistry/${version}". +// +// If you want to completely overwrite the User-Agent header, use WithTransport. +func WithUserAgent(ua string) Option { + return func(l *lister) error { + l.userAgent = ua + return nil + } +} diff --git a/pkg/v1/google/testdata/README.md b/pkg/v1/google/testdata/README.md new file mode 100644 index 0000000..12222aa --- /dev/null +++ b/pkg/v1/google/testdata/README.md @@ -0,0 +1,4 @@ +# testdata + +This key is cribbed from [here](https://github.com/golang/oauth2/blob/d668ce993890a79bda886613ee587a69dd5da7a6/google/testdata/gcloud/credentials). +It's invalid but parses sufficiently to test `NewEnvAuthenticator`. diff --git a/pkg/v1/google/testdata/key.json b/pkg/v1/google/testdata/key.json new file mode 100644 index 0000000..c2d23ce --- /dev/null +++ b/pkg/v1/google/testdata/key.json @@ -0,0 +1,35 @@ +{ + "_class": "OAuth2Credentials", + "_module": "oauth2client.client", + "access_token": "foo_access_token", + "client_id": "foo_client_id", + "client_secret": "foo_client_secret", + "id_token": { + "at_hash": "foo_at_hash", + "aud": "foo_aud", + "azp": "foo_azp", + "cid": "foo_cid", + "email": "foo@example.com", + "email_verified": true, + "exp": 1420573614, + "iat": 1420569714, + "id": "1337", + "iss": "accounts.google.com", + "sub": "1337", + "token_hash": "foo_token_hash", + "verified_email": true + }, + "invalid": false, + "refresh_token": "foo_refresh_token", + "revoke_uri": "https://accounts.google.com/o/oauth2/revoke", + "token_expiry": "3015-01-09T00:51:51Z", + "token_response": { + "access_token": "foo_access_token", + "expires_in": 3600, + "id_token": "foo_id_token", + "token_type": "Bearer" + }, + "token_uri": "https://accounts.google.com/o/oauth2/token", + "user_agent": "Cloud SDK Command Line Tool", + "type": "authorized_user" +} diff --git a/pkg/v1/hash.go b/pkg/v1/hash.go new file mode 100644 index 0000000..f78a5fa --- /dev/null +++ b/pkg/v1/hash.go @@ -0,0 +1,123 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "crypto" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "strconv" + "strings" +) + +// Hash is an unqualified digest of some content, e.g. sha256:deadbeef +type Hash struct { + // Algorithm holds the algorithm used to compute the hash. + Algorithm string + + // Hex holds the hex portion of the content hash. + Hex string +} + +// String reverses NewHash returning the string-form of the hash. +func (h Hash) String() string { + return fmt.Sprintf("%s:%s", h.Algorithm, h.Hex) +} + +// NewHash validates the input string is a hash and returns a strongly type Hash object. +func NewHash(s string) (Hash, error) { + h := Hash{} + if err := h.parse(s); err != nil { + return Hash{}, err + } + return h, nil +} + +// MarshalJSON implements json.Marshaler +func (h Hash) MarshalJSON() ([]byte, error) { + return json.Marshal(h.String()) +} + +// UnmarshalJSON implements json.Unmarshaler +func (h *Hash) UnmarshalJSON(data []byte) error { + s, err := strconv.Unquote(string(data)) + if err != nil { + return err + } + return h.parse(s) +} + +// MarshalText implements encoding.TextMarshaler. This is required to use +// v1.Hash as a key in a map when marshalling JSON. +func (h Hash) MarshalText() (text []byte, err error) { + return []byte(h.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. This is required to use +// v1.Hash as a key in a map when unmarshalling JSON. +func (h *Hash) UnmarshalText(text []byte) error { + return h.parse(string(text)) +} + +// Hasher returns a hash.Hash for the named algorithm (e.g. "sha256") +func Hasher(name string) (hash.Hash, error) { + switch name { + case "sha256": + return crypto.SHA256.New(), nil + default: + return nil, fmt.Errorf("unsupported hash: %q", name) + } +} + +func (h *Hash) parse(unquoted string) error { + parts := strings.Split(unquoted, ":") + if len(parts) != 2 { + return fmt.Errorf("cannot parse hash: %q", unquoted) + } + + rest := strings.TrimLeft(parts[1], "0123456789abcdef") + if len(rest) != 0 { + return fmt.Errorf("found non-hex character in hash: %c", rest[0]) + } + + hasher, err := Hasher(parts[0]) + if err != nil { + return err + } + // Compare the hex to the expected size (2 hex characters per byte) + if len(parts[1]) != hasher.Size()*2 { + return fmt.Errorf("wrong number of hex digits for %s: %s", parts[0], parts[1]) + } + + h.Algorithm = parts[0] + h.Hex = parts[1] + return nil +} + +// SHA256 computes the Hash of the provided io.Reader's content. +func SHA256(r io.Reader) (Hash, int64, error) { + hasher := crypto.SHA256.New() + n, err := io.Copy(hasher, r) + if err != nil { + return Hash{}, 0, err + } + return Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + }, n, nil +} diff --git a/pkg/v1/hash_test.go b/pkg/v1/hash_test.go new file mode 100644 index 0000000..df1be77 --- /dev/null +++ b/pkg/v1/hash_test.go @@ -0,0 +1,115 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "strconv" + "strings" + "testing" +) + +func TestGoodHashes(t *testing.T) { + good := []string{ + "sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + } + + for _, s := range good { + h, err := NewHash(s) + if err != nil { + t.Error("Unexpected error parsing hash:", err) + } + if got, want := h.String(), s; got != want { + t.Errorf("String(); got %q, want %q", got, want) + } + bytes, err := json.Marshal(h) + if err != nil { + t.Error("Unexpected error json.Marshaling hash:", err) + } + if got, want := string(bytes), strconv.Quote(h.String()); got != want { + t.Errorf("json.Marshal(); got %q, want %q", got, want) + } + } +} + +func TestBadHashes(t *testing.T) { + bad := []string{ + // Too short + "sha256:deadbeef", + // Bad character + "sha256:o123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + // Unknown algorithm + "md5:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + // Too few parts + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + // Too many parts + "md5:sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + } + + for _, s := range bad { + h, err := NewHash(s) + if err == nil { + t.Error("Expected error, got:", h) + } + } +} + +func TestSHA256(t *testing.T) { + input := "asdf" + h, n, err := SHA256(strings.NewReader(input)) + if err != nil { + t.Error("SHA256(asdf) =", err) + } + if got, want := h.Algorithm, "sha256"; got != want { + t.Errorf("Algorithm; got %v, want %v", got, want) + } + if got, want := h.Hex, "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"; got != want { + t.Errorf("Hex; got %v, want %v", got, want) + } + if got, want := n, int64(len(input)); got != want { + t.Errorf("n; got %v, want %v", got, want) + } +} + +// This tests that you can use Hash as a key in a map (needs to implement both +// MarshalText and UnmarshalText). +func TestTextMarshalling(t *testing.T) { + foo := make(map[Hash]string) + b, err := json.Marshal(foo) + if err != nil { + t.Fatal("could not marshal:", err) + } + if err := json.Unmarshal(b, &foo); err != nil { + t.Error("could not unmarshal:", err) + } + + h := &Hash{ + Algorithm: "sha256", + Hex: strings.Repeat("a", 64), + } + g := &Hash{} + text, err := h.MarshalText() + if err != nil { + t.Fatal(err) + } + if err := g.UnmarshalText(text); err != nil { + t.Fatal(err) + } + + if h.String() != g.String() { + t.Errorf("mismatched hash: %s != %s", h, g) + } +} diff --git a/pkg/v1/image.go b/pkg/v1/image.go new file mode 100644 index 0000000..8de9e47 --- /dev/null +++ b/pkg/v1/image.go @@ -0,0 +1,59 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Image defines the interface for interacting with an OCI v1 image. +type Image interface { + // Layers returns the ordered collection of filesystem layers that comprise this image. + // The order of the list is oldest/base layer first, and most-recent/top layer last. + Layers() ([]Layer, error) + + // MediaType of this image's manifest. + MediaType() (types.MediaType, error) + + // Size returns the size of the manifest. + Size() (int64, error) + + // ConfigName returns the hash of the image's config file, also known as + // the Image ID. + ConfigName() (Hash, error) + + // ConfigFile returns this image's config file. + ConfigFile() (*ConfigFile, error) + + // RawConfigFile returns the serialized bytes of ConfigFile(). + RawConfigFile() ([]byte, error) + + // Digest returns the sha256 of this image's manifest. + Digest() (Hash, error) + + // Manifest returns this image's Manifest object. + Manifest() (*Manifest, error) + + // RawManifest returns the serialized bytes of Manifest() + RawManifest() ([]byte, error) + + // LayerByDigest returns a Layer for interacting with a particular layer of + // the image, looking it up by "digest" (the compressed hash). + LayerByDigest(Hash) (Layer, error) + + // LayerByDiffID is an analog to LayerByDigest, looking up by "diff id" + // (the uncompressed hash). + LayerByDiffID(Hash) (Layer, error) +} diff --git a/pkg/v1/index.go b/pkg/v1/index.go new file mode 100644 index 0000000..8e7bc8e --- /dev/null +++ b/pkg/v1/index.go @@ -0,0 +1,43 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// ImageIndex defines the interface for interacting with an OCI image index. +type ImageIndex interface { + // MediaType of this image's manifest. + MediaType() (types.MediaType, error) + + // Digest returns the sha256 of this index's manifest. + Digest() (Hash, error) + + // Size returns the size of the manifest. + Size() (int64, error) + + // IndexManifest returns this image index's manifest object. + IndexManifest() (*IndexManifest, error) + + // RawManifest returns the serialized bytes of IndexManifest(). + RawManifest() ([]byte, error) + + // Image returns a v1.Image that this ImageIndex references. + Image(Hash) (Image, error) + + // ImageIndex returns a v1.ImageIndex that this ImageIndex references. + ImageIndex(Hash) (ImageIndex, error) +} diff --git a/pkg/v1/layer.go b/pkg/v1/layer.go new file mode 100644 index 0000000..57447d2 --- /dev/null +++ b/pkg/v1/layer.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "io" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Layer is an interface for accessing the properties of a particular layer of a v1.Image +type Layer interface { + // Digest returns the Hash of the compressed layer. + Digest() (Hash, error) + + // DiffID returns the Hash of the uncompressed layer. + DiffID() (Hash, error) + + // Compressed returns an io.ReadCloser for the compressed layer contents. + Compressed() (io.ReadCloser, error) + + // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. + Uncompressed() (io.ReadCloser, error) + + // Size returns the compressed size of the Layer. + Size() (int64, error) + + // MediaType returns the media type of the Layer. + MediaType() (types.MediaType, error) +} diff --git a/pkg/v1/layout/README.md b/pkg/v1/layout/README.md new file mode 100644 index 0000000..54bee6d --- /dev/null +++ b/pkg/v1/layout/README.md @@ -0,0 +1,5 @@ +# `layout` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout) + +The `layout` package implements support for interacting with an [OCI Image Layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md). diff --git a/pkg/v1/layout/blob.go b/pkg/v1/layout/blob.go new file mode 100644 index 0000000..2e5f435 --- /dev/null +++ b/pkg/v1/layout/blob.go @@ -0,0 +1,37 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return os.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/pkg/v1/layout/doc.go b/pkg/v1/layout/doc.go new file mode 100644 index 0000000..d80d273 --- /dev/null +++ b/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/pkg/v1/layout/image.go b/pkg/v1/layout/image.go new file mode 100644 index 0000000..c9ae966 --- /dev/null +++ b/pkg/v1/layout/image.go @@ -0,0 +1,139 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "os" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return &compressedBlob{ + path: li.path, + desc: manifest.Config, + }, nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + return &compressedBlob{ + path: li.path, + desc: desc, + }, nil + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} + +// Descriptor implements partial.withDescriptor. +func (b *compressedBlob) Descriptor() (*v1.Descriptor, error) { + return &b.desc, nil +} + +// See partial.Exists. +func (b *compressedBlob) Exists() (bool, error) { + _, err := os.Stat(b.path.blobPath(b.desc.Digest)) + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err +} diff --git a/pkg/v1/layout/image_test.go b/pkg/v1/layout/image_test.go new file mode 100644 index 0000000..3614920 --- /dev/null +++ b/pkg/v1/layout/image_test.go @@ -0,0 +1,181 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "path/filepath" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +var ( + indexDigest = v1.Hash{ + Algorithm: "sha256", + Hex: "05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5", + } + manifestDigest = v1.Hash{ + Algorithm: "sha256", + Hex: "eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + } + configDigest = v1.Hash{ + Algorithm: "sha256", + Hex: "6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e", + } + bogusDigest = v1.Hash{ + Algorithm: "sha256", + Hex: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + } + customManifestDigest = v1.Hash{ + Algorithm: "sha256", + Hex: "b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e", + } + bogusPath = filepath.Join("testdata", "does_not_exist") + testPath = filepath.Join("testdata", "test_index") + testPathOneImage = filepath.Join("testdata", "test_index_one_image") + testPathMediaType = filepath.Join("testdata", "test_index_media_type") + customMediaType types.MediaType = "application/tar+gzip" +) + +func TestImage(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(manifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + if err := validate.Image(img); err != nil { + t.Errorf("validate.Image() = %v", err) + } + + mt, err := img.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } else if got, want := mt, types.OCIManifestSchema1; got != want { + t.Errorf("MediaType(); want: %v got: %v", want, got) + } + + cfg, err := img.LayerByDigest(configDigest) + if err != nil { + t.Fatalf("LayerByDigest(%s) = %v", configDigest, err) + } + + cfgName, err := img.ConfigName() + if err != nil { + t.Fatalf("ConfigName() = %v", err) + } + + cfgDigest, err := cfg.Digest() + if err != nil { + t.Fatalf("cfg.Digest() = %v", err) + } + + if got, want := cfgDigest, cfgName; got != want { + t.Errorf("ConfigName(); want: %v got: %v", want, got) + } + + layers, err := img.Layers() + if err != nil { + t.Fatalf("img.Layers() = %v", err) + } + + mediaType, err := layers[0].MediaType() + if err != nil { + t.Fatalf("img.Layers() = %v", err) + } + + // Fixture is a DockerLayer + if got, want := mediaType, types.DockerLayer; got != want { + t.Fatalf("MediaType(); want: %q got: %q", want, got) + } + + if ok, err := partial.Exists(layers[0]); err != nil { + t.Fatal(err) + } else if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } +} + +func TestImageWithEmptyHash(t *testing.T) { + lp, err := FromPath(testPathOneImage) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(v1.Hash{}) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + if err := validate.Image(img); err != nil { + t.Errorf("validate.Image() = %v", err) + } +} + +func TestImageErrors(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(manifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + if _, err := img.LayerByDigest(bogusDigest); err == nil { + t.Errorf("LayerByDigest(%s) = nil, expected err", bogusDigest) + } + + if _, err := lp.Image(bogusDigest); err == nil { + t.Errorf("Image(%s) = nil, expected err", bogusDigest) + } + + if _, err := lp.Image(bogusDigest); err == nil { + t.Errorf("Image(%s, %s) = nil, expected err", bogusPath, bogusDigest) + } +} + +func TestImageCustomMediaType(t *testing.T) { + lp, err := FromPath(testPathMediaType) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(customManifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + mt, err := img.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } else if got, want := mt, types.OCIManifestSchema1; got != want { + t.Errorf("MediaType(); want: %v got: %v", want, got) + } + layers, err := img.Layers() + if err != nil { + t.Fatalf("img.Layers() = %v", err) + } + mediaType, err := layers[0].MediaType() + if err != nil { + t.Fatalf("img.Layers() = %v", err) + } + if got, want := mediaType, customMediaType; got != want { + t.Fatalf("MediaType(); want: %q got: %q", want, got) + } +} diff --git a/pkg/v1/layout/index.go b/pkg/v1/layout/index.go new file mode 100644 index 0000000..7404f18 --- /dev/null +++ b/pkg/v1/layout/index.go @@ -0,0 +1,161 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + mediaType types.MediaType + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := os.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + mediaType: types.OCIImageIndex, + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return i.mediaType, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) Size() (int64, error) { + return partial.Size(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + mediaType: desc.MediaType, + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + if h == (v1.Hash{}) { + if len(im.Manifests) != 1 { + return nil, errors.New("oci layout must contain only a single image to be used with layout.Image") + } + return &(im.Manifests)[0], nil + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/pkg/v1/layout/index_test.go b/pkg/v1/layout/index_test.go new file mode 100644 index 0000000..4478831 --- /dev/null +++ b/pkg/v1/layout/index_test.go @@ -0,0 +1,81 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestIndex(t *testing.T) { + idx, err := ImageIndexFromPath(testPath) + if err != nil { + t.Fatalf("ImageIndexFromPath() = %v", err) + } + + if err := validate.Index(idx); err != nil { + t.Errorf("validate.Index() = %v", err) + } + + mt, err := idx.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + + if got, want := mt, types.OCIImageIndex; got != want { + t.Errorf("MediaType(); want: %v got: %v", want, got) + } + + indexHash, _ := v1.NewHash("sha256:2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb") + ii, err := idx.ImageIndex(indexHash) + if err != nil { + t.Fatalf("ImageIndex() = %v", err) + } + + mt, err = ii.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + + if got, want := mt, types.DockerManifestList; got != want { + t.Errorf("MediaType(); want: %v got: %v", want, got) + } +} + +func TestIndexErrors(t *testing.T) { + idx, err := ImageIndexFromPath(testPath) + if err != nil { + t.Fatalf("ImageIndexFromPath() = %v", err) + } + + if _, err := idx.Image(bogusDigest); err == nil { + t.Errorf("idx.Image(%s) = nil, expected err", bogusDigest) + } + + if _, err := idx.Image(indexDigest); err == nil { + t.Errorf("idx.Image(%s) = nil, expected err", bogusDigest) + } + + if _, err := idx.ImageIndex(bogusDigest); err == nil { + t.Errorf("idx.ImageIndex(%s) = nil, expected err", bogusDigest) + } + + if _, err := idx.ImageIndex(manifestDigest); err == nil { + t.Errorf("idx.ImageIndex(%s) = nil, expected err", bogusDigest) + } +} diff --git a/pkg/v1/layout/layoutpath.go b/pkg/v1/layout/layoutpath.go new file mode 100644 index 0000000..a031ff5 --- /dev/null +++ b/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/pkg/v1/layout/options.go b/pkg/v1/layout/options.go new file mode 100644 index 0000000..a26f9f3 --- /dev/null +++ b/pkg/v1/layout/options.go @@ -0,0 +1,71 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +type Option func(*options) + +type options struct { + descOpts []descriptorOption +} + +func makeOptions(opts ...Option) *options { + o := &options{ + descOpts: []descriptorOption{}, + } + for _, apply := range opts { + apply(o) + } + return o +} + +type descriptorOption func(*v1.Descriptor) + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + }) + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + }) + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + desc.Platform = &platform + }) + } +} diff --git a/pkg/v1/layout/read.go b/pkg/v1/layout/read.go new file mode 100644 index 0000000..796abc7 --- /dev/null +++ b/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/pkg/v1/layout/read_test.go b/pkg/v1/layout/read_test.go new file mode 100644 index 0000000..281fa29 --- /dev/null +++ b/pkg/v1/layout/read_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "testing" +) + +func TestRead(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + if testPath != lp.path() { + t.Errorf("unexpected path %s", lp.path()) + } +} + +func TestReadErrors(t *testing.T) { + if _, err := FromPath(bogusPath); err == nil { + t.Errorf("FromPath(%s) = nil, expected err", bogusPath) + } + + // Found this here: + // https://github.com/golang/go/issues/24195 + invalidPath := "double-null-padded-string\x00\x00" + if _, err := FromPath(invalidPath); err == nil { + t.Errorf("FromPath(%s) = nil, expected err", bogusPath) + } +} diff --git a/pkg/v1/layout/testdata/README.md b/pkg/v1/layout/testdata/README.md new file mode 100644 index 0000000..449ff1c --- /dev/null +++ b/pkg/v1/layout/testdata/README.md @@ -0,0 +1,5 @@ +# Where did this data come from? + +These were manually produced from the pkg/v1/tarball/testadata tarballs. + +TODO: Make this reproducible. There's not currently an easy way to do this. diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 b/pkg/v1/layout/testdata/test_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 new file mode 100644 index 0000000..1597d07 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "1" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb b/pkg/v1/layout/testdata/test_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb new file mode 100644 index 0000000..e6587e2 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "4" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 b/pkg/v1/layout/testdata/test_index/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 new file mode 100644 index 0000000..096f21f Binary files /dev/null and b/pkg/v1/layout/testdata/test_index/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 differ diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 b/pkg/v1/layout/testdata/test_index/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 new file mode 100644 index 0000000..48609c6 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":165,"digest":"sha256:321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3"}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e b/pkg/v1/layout/testdata/test_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e new file mode 100644 index 0000000..4228c89 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e @@ -0,0 +1 @@ +{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}} diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 b/pkg/v1/layout/testdata/test_index/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 new file mode 100644 index 0000000..425c2d0 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 @@ -0,0 +1 @@ +{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:3610aa5267a210147ba6ca02cdd87610dfc08522de9c5f5015edd8ee14853fd8"], "type": "layers"}} diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b b/pkg/v1/layout/testdata/test_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b new file mode 100644 index 0000000..05c6321 Binary files /dev/null and b/pkg/v1/layout/testdata/test_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b differ diff --git a/pkg/v1/layout/testdata/test_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 b/pkg/v1/layout/testdata/test_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 new file mode 100644 index 0000000..21dc412 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_index/index.json b/pkg/v1/layout/testdata/test_index/index.json new file mode 100644 index 0000000..28df736 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/index.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "1" + } + }, + { + "mediaType": "application/vnd.oci.descriptor.v1+json", + "size": 423, + "digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720", + "annotations": { + "org.opencontainers.image.ref.name": "2" + } + }, + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": 314, + "digest": "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5", + "annotations": { + "org.opencontainers.image.ref.name": "3" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "size": 314, + "digest": "sha256:2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb", + "annotations": { + "org.opencontainers.image.ref.name": "4" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_index/oci-layout b/pkg/v1/layout/testdata/test_index/oci-layout new file mode 100644 index 0000000..10ff2f3 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} diff --git a/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e new file mode 100644 index 0000000..53aea8f --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e @@ -0,0 +1,15 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.cncf.helm.config.v1+json", + "digest": "sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356", + "size": 3 + }, + "layers": [ + { + "mediaType": "application/tar+gzip", + "digest": "sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b", + "size": 167 + } + ] +} diff --git a/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 @@ -0,0 +1 @@ +{} diff --git a/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b new file mode 100644 index 0000000..05c6321 Binary files /dev/null and b/pkg/v1/layout/testdata/test_index_media_type/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b differ diff --git a/pkg/v1/layout/testdata/test_index_media_type/index.json b/pkg/v1/layout/testdata/test_index_media_type/index.json new file mode 100644 index 0000000..ffe4d3e --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_media_type/index.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 391, + "digest": "sha256:b544f71ecd82372bc9a3c0dbef378abfd2734fe437df81ff6e242a0d720d8e3e" + } + ] +} diff --git a/pkg/v1/layout/testdata/test_index_media_type/oci-layout b/pkg/v1/layout/testdata/test_index_media_type/oci-layout new file mode 100644 index 0000000..10ff2f3 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_media_type/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} diff --git a/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/381d958b555884ba59574ab5c066e9f6116b5aec3567675aa13bec63331f0810 b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/381d958b555884ba59574ab5c066e9f6116b5aec3567675aa13bec63331f0810 new file mode 100644 index 0000000..e49e018 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/381d958b555884ba59574ab5c066e9f6116b5aec3567675aa13bec63331f0810 @@ -0,0 +1 @@ +{"created":"2020-04-12T10:58:48.626858334Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:59cd31f50f7442a662d7c31b7a12079ade16892bfb465b33da49918e7d13e747"]},"history":[{"created":"2020-04-12T10:58:48.626858334Z","created_by":"/bin/sh -c #(nop) COPY file:a34f6c104b4cb0668083b4de122deebb3e3629e212f82c32fec316dd8e3a1931 in / "}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 new file mode 100644 index 0000000..1e4eb22 Binary files /dev/null and b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0 differ diff --git a/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/98ceaf93e482fe91b9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46 b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/98ceaf93e482fe91b9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46 new file mode 100644 index 0000000..f02779b --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_one_image/blobs/sha256/98ceaf93e482fe91b9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46 @@ -0,0 +1 @@ +{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:381d958b555884ba59574ab5c066e9f6116b5aec3567675aa13bec63331f0810","size":452},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0","size":114}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_index_one_image/index.json b/pkg/v1/layout/testdata/test_index_one_image/index.json new file mode 100644 index 0000000..4ec03cc --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_one_image/index.json @@ -0,0 +1 @@ +{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:98ceaf93e482fe91b9bfd6bba07137c098e49ee2d55e69f09fb6c951e75e0e46","size":344}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_index_one_image/oci-layout b/pkg/v1/layout/testdata/test_index_one_image/oci-layout new file mode 100644 index 0000000..21b1439 --- /dev/null +++ b/pkg/v1/layout/testdata/test_index_one_image/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} \ No newline at end of file diff --git a/pkg/v1/layout/write.go b/pkg/v1/layout/write.go new file mode 100644 index 0000000..906b12a --- /dev/null +++ b/pkg/v1/layout/write.go @@ -0,0 +1,481 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.WriteImage(img); err != nil { + return err + } + + desc, err := partial.Descriptor(img) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + return l.AppendDescriptor(*desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.WriteIndex(ii); err != nil { + return err + } + + desc, err := partial.Descriptor(ii) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + return l.AppendDescriptor(*desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// ReplaceImage writes a v1.Image to the Path and updates +// the index.json to reference it, replacing any existing one that matches matcher, if found. +func (l Path) ReplaceImage(img v1.Image, matcher match.Matcher, options ...Option) error { + if err := l.WriteImage(img); err != nil { + return err + } + + return l.replaceDescriptor(img, matcher, options...) +} + +// ReplaceIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it, replacing any existing one that matches matcher, if found. +func (l Path) ReplaceIndex(ii v1.ImageIndex, matcher match.Matcher, options ...Option) error { + if err := l.WriteIndex(ii); err != nil { + return err + } + + return l.replaceDescriptor(ii, matcher, options...) +} + +// replaceDescriptor adds a descriptor to the index.json of the Path, replacing +// any one matching matcher, if found. +func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, options ...Option) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + desc, err := partial.Descriptor(append) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + add := mutate.IndexAddendum{ + Add: append, + Descriptor: *desc, + } + ii = mutate.AppendManifests(mutate.RemoveManifests(ii, matcher), add) + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// RemoveDescriptors removes any descriptors that match the match.Matcher from the index.json of the Path. +func (l Path) RemoveDescriptors(matcher match.Matcher) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + ii = mutate.RemoveManifests(ii, matcher) + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// WriteFile write a file with arbitrary data at an arbitrary location in a v1 +// layout. Used mostly internally to write files like "oci-layout" and +// "index.json", also can be used to write other arbitrary files. Do *not* use +// this to write blobs. Use only WriteBlob() for that. +func (l Path) WriteFile(name string, data []byte, perm os.FileMode) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return os.WriteFile(l.path(name), data, perm) +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + return l.writeBlob(hash, -1, r, nil) +} + +func (l Path) writeBlob(hash v1.Hash, size int64, rc io.ReadCloser, renamer func() (v1.Hash, error)) error { + if hash.Hex == "" && renamer == nil { + panic("writeBlob called an invalid hash and no renamer") + } + + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + // Check if blob already exists and is the correct size + file := filepath.Join(dir, hash.Hex) + if s, err := os.Stat(file); err == nil && !s.IsDir() && (s.Size() == size || size == -1) { + return nil + } + + // If a renamer func was provided write to a temporary file + open := func() (*os.File, error) { return os.Create(file) } + if renamer != nil { + open = func() (*os.File, error) { return os.CreateTemp(dir, hash.Hex) } + } + w, err := open() + if err != nil { + return err + } + if renamer != nil { + // Delete temp file if an error is encountered before renaming + defer func() { + if err := os.Remove(w.Name()); err != nil && !errors.Is(err, os.ErrNotExist) { + logs.Warn.Printf("error removing temporary file after encountering an error while writing blob: %v", err) + } + }() + } + defer w.Close() + + // Write to file and exit if not renaming + if n, err := io.Copy(w, rc); err != nil || renamer == nil { + return err + } else if size != -1 && n != size { + return fmt.Errorf("expected blob size %d, but only wrote %d", size, n) + } + + // Always close reader before renaming, since Close computes the digest in + // the case of streaming layers. If Close is not called explicitly, it will + // occur in a goroutine that is not guaranteed to succeed before renamer is + // called. When renamer is the layer's Digest method, it can return + // ErrNotComputed. + if err := rc.Close(); err != nil { + return err + } + + // Always close file before renaming + if err := w.Close(); err != nil { + return err + } + + // Rename file based on the final hash + finalHash, err := renamer() + if err != nil { + return fmt.Errorf("error getting final digest of layer: %w", err) + } + + renamePath := l.path("blobs", finalHash.Algorithm, finalHash.Hex) + return os.Rename(w.Name(), renamePath) +} + +// writeLayer writes the compressed layer to a blob. Unlike WriteBlob it will +// write to a temporary file (suffixed with .tmp) within the layout until the +// compressed reader is fully consumed and written to disk. Also unlike +// WriteBlob, it will not skip writing and exit without error when a blob file +// exists, but does not have the correct size. (The blob hash is not +// considered, because it may be expensive to compute.) +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if errors.Is(err, stream.ErrNotComputed) { + // Allow digest errors, since streams may not have calculated the hash + // yet. Instead, use an empty value, which will be transformed into a + // random file name with `os.CreateTemp` and the final digest will be + // calculated after writing to a temp file and before renaming to the + // final path. + d = v1.Hash{Algorithm: "sha256", Hex: ""} + } else if err != nil { + return err + } + + s, err := layer.Size() + if errors.Is(err, stream.ErrNotComputed) { + // Allow size errors, since streams may not have calculated the size + // yet. Instead, use zero as a sentinel value meaning that no size + // comparison can be done and any sized blob file should be considered + // valid and not overwritten. + // + // TODO: Provide an option to always overwrite blobs. + s = -1 + } else if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + if err := l.writeBlob(d, s, r, layer.Digest); err != nil { + return fmt.Errorf("error writing layer: %w", err) + } + return nil +} + +// RemoveBlob removes a file from the blobs directory in the Path +// at blobs/{hash.Algorithm}/{hash.Hex} +// It does *not* remove any reference to it from other manifests or indexes, or +// from the root index.json. +func (l Path) RemoveBlob(hash v1.Hash) error { + dir := l.path("blobs", hash.Algorithm) + err := os.Remove(filepath.Join(dir, hash.Hex)) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// WriteImage writes an image, including its manifest, config and all of its +// layers, to the blobs directory. If any blob already exists, as determined by +// the hash filename, does not write it. +// This function does *not* update the `index.json` file. If you want to write the +// image and also update the `index.json`, call AppendImage(), which wraps this +// and also updates the `index.json`. +func (l Path) WriteImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest))) +} + +type withLayer interface { + Layer(v1.Hash) (v1.Layer, error) +} + +type withBlob interface { + Blob(v1.Hash) (io.ReadCloser, error) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.WriteIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.WriteImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + + var blob io.ReadCloser + // Workaround for #819. + if wl, ok := ii.(withLayer); ok { + layer, lerr := wl.Layer(desc.Digest) + if lerr != nil { + return lerr + } + blob, err = layer.Compressed() + } else if wb, ok := ii.(withBlob); ok { + blob, err = wb.Blob(desc.Digest) + } + if err != nil { + return err + } + if err := l.WriteBlob(desc.Digest, blob); err != nil { + return err + } + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.WriteFile(indexFile, rawIndex, os.ModePerm) +} + +// WriteIndex writes an index to the blobs directory. Walks down the children, +// including its children manifests and/or indexes, and down the tree until all of +// config and all layers, have been written. If any blob already exists, as determined by +// the hash filename, does not write it. +// This function does *not* update the `index.json` file. If you want to write the +// index and also update the `index.json`, call AppendIndex(), which wraps this +// and also updates the `index.json`. +func (l Path) WriteIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// +// Under blobs/, there is, for each image: +// +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} diff --git a/pkg/v1/layout/write_test.go b/pkg/v1/layout/write_test.go new file mode 100644 index 0000000..530e0e8 --- /dev/null +++ b/pkg/v1/layout/write_test.go @@ -0,0 +1,672 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "bytes" + "io" + "log" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestWrite(t *testing.T) { + tmp := t.TempDir() + + original, err := ImageIndexFromPath(testPath) + if err != nil { + t.Fatal(err) + } + + if layoutPath, err := Write(tmp, original); err != nil { + t.Fatalf("Write(%s) = %v", tmp, err) + } else if tmp != layoutPath.path() { + t.Fatalf("unexpected file system path %v", layoutPath) + } + + written, err := ImageIndexFromPath(tmp) + if err != nil { + t.Fatal(err) + } + + if err := validate.Index(written); err != nil { + t.Fatalf("validate.Index() = %v", err) + } +} + +func TestWriteErrors(t *testing.T) { + idx, err := ImageIndexFromPath(testPath) + if err != nil { + t.Fatalf("ImageIndexFromPath() = %v", err) + } + + // Found this here: + // https://github.com/golang/go/issues/24195 + invalidPath := "double-null-padded-string\x00\x00" + if _, err := Write(invalidPath, idx); err == nil { + t.Fatalf("Write(%s) = nil, expected err", invalidPath) + } +} + +func TestAppendDescriptorInitializesIndex(t *testing.T) { + tmp := t.TempDir() + temp, err := Write(tmp, empty.Index) + if err != nil { + t.Fatal(err) + } + + // Append a descriptor to a non-existent layout. + desc := v1.Descriptor{ + Digest: bogusDigest, + Size: 1337, + MediaType: types.MediaType("not real"), + } + if err := temp.AppendDescriptor(desc); err != nil { + t.Fatalf("AppendDescriptor(%s) = %v", tmp, err) + } + + // Read that layout from disk and make sure the descriptor is there. + idx, err := ImageIndexFromPath(tmp) + if err != nil { + t.Fatalf("ImageIndexFromPath() = %v", err) + } + + manifest, err := idx.IndexManifest() + if err != nil { + t.Fatalf("IndexManifest() = %v", err) + } + if diff := cmp.Diff(manifest.Manifests[0], desc); diff != "" { + t.Fatalf("bad descriptor: (-got +want) %s", diff) + } +} + +func TestRoundtrip(t *testing.T) { + tmp := t.TempDir() + + original, err := ImageIndexFromPath(testPath) + if err != nil { + t.Fatal(err) + } + + originalManifest, err := original.IndexManifest() + if err != nil { + t.Fatal(err) + } + + // Write it back. + if _, err := Write(tmp, original); err != nil { + t.Fatal(err) + } + reconstructed, err := ImageIndexFromPath(tmp) + if err != nil { + t.Fatalf("ImageIndexFromPath() = %v", err) + } + reconstructedManifest, err := reconstructed.IndexManifest() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(originalManifest, reconstructedManifest); diff != "" { + t.Fatalf("bad manifest: (-got +want) %s", diff) + } +} + +func TestOptions(t *testing.T) { + tmp := t.TempDir() + temp, err := Write(tmp, empty.Index) + if err != nil { + t.Fatal(err) + } + annotations := map[string]string{ + "foo": "bar", + } + urls := []string{"https://example.com"} + platform := v1.Platform{ + Architecture: "mill", + OS: "haiku", + } + img, err := random.Image(5, 5) + if err != nil { + t.Fatal(err) + } + options := []Option{ + WithAnnotations(annotations), + WithURLs(urls), + WithPlatform(platform), + } + if err := temp.AppendImage(img, options...); err != nil { + t.Fatal(err) + } + idx, err := temp.ImageIndex() + if err != nil { + t.Fatal(err) + } + indexManifest, err := idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + + desc := indexManifest.Manifests[0] + if got, want := desc.Annotations["foo"], "bar"; got != want { + t.Fatalf("wrong annotation; got: %v, want: %v", got, want) + } + if got, want := desc.URLs[0], "https://example.com"; got != want { + t.Fatalf("wrong urls; got: %v, want: %v", got, want) + } + if got, want := desc.Platform.Architecture, "mill"; got != want { + t.Fatalf("wrong Architecture; got: %v, want: %v", got, want) + } + if got, want := desc.Platform.OS, "haiku"; got != want { + t.Fatalf("wrong OS; got: %v, want: %v", got, want) + } +} + +func TestDeduplicatedWrites(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + b, err := lp.Blob(configDigest) + if err != nil { + t.Fatal(err) + } + + buf := bytes.NewBuffer([]byte{}) + if _, err := io.Copy(buf, b); err != nil { + log.Fatal(err) + } + + if err := lp.WriteBlob(configDigest, io.NopCloser(bytes.NewBuffer(buf.Bytes()))); err != nil { + t.Fatal(err) + } + + if err := lp.WriteBlob(configDigest, io.NopCloser(bytes.NewBuffer(buf.Bytes()))); err != nil { + t.Fatal(err) + } +} + +func TestRemoveDescriptor(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex + ii = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // add two images + image1, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendImage(image1); err != nil { + t.Fatal(err) + } + image2, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendImage(image2); err != nil { + t.Fatal(err) + } + + // remove one of the images by descriptor and ensure it is correct + digest1, err := image1.Digest() + if err != nil { + t.Fatal(err) + } + digest2, err := image2.Digest() + if err != nil { + t.Fatal(err) + } + if err := l.RemoveDescriptors(match.Digests(digest1)); err != nil { + t.Fatal(err) + } + // ensure we only have one + ii, err = l.ImageIndex() + if err != nil { + t.Fatal(err) + } + manifest, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(manifest.Manifests) != 1 { + t.Fatalf("mismatched manifests count, had %d, expected %d", len(manifest.Manifests), 1) + } + if manifest.Manifests[0].Digest != digest2 { + t.Fatal("removed wrong digest") + } +} + +func TestReplaceIndex(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex + ii = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // add two indexes + index1, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendIndex(index1); err != nil { + t.Fatal(err) + } + index2, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendIndex(index2); err != nil { + t.Fatal(err) + } + index3, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + + // remove one of the indexes by descriptor and ensure it is correct + digest1, err := index1.Digest() + if err != nil { + t.Fatal(err) + } + digest3, err := index3.Digest() + if err != nil { + t.Fatal(err) + } + if err := l.ReplaceIndex(index3, match.Digests(digest1)); err != nil { + t.Fatal(err) + } + // ensure we only have one + ii, err = l.ImageIndex() + if err != nil { + t.Fatal(err) + } + manifest, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(manifest.Manifests) != 2 { + t.Fatalf("mismatched manifests count, had %d, expected %d", len(manifest.Manifests), 2) + } + // we should have digest3, and *not* have digest1 + var have3 bool + for _, m := range manifest.Manifests { + if m.Digest == digest1 { + t.Fatal("found digest1 still not replaced", digest1) + } + if m.Digest == digest3 { + have3 = true + } + } + if !have3 { + t.Fatal("could not find digest3", digest3) + } +} + +func TestReplaceImage(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex + ii = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // add two images + image1, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendImage(image1); err != nil { + t.Fatal(err) + } + image2, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + if err := l.AppendImage(image2); err != nil { + t.Fatal(err) + } + image3, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + + // remove one of the images by descriptor and ensure it is correct + digest1, err := image1.Digest() + if err != nil { + t.Fatal(err) + } + digest3, err := image3.Digest() + if err != nil { + t.Fatal(err) + } + if err := l.ReplaceImage(image3, match.Digests(digest1)); err != nil { + t.Fatal(err) + } + // ensure we only have one + ii, err = l.ImageIndex() + if err != nil { + t.Fatal(err) + } + manifest, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(manifest.Manifests) != 2 { + t.Fatalf("mismatched manifests count, had %d, expected %d", len(manifest.Manifests), 2) + } + // we should have digest3, and *not* have digest1 + var have3 bool + for _, m := range manifest.Manifests { + if m.Digest == digest1 { + t.Fatal("found digest1 still not replaced", digest1) + } + if m.Digest == digest3 { + have3 = true + } + } + if !have3 { + t.Fatal("could not find digest3", digest3) + } +} + +func TestRemoveBlob(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // create a random blob + b := []byte("abcdefghijklmnop") + hash, _, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + + if err := l.WriteBlob(hash, io.NopCloser(bytes.NewReader(b))); err != nil { + t.Fatal(err) + } + // make sure it exists + b2, err := l.Bytes(hash) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, b2) { + t.Fatal("mismatched bytes") + } + // now the real test, delete it + if err := l.RemoveBlob(hash); err != nil { + t.Fatal(err) + } + // now it should not exist + if _, err = l.Bytes(hash); err == nil { + t.Fatal("still existed after deletion") + } +} + +func TestStreamingWriteLayer(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // create a random streaming image and persist + pr, pw := io.Pipe() + tw := tar.NewWriter(pw) + go func() { + pw.CloseWithError(func() error { + body := "test file" + if err := tw.WriteHeader(&tar.Header{ + Name: "test.txt", + Mode: 0600, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + if _, err := tw.Write([]byte(body)); err != nil { + return err + } + return tw.Close() + }()) + }() + img, err := mutate.Append(empty.Image, mutate.Addendum{ + Layer: stream.NewLayer(pr), + }) + if err != nil { + t.Fatalf("creating random streaming image failed: %v", err) + } + if _, err := img.Digest(); err == nil { + t.Fatal("digesting image before stream is consumed; (v1.Image).Digest() = nil, expected err") + } + // AppendImage uses writeLayer + if err := l.AppendImage(img); err != nil { + t.Fatalf("(Path).AppendImage() = %v", err) + } + + // Check that image was persisted and is valid + imgDigest, err := img.Digest() + if err != nil { + t.Fatalf("(v1.Image).Digest() = %v", err) + } + img, err = l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading image after writeLayer for validation; (Path).Image = %v", err) + } + if err := validate.Image(img); err != nil { + t.Fatalf("validate.Image() = %v", err) + } +} + +func TestOverwriteWithWriteLayer(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // create a random image and persist + img, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + imgDigest, err := img.Digest() + if err != nil { + t.Fatalf("(v1.Image).Digest() = %v", err) + } + if err := l.AppendImage(img); err != nil { + t.Fatalf("(Path).AppendImage() = %v", err) + } + if err := validate.Image(img); err != nil { + t.Fatalf("validate.Image() = %v", err) + } + + // get the random image's layer + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + if n := len(layers); n != 1 { + t.Fatalf("expected image with 1 layer, got %d", n) + } + + layer := layers[0] + layerDigest, err := layer.Digest() + if err != nil { + t.Fatalf("(v1.Layer).Digest() = %v", err) + } + + // truncate the layer contents on disk + completeLayerBytes, err := l.Bytes(layerDigest) + if err != nil { + t.Fatalf("(Path).Bytes() = %v", err) + } + truncatedLayerBytes := completeLayerBytes[:512] + + path := l.path("blobs", layerDigest.Algorithm, layerDigest.Hex) + if err := os.WriteFile(path, truncatedLayerBytes, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(layerPath, truncated) = %v", err) + } + + // ensure validation fails + img, err = l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading truncated image for validation; (Path).Image = %v", err) + } + if err := validate.Image(img); err == nil { + t.Fatal("validating image after truncating layer; validate.Image() = nil, expected err") + } + + // try writing expected contents with WriteBlob + if err := l.WriteBlob(layerDigest, io.NopCloser(bytes.NewBuffer(completeLayerBytes))); err != nil { + t.Fatalf("error attempting to overwrite truncated layer with valid layer; (Path).WriteBlob = %v", err) + } + + // validation should still fail + img, err = l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading truncated image after WriteBlob for validation; (Path).Image = %v", err) + } + if err := validate.Image(img); err == nil { + t.Fatal("validating image after attempting repair of truncated layer with WriteBlob; validate.Image() = nil, expected err") + } + + // try writing expected contents with writeLayer + if err := l.writeLayer(layer); err != nil { + t.Fatalf("error attempting to overwrite truncated layer with valid layer; (Path).writeLayer = %v", err) + } + + // validation should now succeed + img, err = l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading truncated image after writeLayer for validation; (Path).Image = %v", err) + } + if err := validate.Image(img); err != nil { + t.Fatalf("validating image after attempting repair of truncated layer with writeLayer; validate.Image() = %v", err) + } +} + +func TestOverwriteWithReplaceImage(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // create a random image and persist + img, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + imgDigest, err := img.Digest() + if err != nil { + t.Fatalf("(v1.Image).Digest() = %v", err) + } + if err := l.AppendImage(img); err != nil { + t.Fatalf("(Path).AppendImage() = %v", err) + } + if err := validate.Image(img); err != nil { + t.Fatalf("validate.Image() = %v", err) + } + + // get the random image's layer + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + if n := len(layers); n != 1 { + t.Fatalf("expected image with 1 layer, got %d", n) + } + + layer := layers[0] + layerDigest, err := layer.Digest() + if err != nil { + t.Fatalf("(v1.Layer).Digest() = %v", err) + } + + // truncate the layer contents on disk + completeLayerBytes, err := l.Bytes(layerDigest) + if err != nil { + t.Fatalf("(Path).Bytes() = %v", err) + } + truncatedLayerBytes := completeLayerBytes[:512] + + path := l.path("blobs", layerDigest.Algorithm, layerDigest.Hex) + if err := os.WriteFile(path, truncatedLayerBytes, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(layerPath, truncated) = %v", err) + } + + // ensure validation fails + truncatedImg, err := l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading truncated image for validation; (Path).Image = %v", err) + } + if err := validate.Image(truncatedImg); err == nil { + t.Fatal("validating image after truncating layer; validate.Image() = nil, expected err") + } else if strings.Contains(err.Error(), "unexpected EOF") { + t.Fatalf("validating image after truncating layer; validate.Image() error is not helpful: %v", err) + } + + // try writing expected contents with ReplaceImage + if err := l.ReplaceImage(img, match.Digests(imgDigest)); err != nil { + t.Fatalf("error attempting to overwrite truncated layer with valid layer; (Path).ReplaceImage = %v", err) + } + + // validation should now succeed + repairedImg, err := l.Image(imgDigest) + if err != nil { + t.Fatalf("error loading truncated image after ReplaceImage for validation; (Path).Image = %v", err) + } + if err := validate.Image(repairedImg); err != nil { + t.Fatalf("validating image after attempting repair of truncated layer with ReplaceImage; validate.Image() = %v", err) + } +} diff --git a/pkg/v1/manifest.go b/pkg/v1/manifest.go new file mode 100644 index 0000000..22d483f --- /dev/null +++ b/pkg/v1/manifest.go @@ -0,0 +1,71 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "io" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Manifest represents the OCI image manifest in a structured way. +type Manifest struct { + SchemaVersion int64 `json:"schemaVersion"` + MediaType types.MediaType `json:"mediaType,omitempty"` + Config Descriptor `json:"config"` + Layers []Descriptor `json:"layers"` + Annotations map[string]string `json:"annotations,omitempty"` + Subject *Descriptor `json:"subject,omitempty"` +} + +// IndexManifest represents an OCI image index in a structured way. +type IndexManifest struct { + SchemaVersion int64 `json:"schemaVersion"` + MediaType types.MediaType `json:"mediaType,omitempty"` + Manifests []Descriptor `json:"manifests"` + Annotations map[string]string `json:"annotations,omitempty"` + Subject *Descriptor `json:"subject,omitempty"` +} + +// Descriptor holds a reference from the manifest to one of its constituent elements. +type Descriptor struct { + MediaType types.MediaType `json:"mediaType"` + Size int64 `json:"size"` + Digest Hash `json:"digest"` + Data []byte `json:"data,omitempty"` + URLs []string `json:"urls,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Platform *Platform `json:"platform,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` +} + +// ParseManifest parses the io.Reader's contents into a Manifest. +func ParseManifest(r io.Reader) (*Manifest, error) { + m := Manifest{} + if err := json.NewDecoder(r).Decode(&m); err != nil { + return nil, err + } + return &m, nil +} + +// ParseIndexManifest parses the io.Reader's contents into an IndexManifest. +func ParseIndexManifest(r io.Reader) (*IndexManifest, error) { + im := IndexManifest{} + if err := json.NewDecoder(r).Decode(&im); err != nil { + return nil, err + } + return &im, nil +} diff --git a/pkg/v1/manifest_test.go b/pkg/v1/manifest_test.go new file mode 100644 index 0000000..5cd5526 --- /dev/null +++ b/pkg/v1/manifest_test.go @@ -0,0 +1,76 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestGoodManifestSimple(t *testing.T) { + got, err := ParseManifest(strings.NewReader(`{}`)) + if err != nil { + t.Errorf("Unexpected error parsing manifest: %v", err) + } + + want := Manifest{} + if diff := cmp.Diff(want, *got); diff != "" { + t.Errorf("ParseManifest({}); (-want +got) %s", diff) + } +} + +func TestGoodManifestWithHash(t *testing.T) { + good, err := ParseManifest(strings.NewReader(`{ + "config": { + "digest": "sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + } +}`)) + if err != nil { + t.Errorf("Unexpected error parsing manifest: %v", err) + } + + if got, want := good.Config.Digest.Algorithm, "sha256"; got != want { + t.Errorf("ParseManifest().Config.Digest.Algorithm; got %v, want %v", got, want) + } +} + +func TestManifestWithBadHash(t *testing.T) { + bad, err := ParseManifest(strings.NewReader(`{ + "config": { + "digest": "sha256:deadbeed" + } +}`)) + if err == nil { + t.Errorf("Expected error parsing manifest, but got: %v", bad) + } +} + +func TestParseIndexManifest(t *testing.T) { + got, err := ParseIndexManifest(strings.NewReader(`{}`)) + if err != nil { + t.Errorf("Unexpected error parsing manifest: %v", err) + } + + want := IndexManifest{} + if diff := cmp.Diff(want, *got); diff != "" { + t.Errorf("ParseIndexManifest({}); (-want +got) %s", diff) + } + + if got, err := ParseIndexManifest(strings.NewReader("{")); err == nil { + t.Errorf("expected error, got: %v", got) + } +} diff --git a/pkg/v1/match/match.go b/pkg/v1/match/match.go new file mode 100644 index 0000000..98b1ff9 --- /dev/null +++ b/pkg/v1/match/match.go @@ -0,0 +1,92 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package match provides functionality for conveniently matching a v1.Descriptor. +package match + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Matcher function that is given a v1.Descriptor, and returns whether or +// not it matches a given rule. Can match on anything it wants in the Descriptor. +type Matcher func(desc v1.Descriptor) bool + +// Name returns a match.Matcher that matches based on the value of the +// +// "org.opencontainers.image.ref.name" annotation: +// +// github.com/opencontainers/image-spec/blob/v1.0.1/annotations.md#pre-defined-annotation-keys +func Name(name string) Matcher { + return Annotation(imagespec.AnnotationRefName, name) +} + +// Annotation returns a match.Matcher that matches based on the provided annotation. +func Annotation(key, value string) Matcher { + return func(desc v1.Descriptor) bool { + if desc.Annotations == nil { + return false + } + if aValue, ok := desc.Annotations[key]; ok && aValue == value { + return true + } + return false + } +} + +// Platforms returns a match.Matcher that matches on any one of the provided platforms. +// Ignores any descriptors that do not have a platform. +func Platforms(platforms ...v1.Platform) Matcher { + return func(desc v1.Descriptor) bool { + if desc.Platform == nil { + return false + } + for _, platform := range platforms { + if desc.Platform.Equals(platform) { + return true + } + } + return false + } +} + +// MediaTypes returns a match.Matcher that matches at least one of the provided media types. +func MediaTypes(mediaTypes ...string) Matcher { + mts := map[string]bool{} + for _, media := range mediaTypes { + mts[media] = true + } + return func(desc v1.Descriptor) bool { + if desc.MediaType == "" { + return false + } + if _, ok := mts[string(desc.MediaType)]; ok { + return true + } + return false + } +} + +// Digests returns a match.Matcher that matches at least one of the provided Digests +func Digests(digests ...v1.Hash) Matcher { + digs := map[v1.Hash]bool{} + for _, digest := range digests { + digs[digest] = true + } + return func(desc v1.Descriptor) bool { + _, ok := digs[desc.Digest] + return ok + } +} diff --git a/pkg/v1/match/match_test.go b/pkg/v1/match/match_test.go new file mode 100644 index 0000000..c54319d --- /dev/null +++ b/pkg/v1/match/match_test.go @@ -0,0 +1,131 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package match_test + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/types" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestName(t *testing.T) { + tests := []struct { + desc v1.Descriptor + name string + match bool + }{ + {v1.Descriptor{Annotations: map[string]string{imagespec.AnnotationRefName: "foo"}}, "foo", true}, + {v1.Descriptor{Annotations: map[string]string{imagespec.AnnotationRefName: "foo"}}, "bar", false}, + {v1.Descriptor{Annotations: map[string]string{}}, "bar", false}, + {v1.Descriptor{Annotations: nil}, "bar", false}, + {v1.Descriptor{}, "bar", false}, + } + for i, tt := range tests { + f := match.Name(tt.name) + if match := f(tt.desc); match != tt.match { + t.Errorf("%d: mismatched, got %v expected %v for desc %#v name %s", i, match, tt.match, tt.desc, tt.name) + } + } +} + +func TestAnnotation(t *testing.T) { + tests := []struct { + desc v1.Descriptor + key string + value string + match bool + }{ + {v1.Descriptor{Annotations: map[string]string{"foo": "bar"}}, "foo", "bar", true}, + {v1.Descriptor{Annotations: map[string]string{"foo": "bar"}}, "bar", "foo", false}, + {v1.Descriptor{Annotations: map[string]string{}}, "foo", "bar", false}, + {v1.Descriptor{Annotations: nil}, "foo", "bar", false}, + {v1.Descriptor{}, "foo", "bar", false}, + } + for i, tt := range tests { + f := match.Annotation(tt.key, tt.value) + if match := f(tt.desc); match != tt.match { + t.Errorf("%d: mismatched, got %v expected %v for desc %#v annotation %s:%s", i, match, tt.match, tt.desc, tt.key, tt.value) + } + } +} + +func TestPlatforms(t *testing.T) { + tests := []struct { + desc v1.Descriptor + platforms []v1.Platform + match bool + }{ + {v1.Descriptor{Platform: &v1.Platform{Architecture: "amd64", OS: "linux"}}, []v1.Platform{{Architecture: "amd64", OS: "darwin"}, {Architecture: "amd64", OS: "linux"}}, true}, + {v1.Descriptor{Platform: &v1.Platform{Architecture: "amd64", OS: "linux"}}, []v1.Platform{{Architecture: "arm64", OS: "linux"}, {Architecture: "s390x", OS: "linux"}}, false}, + {v1.Descriptor{Platform: &v1.Platform{OS: "linux"}}, []v1.Platform{{Architecture: "arm64", OS: "linux"}}, false}, + {v1.Descriptor{Platform: &v1.Platform{}}, []v1.Platform{{Architecture: "arm64", OS: "linux"}}, false}, + {v1.Descriptor{Platform: nil}, []v1.Platform{{Architecture: "arm64", OS: "linux"}}, false}, + {v1.Descriptor{}, []v1.Platform{{Architecture: "arm64", OS: "linux"}}, false}, + } + for i, tt := range tests { + f := match.Platforms(tt.platforms...) + if match := f(tt.desc); match != tt.match { + t.Errorf("%d: mismatched, got %v expected %v for desc %#v platform %#v", i, match, tt.match, tt.desc, tt.platforms) + } + } +} + +func TestMediaTypes(t *testing.T) { + tests := []struct { + desc v1.Descriptor + mediaTypes []string + match bool + }{ + {v1.Descriptor{MediaType: types.OCIImageIndex}, []string{string(types.OCIImageIndex)}, true}, + {v1.Descriptor{MediaType: types.OCIImageIndex}, []string{string(types.OCIManifestSchema1)}, false}, + {v1.Descriptor{MediaType: types.OCIImageIndex}, []string{string(types.OCIManifestSchema1), string(types.OCIImageIndex)}, true}, + {v1.Descriptor{MediaType: types.OCIImageIndex}, []string{"a", "b"}, false}, + {v1.Descriptor{}, []string{string(types.OCIManifestSchema1), string(types.OCIImageIndex)}, false}, + } + for i, tt := range tests { + f := match.MediaTypes(tt.mediaTypes...) + if match := f(tt.desc); match != tt.match { + t.Errorf("%d: mismatched, got %v expected %v for desc %#v mediaTypes %#v", i, match, tt.match, tt.desc, tt.mediaTypes) + } + } +} + +func TestDigests(t *testing.T) { + hashes := []string{ + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "abcde1111111222f0123456789abcdef0123456789abcdef0123456789abcdef", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + } + algo := "sha256" + + tests := []struct { + desc v1.Descriptor + digests []v1.Hash + match bool + }{ + {v1.Descriptor{Digest: v1.Hash{Algorithm: algo, Hex: hashes[0]}}, []v1.Hash{{Algorithm: algo, Hex: hashes[0]}, {Algorithm: algo, Hex: hashes[1]}}, true}, + {v1.Descriptor{Digest: v1.Hash{Algorithm: algo, Hex: hashes[1]}}, []v1.Hash{{Algorithm: algo, Hex: hashes[0]}, {Algorithm: algo, Hex: hashes[1]}}, true}, + {v1.Descriptor{Digest: v1.Hash{Algorithm: algo, Hex: hashes[2]}}, []v1.Hash{{Algorithm: algo, Hex: hashes[0]}, {Algorithm: algo, Hex: hashes[1]}}, false}, + } + for i, tt := range tests { + f := match.Digests(tt.digests...) + if match := f(tt.desc); match != tt.match { + t.Errorf("%d: mismatched, got %v expected %v for desc %#v digests %#v", i, match, tt.match, tt.desc, tt.digests) + } + } +} diff --git a/pkg/v1/mutate/README.md b/pkg/v1/mutate/README.md new file mode 100644 index 0000000..19e1612 --- /dev/null +++ b/pkg/v1/mutate/README.md @@ -0,0 +1,56 @@ +# `mutate` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/mutate?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/mutate) + +The `v1.Image`, `v1.ImageIndex`, and `v1.Layer` interfaces provide only +accessor methods, so they are essentially immutable. If you want to change +something about them, you need to produce a new instance of that interface. + +A common use case for this library is to read an image from somewhere (a source), +change something about it, and write the image somewhere else (a sink). + +Graphically, this looks something like: + +

+ +

+ +## Mutations + +This is obviously not a comprehensive set of useful transformations (PRs welcome!), +but a rough summary of what the `mutate` package currently does: + +### `Config` and `ConfigFile` + +These allow you to change the [image configuration](https://github.com/opencontainers/image-spec/blob/master/config.md#properties), +e.g. to change the entrypoint, environment, author, etc. + +### `Time`, `Canonical`, and `CreatedAt` + +These are useful in the context of [reproducible builds](https://reproducible-builds.org/), +where you may want to strip timestamps and other non-reproducible information. + +### `Append`, `AppendLayers`, and `AppendManifests` + +These functions allow the extension of a `v1.Image` or `v1.ImageIndex` with +new layers or manifests. + +For constructing an image `FROM scratch`, see the [`empty`](/pkg/v1/empty) package. + +### `MediaType` and `IndexMediaType` + +Sometimes, it is necessary to change the media type of an image or index, +e.g. to appease a registry with strict validation of images (_looking at you, GCR_). + +### `Rebase` + +Rebase has [its own README](/cmd/crane/rebase.md). + +This is the underlying implementation of [`crane rebase`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane_rebase.md). + +### `Extract` + +Extract will flatten an image filesystem into a single tar stream, +respecting whiteout files. + +This is the underlying implementation of [`crane export`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane_export.md). diff --git a/pkg/v1/mutate/doc.go b/pkg/v1/mutate/doc.go new file mode 100644 index 0000000..dfbd995 --- /dev/null +++ b/pkg/v1/mutate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mutate provides facilities for mutating v1.Images of any kind. +package mutate diff --git a/pkg/v1/mutate/image.go b/pkg/v1/mutate/image.go new file mode 100644 index 0000000..727abe2 --- /dev/null +++ b/pkg/v1/mutate/image.go @@ -0,0 +1,287 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "bytes" + "encoding/json" + "errors" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type image struct { + base v1.Image + adds []Addendum + + computed bool + configFile *v1.ConfigFile + manifest *v1.Manifest + annotations map[string]string + mediaType *types.MediaType + configMediaType *types.MediaType + diffIDMap map[v1.Hash]v1.Layer + digestMap map[v1.Hash]v1.Layer + subject *v1.Descriptor +} + +var _ v1.Image = (*image)(nil) + +func (i *image) MediaType() (types.MediaType, error) { + if i.mediaType != nil { + return *i.mediaType, nil + } + return i.base.MediaType() +} + +func (i *image) compute() error { + // Don't re-compute if already computed. + if i.computed { + return nil + } + var configFile *v1.ConfigFile + if i.configFile != nil { + configFile = i.configFile + } else { + cf, err := i.base.ConfigFile() + if err != nil { + return err + } + configFile = cf.DeepCopy() + } + diffIDs := configFile.RootFS.DiffIDs + history := configFile.History + + diffIDMap := make(map[v1.Hash]v1.Layer) + digestMap := make(map[v1.Hash]v1.Layer) + + for _, add := range i.adds { + history = append(history, add.History) + if add.Layer != nil { + diffID, err := add.Layer.DiffID() + if err != nil { + return err + } + diffIDs = append(diffIDs, diffID) + diffIDMap[diffID] = add.Layer + } + } + + m, err := i.base.Manifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + manifestLayers := manifest.Layers + for _, add := range i.adds { + if add.Layer == nil { + // Empty layers include only history in manifest. + continue + } + + desc, err := partial.Descriptor(add.Layer) + if err != nil { + return err + } + + // Fields in the addendum override the original descriptor. + if len(add.Annotations) != 0 { + desc.Annotations = add.Annotations + } + if len(add.URLs) != 0 { + desc.URLs = add.URLs + } + + if add.MediaType != "" { + desc.MediaType = add.MediaType + } + + manifestLayers = append(manifestLayers, *desc) + digestMap[desc.Digest] = add.Layer + } + + configFile.RootFS.DiffIDs = diffIDs + configFile.History = history + + manifest.Layers = manifestLayers + + rcfg, err := json.Marshal(configFile) + if err != nil { + return err + } + d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) + if err != nil { + return err + } + manifest.Config.Digest = d + manifest.Config.Size = sz + + // If Data was set in the base image, we need to update it in the mutated image. + if m.Config.Data != nil { + manifest.Config.Data = rcfg + } + + // If the user wants to mutate the media type of the config + if i.configMediaType != nil { + manifest.Config.MediaType = *i.configMediaType + } + + if i.mediaType != nil { + manifest.MediaType = *i.mediaType + } + + if i.annotations != nil { + if manifest.Annotations == nil { + manifest.Annotations = map[string]string{} + } + + for k, v := range i.annotations { + manifest.Annotations[k] = v + } + } + manifest.Subject = i.subject + + i.configFile = configFile + i.manifest = manifest + i.diffIDMap = diffIDMap + i.digestMap = digestMap + i.computed = true + return nil +} + +// Layers returns the ordered collection of filesystem layers that comprise this image. +// The order of the list is oldest/base layer first, and most-recent/top layer last. +func (i *image) Layers() ([]v1.Layer, error) { + if err := i.compute(); errors.Is(err, stream.ErrNotComputed) { + // Image contains a streamable layer which has not yet been + // consumed. Just return the layers we have in case the caller + // is going to consume the layers. + layers, err := i.base.Layers() + if err != nil { + return nil, err + } + for _, add := range i.adds { + layers = append(layers, add.Layer) + } + return layers, nil + } else if err != nil { + return nil, err + } + + diffIDs, err := partial.DiffIDs(i) + if err != nil { + return nil, err + } + ls := make([]v1.Layer, 0, len(diffIDs)) + for _, h := range diffIDs { + l, err := i.LayerByDiffID(h) + if err != nil { + return nil, err + } + ls = append(ls, l) + } + return ls, nil +} + +// ConfigName returns the hash of the image's config file. +func (i *image) ConfigName() (v1.Hash, error) { + if err := i.compute(); err != nil { + return v1.Hash{}, err + } + return partial.ConfigName(i) +} + +// ConfigFile returns this image's config file. +func (i *image) ConfigFile() (*v1.ConfigFile, error) { + if err := i.compute(); err != nil { + return nil, err + } + return i.configFile.DeepCopy(), nil +} + +// RawConfigFile returns the serialized bytes of ConfigFile() +func (i *image) RawConfigFile() ([]byte, error) { + if err := i.compute(); err != nil { + return nil, err + } + return json.Marshal(i.configFile) +} + +// Digest returns the sha256 of this image's manifest. +func (i *image) Digest() (v1.Hash, error) { + if err := i.compute(); err != nil { + return v1.Hash{}, err + } + return partial.Digest(i) +} + +// Size implements v1.Image. +func (i *image) Size() (int64, error) { + if err := i.compute(); err != nil { + return -1, err + } + return partial.Size(i) +} + +// Manifest returns this image's Manifest object. +func (i *image) Manifest() (*v1.Manifest, error) { + if err := i.compute(); err != nil { + return nil, err + } + return i.manifest.DeepCopy(), nil +} + +// RawManifest returns the serialized bytes of Manifest() +func (i *image) RawManifest() ([]byte, error) { + if err := i.compute(); err != nil { + return nil, err + } + return json.Marshal(i.manifest) +} + +// LayerByDigest returns a Layer for interacting with a particular layer of +// the image, looking it up by "digest" (the compressed hash). +func (i *image) LayerByDigest(h v1.Hash) (v1.Layer, error) { + if cn, err := i.ConfigName(); err != nil { + return nil, err + } else if h == cn { + return partial.ConfigLayer(i) + } + if layer, ok := i.digestMap[h]; ok { + return layer, nil + } + return i.base.LayerByDigest(h) +} + +// LayerByDiffID is an analog to LayerByDigest, looking up by "diff id" +// (the uncompressed hash). +func (i *image) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.diffIDMap[h]; ok { + return layer, nil + } + return i.base.LayerByDiffID(h) +} + +func validate(adds []Addendum) error { + for _, add := range adds { + if add.Layer == nil && !add.History.EmptyLayer { + return errors.New("unable to add a nil layer to the image") + } + } + return nil +} diff --git a/pkg/v1/mutate/index.go b/pkg/v1/mutate/index.go new file mode 100644 index 0000000..ba062f9 --- /dev/null +++ b/pkg/v1/mutate/index.go @@ -0,0 +1,204 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "encoding/json" + "fmt" + + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) { + desc, err := partial.Descriptor(ia.Add) + if err != nil { + return nil, err + } + + // The IndexAddendum allows overriding Descriptor values. + if ia.Descriptor.Size != 0 { + desc.Size = ia.Descriptor.Size + } + if string(ia.Descriptor.MediaType) != "" { + desc.MediaType = ia.Descriptor.MediaType + } + if ia.Descriptor.Digest != (v1.Hash{}) { + desc.Digest = ia.Descriptor.Digest + } + if ia.Descriptor.Platform != nil { + desc.Platform = ia.Descriptor.Platform + } + if len(ia.Descriptor.URLs) != 0 { + desc.URLs = ia.Descriptor.URLs + } + if len(ia.Descriptor.Annotations) != 0 { + desc.Annotations = ia.Descriptor.Annotations + } + if ia.Descriptor.Data != nil { + desc.Data = ia.Descriptor.Data + } + + return desc, nil +} + +type index struct { + base v1.ImageIndex + adds []IndexAddendum + // remove is removed before adds + remove match.Matcher + + computed bool + manifest *v1.IndexManifest + annotations map[string]string + mediaType *types.MediaType + imageMap map[v1.Hash]v1.Image + indexMap map[v1.Hash]v1.ImageIndex + layerMap map[v1.Hash]v1.Layer + subject *v1.Descriptor +} + +var _ v1.ImageIndex = (*index)(nil) + +func (i *index) MediaType() (types.MediaType, error) { + if i.mediaType != nil { + return *i.mediaType, nil + } + return i.base.MediaType() +} + +func (i *index) Size() (int64, error) { return partial.Size(i) } + +func (i *index) compute() error { + // Don't re-compute if already computed. + if i.computed { + return nil + } + + i.imageMap = make(map[v1.Hash]v1.Image) + i.indexMap = make(map[v1.Hash]v1.ImageIndex) + i.layerMap = make(map[v1.Hash]v1.Layer) + + m, err := i.base.IndexManifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + manifests := manifest.Manifests + + if i.remove != nil { + var cleanedManifests []v1.Descriptor + for _, m := range manifests { + if !i.remove(m) { + cleanedManifests = append(cleanedManifests, m) + } + } + manifests = cleanedManifests + } + + for _, add := range i.adds { + desc, err := computeDescriptor(add) + if err != nil { + return err + } + + manifests = append(manifests, *desc) + if idx, ok := add.Add.(v1.ImageIndex); ok { + i.indexMap[desc.Digest] = idx + } else if img, ok := add.Add.(v1.Image); ok { + i.imageMap[desc.Digest] = img + } else if l, ok := add.Add.(v1.Layer); ok { + i.layerMap[desc.Digest] = l + } else { + logs.Warn.Printf("Unexpected index addendum: %T", add.Add) + } + } + + manifest.Manifests = manifests + + if i.mediaType != nil { + manifest.MediaType = *i.mediaType + } + + if i.annotations != nil { + if manifest.Annotations == nil { + manifest.Annotations = map[string]string{} + } + for k, v := range i.annotations { + manifest.Annotations[k] = v + } + } + manifest.Subject = i.subject + + i.manifest = manifest + i.computed = true + return nil +} + +func (i *index) Image(h v1.Hash) (v1.Image, error) { + if img, ok := i.imageMap[h]; ok { + return img, nil + } + return i.base.Image(h) +} + +func (i *index) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + if idx, ok := i.indexMap[h]; ok { + return idx, nil + } + return i.base.ImageIndex(h) +} + +type withLayer interface { + Layer(v1.Hash) (v1.Layer, error) +} + +// Workaround for #819. +func (i *index) Layer(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.layerMap[h]; ok { + return layer, nil + } + if wl, ok := i.base.(withLayer); ok { + return wl.Layer(h) + } + return nil, fmt.Errorf("layer not found: %s", h) +} + +// Digest returns the sha256 of this image's manifest. +func (i *index) Digest() (v1.Hash, error) { + if err := i.compute(); err != nil { + return v1.Hash{}, err + } + return partial.Digest(i) +} + +// Manifest returns this image's Manifest object. +func (i *index) IndexManifest() (*v1.IndexManifest, error) { + if err := i.compute(); err != nil { + return nil, err + } + return i.manifest.DeepCopy(), nil +} + +// RawManifest returns the serialized bytes of Manifest() +func (i *index) RawManifest() ([]byte, error) { + if err := i.compute(); err != nil { + return nil, err + } + return json.Marshal(i.manifest) +} diff --git a/pkg/v1/mutate/index_test.go b/pkg/v1/mutate/index_test.go new file mode 100644 index 0000000..1f542d1 --- /dev/null +++ b/pkg/v1/mutate/index_test.go @@ -0,0 +1,235 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate_test + +import ( + "log" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestAppendIndex(t *testing.T) { + base, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + idx, err := random.Index(2048, 1, 2) + if err != nil { + t.Fatal(err) + } + img, err := random.Image(4096, 5) + if err != nil { + t.Fatal(err) + } + l, err := random.Layer(1024, types.OCIUncompressedRestrictedLayer) + if err != nil { + t.Fatal(err) + } + + weirdHash := v1.Hash{ + Algorithm: "sha256", + Hex: strings.Repeat("0", 64), + } + + add := mutate.AppendManifests(base, mutate.IndexAddendum{ + Add: idx, + Descriptor: v1.Descriptor{ + URLs: []string{"index.example.com"}, + }, + }, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{ + URLs: []string{"image.example.com"}, + }, + }, mutate.IndexAddendum{ + Add: l, + Descriptor: v1.Descriptor{ + MediaType: types.MediaType("application/xml"), + URLs: []string{"blob.example.com"}, + }, + }, mutate.IndexAddendum{ + Add: l, + Descriptor: v1.Descriptor{ + URLs: []string{"layer.example.com"}, + Size: 1337, + Digest: weirdHash, + Platform: &v1.Platform{ + OS: "haiku", + Architecture: "toaster", + }, + Annotations: map[string]string{"weird": "true"}, + }, + }) + + if err := validate.Index(add); err != nil { + t.Errorf("Validate() = %v", err) + } + + got, err := add.MediaType() + if err != nil { + t.Fatal(err) + } + want, err := base.MediaType() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("MediaType() = %s != %s", got, want) + } + + // TODO(jonjohnsonjr): There's no way to grab layers from v1.ImageIndex. + m, err := add.IndexManifest() + if err != nil { + log.Fatal(err) + } + + for i, want := range map[int]string{ + 3: "index.example.com", + 4: "image.example.com", + 5: "blob.example.com", + 6: "layer.example.com", + } { + if got := m.Manifests[i].URLs[0]; got != want { + t.Errorf("wrong URLs[0] for Manifests[%d]: %s != %s", i, got, want) + } + } + + if got, want := m.Manifests[5].MediaType, types.MediaType("application/xml"); got != want { + t.Errorf("wrong MediaType for layer: %s != %s", got, want) + } + + if got, want := m.Manifests[6].MediaType, types.OCIUncompressedRestrictedLayer; got != want { + t.Errorf("wrong MediaType for layer: %s != %s", got, want) + } + + // Append the index to itself and make sure it still validates. + add = mutate.AppendManifests(add, mutate.IndexAddendum{ + Add: add, + }) + if err := validate.Index(add); err != nil { + t.Errorf("Validate() = %v", err) + } + + // Wrap the whole thing in another index and make sure it still validates. + add = mutate.AppendManifests(empty.Index, mutate.IndexAddendum{ + Add: add, + }) + if err := validate.Index(add); err != nil { + t.Errorf("Validate() = %v", err) + } +} + +func TestIndexImmutability(t *testing.T) { + base, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatal(err) + } + ii, err := random.Index(2048, 1, 2) + if err != nil { + t.Fatal(err) + } + i, err := random.Image(4096, 5) + if err != nil { + t.Fatal(err) + } + idx := mutate.AppendManifests(base, mutate.IndexAddendum{ + Add: ii, + Descriptor: v1.Descriptor{ + URLs: []string{"index.example.com"}, + }, + }, mutate.IndexAddendum{ + Add: i, + Descriptor: v1.Descriptor{ + URLs: []string{"image.example.com"}, + }, + }) + + t.Run("index manifest", func(t *testing.T) { + // Check that Manifest is immutable. + changed, err := idx.IndexManifest() + if err != nil { + t.Errorf("IndexManifest() = %v", err) + } + want := changed.DeepCopy() // Create a copy of original before mutating it. + changed.MediaType = types.DockerManifestList + + if got, err := idx.IndexManifest(); err != nil { + t.Errorf("IndexManifest() = %v", err) + } else if !cmp.Equal(got, want) { + t.Errorf("IndexManifest changed! %s", cmp.Diff(got, want)) + } + }) +} + +// TestAppend_ArtifactType tests that appending an image manifest that has a +// non-standard config.mediaType to an index, results in the image's +// config.mediaType being hoisted into the descriptor inside the index, +// as artifactType. +func TestAppend_ArtifactType(t *testing.T) { + for _, c := range []struct { + desc, configMediaType, wantArtifactType string + }{{ + desc: "standard config.mediaType, no artifactType", + configMediaType: string(types.DockerConfigJSON), + wantArtifactType: "", + }, { + desc: "non-standard config.mediaType, want artifactType", + configMediaType: "application/vnd.custom.something", + wantArtifactType: "application/vnd.custom.something", + }} { + t.Run(c.desc, func(t *testing.T) { + img, err := random.Image(1, 1) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + img = mutate.ConfigMediaType(img, types.MediaType(c.configMediaType)) + idx := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{ + Add: img, + }) + mf, err := idx.IndexManifest() + if err != nil { + t.Fatalf("IndexManifest: %v", err) + } + if got := mf.Manifests[0].ArtifactType; got != c.wantArtifactType { + t.Errorf("manifest artifactType: got %q, want %q", got, c.wantArtifactType) + } + + desc, err := partial.Descriptor(img) + if err != nil { + t.Fatalf("partial.Descriptor: %v", err) + } + if got := desc.ArtifactType; got != c.wantArtifactType { + t.Errorf("descriptor artifactType: got %q, want %q", got, c.wantArtifactType) + } + + gotAT, err := partial.ArtifactType(img) + if err != nil { + t.Fatalf("partial.ArtifactType: %v", err) + } + if gotAT != c.wantArtifactType { + t.Errorf("partial.ArtifactType: got %q, want %q", gotAT, c.wantArtifactType) + } + }) + } +} diff --git a/pkg/v1/mutate/mutate.go b/pkg/v1/mutate/mutate.go new file mode 100644 index 0000000..e4a0e52 --- /dev/null +++ b/pkg/v1/mutate/mutate.go @@ -0,0 +1,553 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/google/go-containerregistry/internal/gzip" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +const whiteoutPrefix = ".wh." + +// Addendum contains layers and history to be appended +// to a base image +type Addendum struct { + Layer v1.Layer + History v1.History + URLs []string + Annotations map[string]string + MediaType types.MediaType +} + +// AppendLayers applies layers to a base image. +func AppendLayers(base v1.Image, layers ...v1.Layer) (v1.Image, error) { + additions := make([]Addendum, 0, len(layers)) + for _, layer := range layers { + additions = append(additions, Addendum{Layer: layer}) + } + + return Append(base, additions...) +} + +// Append will apply the list of addendums to the base image +func Append(base v1.Image, adds ...Addendum) (v1.Image, error) { + if len(adds) == 0 { + return base, nil + } + if err := validate(adds); err != nil { + return nil, err + } + + return &image{ + base: base, + adds: adds, + }, nil +} + +// Appendable is an interface that represents something that can be appended +// to an ImageIndex. We need to be able to construct a v1.Descriptor in order +// to append something, and this is the minimum required information for that. +type Appendable interface { + MediaType() (types.MediaType, error) + Digest() (v1.Hash, error) + Size() (int64, error) +} + +// IndexAddendum represents an appendable thing and all the properties that +// we may want to override in the resulting v1.Descriptor. +type IndexAddendum struct { + Add Appendable + v1.Descriptor +} + +// AppendManifests appends a manifest to the ImageIndex. +func AppendManifests(base v1.ImageIndex, adds ...IndexAddendum) v1.ImageIndex { + return &index{ + base: base, + adds: adds, + } +} + +// RemoveManifests removes any descriptors that match the match.Matcher. +func RemoveManifests(base v1.ImageIndex, matcher match.Matcher) v1.ImageIndex { + return &index{ + base: base, + remove: matcher, + } +} + +// Config mutates the provided v1.Image to have the provided v1.Config +func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { + cf, err := base.ConfigFile() + if err != nil { + return nil, err + } + + cf.Config = cfg + + return ConfigFile(base, cf) +} + +// Subject mutates the subject on an image or index manifest. +// +// The input is expected to be a v1.Image or v1.ImageIndex, and +// returns the same type. You can type-assert the result like so: +// +// img := Subject(empty.Image, subj).(v1.Image) +// +// Or for an index: +// +// idx := Subject(empty.Index, subj).(v1.ImageIndex) +// +// If the input is not an Image or ImageIndex, the result will +// attempt to lazily annotate the raw manifest. +func Subject(f partial.WithRawManifest, subject v1.Descriptor) partial.WithRawManifest { + if img, ok := f.(v1.Image); ok { + return &image{ + base: img, + subject: &subject, + } + } + if idx, ok := f.(v1.ImageIndex); ok { + return &index{ + base: idx, + subject: &subject, + } + } + return arbitraryRawManifest{a: f, subject: &subject} +} + +// Annotations mutates the annotations on an annotatable image or index manifest. +// +// The annotatable input is expected to be a v1.Image or v1.ImageIndex, and +// returns the same type. You can type-assert the result like so: +// +// img := Annotations(empty.Image, map[string]string{ +// "foo": "bar", +// }).(v1.Image) +// +// Or for an index: +// +// idx := Annotations(empty.Index, map[string]string{ +// "foo": "bar", +// }).(v1.ImageIndex) +// +// If the input Annotatable is not an Image or ImageIndex, the result will +// attempt to lazily annotate the raw manifest. +func Annotations(f partial.WithRawManifest, anns map[string]string) partial.WithRawManifest { + if img, ok := f.(v1.Image); ok { + return &image{ + base: img, + annotations: anns, + } + } + if idx, ok := f.(v1.ImageIndex); ok { + return &index{ + base: idx, + annotations: anns, + } + } + return arbitraryRawManifest{a: f, anns: anns} +} + +type arbitraryRawManifest struct { + a partial.WithRawManifest + anns map[string]string + subject *v1.Descriptor +} + +func (a arbitraryRawManifest) RawManifest() ([]byte, error) { + b, err := a.a.RawManifest() + if err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + if ann, ok := m["annotations"]; ok { + if annm, ok := ann.(map[string]string); ok { + for k, v := range a.anns { + annm[k] = v + } + } else { + return nil, fmt.Errorf(".annotations is not a map: %T", ann) + } + } else { + m["annotations"] = a.anns + } + if a.subject != nil { + m["subject"] = a.subject + } + return json.Marshal(m) +} + +// ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile +func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { + m, err := base.Manifest() + if err != nil { + return nil, err + } + + image := &image{ + base: base, + manifest: m.DeepCopy(), + configFile: cfg, + } + + return image, nil +} + +// CreatedAt mutates the provided v1.Image to have the provided v1.Time +func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { + cf, err := base.ConfigFile() + if err != nil { + return nil, err + } + + cfg := cf.DeepCopy() + cfg.Created = created + + return ConfigFile(base, cfg) +} + +// Extract takes an image and returns an io.ReadCloser containing the image's +// flattened filesystem. +// +// Callers can read the filesystem contents by passing the reader to +// tar.NewReader, or io.Copy it directly to some output. +// +// If a caller doesn't read the full contents, they should Close it to free up +// resources used during extraction. +func Extract(img v1.Image) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + // Close the writer with any errors encountered during + // extraction. These errors will be returned by the reader end + // on subsequent reads. If err == nil, the reader will return + // EOF. + pw.CloseWithError(extract(img, pw)) + }() + + return pr +} + +// Adapted from https://github.com/google/containerregistry/blob/da03b395ccdc4e149e34fbb540483efce962dc64/client/v2_2/docker_image_.py#L816 +func extract(img v1.Image, w io.Writer) error { + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + fileMap := map[string]bool{} + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("retrieving image layers: %w", err) + } + + // we iterate through the layers in reverse order because it makes handling + // whiteout layers more efficient, since we can just keep track of the removed + // files as we see .wh. layers and ignore those in previous layers. + for i := len(layers) - 1; i >= 0; i-- { + layer := layers[i] + layerReader, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("reading layer contents: %w", err) + } + defer layerReader.Close() + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading tar: %w", err) + } + + // Some tools prepend everything with "./", so if we don't Clean the + // name, we may have duplicate entries, which angers tar-split. + header.Name = filepath.Clean(header.Name) + // force PAX format to remove Name/Linkname length limit of 100 characters + // required by USTAR and to not depend on internal tar package guess which + // prefers USTAR over PAX + header.Format = tar.FormatPAX + + basename := filepath.Base(header.Name) + dirname := filepath.Dir(header.Name) + tombstone := strings.HasPrefix(basename, whiteoutPrefix) + if tombstone { + basename = basename[len(whiteoutPrefix):] + } + + // check if we have seen value before + // if we're checking a directory, don't filepath.Join names + var name string + if header.Typeflag == tar.TypeDir { + name = header.Name + } else { + name = filepath.Join(dirname, basename) + } + + if _, ok := fileMap[name]; ok { + continue + } + + // check for a whited out parent directory + if inWhiteoutDir(fileMap, name) { + continue + } + + // mark file as handled. non-directory implicitly tombstones + // any entries with a matching (or child) name + fileMap[name] = tombstone || !(header.Typeflag == tar.TypeDir) + if !tombstone { + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if header.Size > 0 { + if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { + return err + } + } + } + } + } + return nil +} + +func inWhiteoutDir(fileMap map[string]bool, file string) bool { + for { + if file == "" { + break + } + dirname := filepath.Dir(file) + if file == dirname { + break + } + if val, ok := fileMap[dirname]; ok && val { + return true + } + file = dirname + } + return false +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +// Time sets all timestamps in an image to the given timestamp. +func Time(img v1.Image, t time.Time) (v1.Image, error) { + newImage := empty.Image + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("getting image layers: %w", err) + } + + ocf, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("getting original config file: %w", err) + } + + addendums := make([]Addendum, max(len(ocf.History), len(layers))) + var historyIdx, addendumIdx int + for layerIdx := 0; layerIdx < len(layers); addendumIdx, layerIdx = addendumIdx+1, layerIdx+1 { + newLayer, err := layerTime(layers[layerIdx], t) + if err != nil { + return nil, fmt.Errorf("setting layer times: %w", err) + } + + // try to search for the history entry that corresponds to this layer + for ; historyIdx < len(ocf.History); historyIdx++ { + addendums[addendumIdx].History = ocf.History[historyIdx] + // if it's an EmptyLayer, do not set the Layer and have the Addendum with just the History + // and move on to the next History entry + if ocf.History[historyIdx].EmptyLayer { + addendumIdx++ + continue + } + // otherwise, we can exit from the cycle + historyIdx++ + break + } + addendums[addendumIdx].Layer = newLayer + } + + // add all leftover History entries + for ; historyIdx < len(ocf.History); historyIdx, addendumIdx = historyIdx+1, addendumIdx+1 { + addendums[addendumIdx].History = ocf.History[historyIdx] + } + + newImage, err = Append(newImage, addendums...) + if err != nil { + return nil, fmt.Errorf("appending layers: %w", err) + } + + cf, err := newImage.ConfigFile() + if err != nil { + return nil, fmt.Errorf("setting config file: %w", err) + } + + cfg := cf.DeepCopy() + + // Copy basic config over + cfg.Architecture = ocf.Architecture + cfg.OS = ocf.OS + cfg.OSVersion = ocf.OSVersion + cfg.Config = ocf.Config + + // Strip away timestamps from the config file + cfg.Created = v1.Time{Time: t} + + for i, h := range cfg.History { + h.Created = v1.Time{Time: t} + h.CreatedBy = ocf.History[i].CreatedBy + h.Comment = ocf.History[i].Comment + h.EmptyLayer = ocf.History[i].EmptyLayer + // Explicitly ignore Author field; which hinders reproducibility + h.Author = "" + cfg.History[i] = h + } + + return ConfigFile(newImage, cfg) +} + +func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { + layerReader, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("getting layer: %w", err) + } + defer layerReader.Close() + w := new(bytes.Buffer) + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("reading layer: %w", err) + } + + header.ModTime = t + + //PAX and GNU Format support additional timestamps in the header + if header.Format == tar.FormatPAX || header.Format == tar.FormatGNU { + header.AccessTime = t + header.ChangeTime = t + } + + if err := tarWriter.WriteHeader(header); err != nil { + return nil, fmt.Errorf("writing tar header: %w", err) + } + + if header.Typeflag == tar.TypeReg { + // TODO(#1168): This should be lazy, and not buffer the entire layer contents. + if _, err = io.CopyN(tarWriter, tarReader, header.Size); err != nil { + return nil, fmt.Errorf("writing layer file: %w", err) + } + } + } + + if err := tarWriter.Close(); err != nil { + return nil, err + } + + b := w.Bytes() + // gzip the contents, then create the layer + opener := func() (io.ReadCloser, error) { + return gzip.ReadCloser(io.NopCloser(bytes.NewReader(b))), nil + } + layer, err = tarball.LayerFromOpener(opener) + if err != nil { + return nil, fmt.Errorf("creating layer: %w", err) + } + + return layer, nil +} + +// Canonical is a helper function to combine Time and configFile +// to remove any randomness during a docker build. +func Canonical(img v1.Image) (v1.Image, error) { + // Set all timestamps to 0 + created := time.Time{} + img, err := Time(img, created) + if err != nil { + return nil, err + } + + cf, err := img.ConfigFile() + if err != nil { + return nil, err + } + + // Get rid of host-dependent random config + cfg := cf.DeepCopy() + + cfg.Container = "" + cfg.Config.Hostname = "" + cfg.DockerVersion = "" + + return ConfigFile(img, cfg) +} + +// MediaType modifies the MediaType() of the given image. +func MediaType(img v1.Image, mt types.MediaType) v1.Image { + return &image{ + base: img, + mediaType: &mt, + } +} + +// ConfigMediaType modifies the MediaType() of the given image's Config. +// +// If !mt.IsConfig(), this will be the image's artifactType in any indexes it's a part of. +func ConfigMediaType(img v1.Image, mt types.MediaType) v1.Image { + return &image{ + base: img, + configMediaType: &mt, + } +} + +// IndexMediaType modifies the MediaType() of the given index. +func IndexMediaType(idx v1.ImageIndex, mt types.MediaType) v1.ImageIndex { + return &index{ + base: idx, + mediaType: &mt, + } +} diff --git a/pkg/v1/mutate/mutate_test.go b/pkg/v1/mutate/mutate_test.go new file mode 100644 index 0000000..c4fdba6 --- /dev/null +++ b/pkg/v1/mutate/mutate_test.go @@ -0,0 +1,770 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate_test + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestExtractWhiteout(t *testing.T) { + img, err := tarball.ImageFromPath("testdata/whiteout_image.tar", nil) + if err != nil { + t.Errorf("Error loading image: %v", err) + } + tarPath, _ := filepath.Abs("img.tar") + defer os.Remove(tarPath) + tr := tar.NewReader(mutate.Extract(img)) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + name := header.Name + for _, part := range filepath.SplitList(name) { + if part == "foo" { + t.Errorf("whiteout file found in tar: %v", name) + } + } + } +} + +func TestExtractOverwrittenFile(t *testing.T) { + img, err := tarball.ImageFromPath("testdata/overwritten_file.tar", nil) + if err != nil { + t.Fatalf("Error loading image: %v", err) + } + tr := tar.NewReader(mutate.Extract(img)) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + name := header.Name + if strings.Contains(name, "foo.txt") { + var buf bytes.Buffer + buf.ReadFrom(tr) + if strings.Contains(buf.String(), "foo") { + t.Errorf("Contents of file were not correctly overwritten") + } + } + } +} + +// TestExtractError tests that if there are any errors encountered +func TestExtractError(t *testing.T) { + rc := mutate.Extract(invalidImage{}) + if _, err := io.Copy(io.Discard, rc); err == nil { + t.Errorf("rc.Read; got nil error") + } else if !strings.Contains(err.Error(), errInvalidImage.Error()) { + t.Errorf("rc.Read; got %v, want %v", err, errInvalidImage) + } +} + +// TestExtractPartialRead tests that the reader can be partially read (e.g., +// tar headers) and closed without error. +func TestExtractPartialRead(t *testing.T) { + rc := mutate.Extract(invalidImage{}) + if _, err := io.Copy(io.Discard, io.LimitReader(rc, 1)); err != nil { + t.Errorf("Could not read one byte from reader") + } + if err := rc.Close(); err != nil { + t.Errorf("rc.Close: %v", err) + } +} + +// invalidImage is an image which returns an error when Layers() is called. +type invalidImage struct { + v1.Image +} + +var errInvalidImage = errors.New("invalid image") + +func (invalidImage) Layers() ([]v1.Layer, error) { + return nil, errInvalidImage +} + +func TestNoopCondition(t *testing.T) { + source := sourceImage(t) + + result, err := mutate.AppendLayers(source, []v1.Layer{}...) + if err != nil { + t.Fatalf("Unexpected error creating a writable image: %v", err) + } + + if !manifestsAreEqual(t, source, result) { + t.Error("manifests are not the same") + } + + if !configFilesAreEqual(t, source, result) { + t.Fatal("config files are not the same") + } +} + +func TestAppendWithAddendum(t *testing.T) { + source := sourceImage(t) + + addendum := mutate.Addendum{ + Layer: mockLayer{}, + History: v1.History{ + Author: "dave", + }, + URLs: []string{ + "example.com", + }, + Annotations: map[string]string{ + "foo": "bar", + }, + MediaType: types.MediaType("foo"), + } + + result, err := mutate.Append(source, addendum) + if err != nil { + t.Fatalf("failed to append: %v", err) + } + + layers := getLayers(t, result) + + if diff := cmp.Diff(layers[1], mockLayer{}); diff != "" { + t.Fatalf("correct layer was not appended (-got, +want) %v", diff) + } + + if configSizesAreEqual(t, source, result) { + t.Fatal("adding a layer MUST change the config file size") + } + + cf := getConfigFile(t, result) + + if diff := cmp.Diff(cf.History[1], addendum.History); diff != "" { + t.Fatalf("the appended history is not the same (-got, +want) %s", diff) + } + + m, err := result.Manifest() + if err != nil { + t.Fatalf("failed to get manifest: %v", err) + } + + if diff := cmp.Diff(m.Layers[1].URLs, addendum.URLs); diff != "" { + t.Fatalf("the appended URLs is not the same (-got, +want) %s", diff) + } + + if diff := cmp.Diff(m.Layers[1].Annotations, addendum.Annotations); diff != "" { + t.Fatalf("the appended Annotations is not the same (-got, +want) %s", diff) + } + if diff := cmp.Diff(m.Layers[1].MediaType, addendum.MediaType); diff != "" { + t.Fatalf("the appended MediaType is not the same (-got, +want) %s", diff) + } +} + +func TestAppendLayers(t *testing.T) { + source := sourceImage(t) + layer, err := random.Layer(100, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + result, err := mutate.AppendLayers(source, layer) + if err != nil { + t.Fatalf("failed to append a layer: %v", err) + } + + if manifestsAreEqual(t, source, result) { + t.Fatal("appending a layer did not mutate the manifest") + } + + if configFilesAreEqual(t, source, result) { + t.Fatal("appending a layer did not mutate the config file") + } + + if configSizesAreEqual(t, source, result) { + t.Fatal("adding a layer MUST change the config file size") + } + + layers := getLayers(t, result) + + if got, want := len(layers), 2; got != want { + t.Fatalf("Layers did not return the appended layer "+ + "- got size %d; expected 2", len(layers)) + } + + if layers[1] != layer { + t.Errorf("correct layer was not appended: got %v; want %v", layers[1], layer) + } + + if err := validate.Image(result); err != nil { + t.Errorf("validate.Image() = %v", err) + } +} + +func TestMutateConfig(t *testing.T) { + source := sourceImage(t) + cfg, err := source.ConfigFile() + if err != nil { + t.Fatalf("error getting source config file") + } + + newEnv := []string{"foo=bar"} + cfg.Config.Env = newEnv + result, err := mutate.Config(source, cfg.Config) + if err != nil { + t.Fatalf("failed to mutate a config: %v", err) + } + + if manifestsAreEqual(t, source, result) { + t.Error("mutating the config MUST mutate the manifest") + } + + if configFilesAreEqual(t, source, result) { + t.Error("mutating the config did not mutate the config file") + } + + if configSizesAreEqual(t, source, result) { + t.Error("adding an environment variable MUST change the config file size") + } + + if configDigestsAreEqual(t, source, result) { + t.Errorf("mutating the config MUST mutate the config digest") + } + + if !reflect.DeepEqual(cfg.Config.Env, newEnv) { + t.Errorf("incorrect environment set %v!=%v", cfg.Config.Env, newEnv) + } + + if err := validate.Image(result); err != nil { + t.Errorf("validate.Image() = %v", err) + } +} + +type arbitrary struct { +} + +func (arbitrary) RawManifest() ([]byte, error) { + return []byte(`{"hello":"world"}`), nil +} +func TestAnnotations(t *testing.T) { + anns := map[string]string{ + "foo": "bar", + } + + for _, c := range []struct { + desc string + in partial.WithRawManifest + want string + }{{ + desc: "image", + in: empty.Image, + want: `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":115,"digest":"sha256:5b943e2b943f6c81dbbd4e2eca5121f4fcc39139e3d1219d6d89bd925b77d9fe"},"layers":[],"annotations":{"foo":"bar"}}`, + }, { + desc: "index", + in: empty.Index, + want: `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":null,"annotations":{"foo":"bar"}}`, + }, { + desc: "arbitrary", + in: arbitrary{}, + want: `{"annotations":{"foo":"bar"},"hello":"world"}`, + }} { + t.Run(c.desc, func(t *testing.T) { + got, err := mutate.Annotations(c.in, anns).RawManifest() + if err != nil { + t.Fatalf("Annotations: %v", err) + } + if d := cmp.Diff(c.want, string(got)); d != "" { + t.Errorf("Diff(-want,+got): %s", d) + } + }) + } +} + +func TestMutateCreatedAt(t *testing.T) { + source := sourceImage(t) + want := time.Now().Add(-2 * time.Minute) + result, err := mutate.CreatedAt(source, v1.Time{Time: want}) + if err != nil { + t.Fatalf("CreatedAt: %v", err) + } + + if configDigestsAreEqual(t, source, result) { + t.Errorf("mutating the created time MUST mutate the config digest") + } + + got := getConfigFile(t, result).Created.Time + if got != want { + t.Errorf("mutating the created time MUST mutate the time from %v to %v", got, want) + } +} + +func TestMutateTime(t *testing.T) { + for _, tc := range []struct { + name string + source v1.Image + }{ + { + name: "image with matching history and layers", + source: sourceImage(t), + }, + { + name: "image with empty_layer history entries", + source: sourceImagePath(t, "testdata/source_image_with_empty_layer_history.tar"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + want := time.Time{} + result, err := mutate.Time(tc.source, want) + if err != nil { + t.Fatalf("failed to mutate a config: %v", err) + } + + if configDigestsAreEqual(t, tc.source, result) { + t.Fatal("mutating the created time MUST mutate the config digest") + } + + mutatedOriginalConfig := getConfigFile(t, tc.source).DeepCopy() + gotConfig := getConfigFile(t, result) + + // manually change the fields we expect to be changed by mutate.Time + mutatedOriginalConfig.Author = "" + mutatedOriginalConfig.Created = v1.Time{Time: want} + for i := range mutatedOriginalConfig.History { + mutatedOriginalConfig.History[i].Created = v1.Time{Time: want} + mutatedOriginalConfig.History[i].Author = "" + } + + if diff := cmp.Diff(mutatedOriginalConfig, gotConfig, + cmpopts.IgnoreFields(v1.RootFS{}, "DiffIDs"), + ); diff != "" { + t.Errorf("configFile() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestMutateMediaType(t *testing.T) { + want := types.OCIManifestSchema1 + wantCfg := types.OCIConfigJSON + img := mutate.MediaType(empty.Image, want) + img = mutate.ConfigMediaType(img, wantCfg) + got, err := img.MediaType() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Errorf("%q != %q", want, got) + } + manifest, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + if manifest.MediaType == "" { + t.Error("MediaType should be set for OCI media types") + } + if gotCfg := manifest.Config.MediaType; gotCfg != wantCfg { + t.Errorf("manifest.Config.MediaType = %v, wanted %v", gotCfg, wantCfg) + } + + want = types.DockerManifestSchema2 + wantCfg = types.DockerConfigJSON + img = mutate.MediaType(img, want) + img = mutate.ConfigMediaType(img, wantCfg) + got, err = img.MediaType() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Errorf("%q != %q", want, got) + } + manifest, err = img.Manifest() + if err != nil { + t.Fatal(err) + } + if manifest.MediaType != want { + t.Errorf("MediaType should be set for Docker media types: %v", manifest.MediaType) + } + if gotCfg := manifest.Config.MediaType; gotCfg != wantCfg { + t.Errorf("manifest.Config.MediaType = %v, wanted %v", gotCfg, wantCfg) + } + + want = types.OCIImageIndex + idx := mutate.IndexMediaType(empty.Index, want) + got, err = idx.MediaType() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Errorf("%q != %q", want, got) + } + im, err := idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + if im.MediaType == "" { + t.Error("MediaType should be set for OCI media types") + } + + want = types.DockerManifestList + idx = mutate.IndexMediaType(idx, want) + got, err = idx.MediaType() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Errorf("%q != %q", want, got) + } + im, err = idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + if im.MediaType != want { + t.Errorf("MediaType should be set for Docker media types: %v", im.MediaType) + } +} + +func TestAppendStreamableLayer(t *testing.T) { + img, err := mutate.AppendLayers( + sourceImage(t), + stream.NewLayer(io.NopCloser(strings.NewReader(strings.Repeat("a", 100)))), + stream.NewLayer(io.NopCloser(strings.NewReader(strings.Repeat("b", 100)))), + stream.NewLayer(io.NopCloser(strings.NewReader(strings.Repeat("c", 100)))), + ) + if err != nil { + t.Fatalf("AppendLayers: %v", err) + } + + // Until the streams are consumed, the image manifest is not yet computed. + if _, err := img.Manifest(); !errors.Is(err, stream.ErrNotComputed) { + t.Errorf("Manifest: got %v, want %v", err, stream.ErrNotComputed) + } + + // We can still get Layers while some are not yet computed. + ls, err := img.Layers() + if err != nil { + t.Errorf("Layers: %v", err) + } + wantDigests := []string{ + "sha256:bfa1c600931132f55789459e2f5a5eb85659ac91bc5a54ce09e3ed14809f8a7f", + "sha256:77a52b9a141dcc4d3d277d053193765dca725626f50eaf56b903ac2439cf7fd1", + "sha256:b78472d63f6e3d31059819173b56fcb0d9479a2b13c097d4addd84889f6aff06", + } + for i, l := range ls[1:] { + rc, err := l.Compressed() + if err != nil { + t.Errorf("Layer %d Compressed: %v", i, err) + } + + // Consume the layer's stream and close it to compute the + // layer's metadata. + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Errorf("Reading layer %d: %v", i, err) + } + if err := rc.Close(); err != nil { + t.Errorf("Closing layer %d: %v", i, err) + } + + // The layer's metadata is now available. + h, err := l.Digest() + if err != nil { + t.Errorf("Digest after consuming layer %d: %v", i, err) + } + if h.String() != wantDigests[i] { + t.Errorf("Layer %d digest got %q, want %q", i, h, wantDigests[i]) + } + } + + // Now that the streamable layers have been consumed, the image's + // manifest can be computed. + if _, err := img.Manifest(); err != nil { + t.Errorf("Manifest: %v", err) + } + + h, err := img.Digest() + if err != nil { + t.Errorf("Digest: %v", err) + } + wantDigest := "sha256:14d140947afedc6901b490265a08bc8ebe7f9d9faed6fdf19a451f054a7dd746" + if h.String() != wantDigest { + t.Errorf("Image digest got %q, want %q", h, wantDigest) + } +} + +func TestCanonical(t *testing.T) { + source := sourceImage(t) + img, err := mutate.Canonical(source) + if err != nil { + t.Fatal(err) + } + sourceCf, err := source.ConfigFile() + if err != nil { + t.Fatal(err) + } + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + for _, h := range cf.History { + want := "bazel build ..." + got := h.CreatedBy + if want != got { + t.Errorf("%q != %q", want, got) + } + } + var want, got string + want = cf.Architecture + got = sourceCf.Architecture + if want != got { + t.Errorf("%q != %q", want, got) + } + want = cf.OS + got = sourceCf.OS + if want != got { + t.Errorf("%q != %q", want, got) + } + want = cf.OSVersion + got = sourceCf.OSVersion + if want != got { + t.Errorf("%q != %q", want, got) + } + for _, s := range []string{ + cf.Container, + cf.Config.Hostname, + cf.DockerVersion, + } { + if s != "" { + t.Errorf("non-zeroed string: %v", s) + } + } + + expectedLayerTime := time.Unix(0, 0) + layers := getLayers(t, img) + for _, layer := range layers { + assertMTime(t, layer, expectedLayerTime) + } +} + +func TestRemoveManifests(t *testing.T) { + // Load up the registry. + count := 3 + for i := 0; i < count; i++ { + ii, err := random.Index(1024, int64(count), int64(count)) + if err != nil { + t.Fatal(err) + } + // test removing the first layer, second layer or the third layer + manifest, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(manifest.Manifests) != count { + t.Fatalf("mismatched manifests on setup, had %d, expected %d", len(manifest.Manifests), count) + } + digest := manifest.Manifests[i].Digest + ii = mutate.RemoveManifests(ii, match.Digests(digest)) + manifest, err = ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(manifest.Manifests) != (count - 1) { + t.Fatalf("mismatched manifests after removal, had %d, expected %d", len(manifest.Manifests), count-1) + } + for j, m := range manifest.Manifests { + if m.Digest == digest { + t.Fatalf("unexpectedly found removed hash %v at position %d", digest, j) + } + } + } +} + +func TestImageImmutability(t *testing.T) { + img := mutate.MediaType(empty.Image, types.OCIManifestSchema1) + + t.Run("manifest", func(t *testing.T) { + // Check that Manifest is immutable. + changed, err := img.Manifest() + if err != nil { + t.Errorf("Manifest() = %v", err) + } + want := changed.DeepCopy() // Create a copy of original before mutating it. + changed.MediaType = types.DockerManifestList + + if got, err := img.Manifest(); err != nil { + t.Errorf("Manifest() = %v", err) + } else if !cmp.Equal(got, want) { + t.Errorf("manifest changed! %s", cmp.Diff(got, want)) + } + }) + + t.Run("config file", func(t *testing.T) { + // Check that ConfigFile is immutable. + changed, err := img.ConfigFile() + if err != nil { + t.Errorf("ConfigFile() = %v", err) + } + want := changed.DeepCopy() // Create a copy of original before mutating it. + changed.Author = "Jay Pegg" + + if got, err := img.ConfigFile(); err != nil { + t.Errorf("ConfigFile() = %v", err) + } else if !cmp.Equal(got, want) { + t.Errorf("ConfigFile changed! %s", cmp.Diff(got, want)) + } + }) +} + +func assertMTime(t *testing.T, layer v1.Layer, expectedTime time.Time) { + l, err := layer.Uncompressed() + + if err != nil { + t.Fatalf("reading layer failed: %v", err) + } + + tr := tar.NewReader(l) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("Error reading layer: %v", err) + } + + mtime := header.ModTime + if mtime.Equal(expectedTime) == false { + t.Errorf("unexpected mod time for layer. expected %v, got %v.", expectedTime, mtime) + } + } +} + +func sourceImage(t *testing.T) v1.Image { + return sourceImagePath(t, "testdata/source_image.tar") +} + +func sourceImagePath(t *testing.T, tarPath string) v1.Image { + t.Helper() + + image, err := tarball.ImageFromPath(tarPath, nil) + if err != nil { + t.Fatalf("Error loading image: %v", err) + } + return image +} + +func getManifest(t *testing.T, i v1.Image) *v1.Manifest { + t.Helper() + + m, err := i.Manifest() + if err != nil { + t.Fatalf("Error fetching image manifest: %v", err) + } + + return m +} + +func getLayers(t *testing.T, i v1.Image) []v1.Layer { + t.Helper() + + l, err := i.Layers() + if err != nil { + t.Fatalf("Error fetching image layers: %v", err) + } + + return l +} + +func getConfigFile(t *testing.T, i v1.Image) *v1.ConfigFile { + t.Helper() + + c, err := i.ConfigFile() + if err != nil { + t.Fatalf("Error fetching image config file: %v", err) + } + + return c +} + +func configFilesAreEqual(t *testing.T, first, second v1.Image) bool { + t.Helper() + + fc := getConfigFile(t, first) + sc := getConfigFile(t, second) + + return cmp.Equal(fc, sc) +} + +func configDigestsAreEqual(t *testing.T, first, second v1.Image) bool { + t.Helper() + + fm := getManifest(t, first) + sm := getManifest(t, second) + + return fm.Config.Digest == sm.Config.Digest +} + +func configSizesAreEqual(t *testing.T, first, second v1.Image) bool { + t.Helper() + + fm := getManifest(t, first) + sm := getManifest(t, second) + + return fm.Config.Size == sm.Config.Size +} + +func manifestsAreEqual(t *testing.T, first, second v1.Image) bool { + t.Helper() + + fm := getManifest(t, first) + sm := getManifest(t, second) + + return cmp.Equal(fm, sm) +} + +type mockLayer struct{} + +func (m mockLayer) Digest() (v1.Hash, error) { + return v1.Hash{Algorithm: "fake", Hex: "digest"}, nil +} + +func (m mockLayer) DiffID() (v1.Hash, error) { + return v1.Hash{Algorithm: "fake", Hex: "diff id"}, nil +} + +func (m mockLayer) MediaType() (types.MediaType, error) { + return "some-media-type", nil +} + +func (m mockLayer) Size() (int64, error) { return 137438691328, nil } +func (m mockLayer) Compressed() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("compressed times")), nil +} +func (m mockLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("uncompressed")), nil +} diff --git a/pkg/v1/mutate/rebase.go b/pkg/v1/mutate/rebase.go new file mode 100644 index 0000000..c606e0b --- /dev/null +++ b/pkg/v1/mutate/rebase.go @@ -0,0 +1,144 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" +) + +// Rebase returns a new v1.Image where the oldBase in orig is replaced by newBase. +func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) { + // Verify that oldBase's layers are present in orig, otherwise orig is + // not based on oldBase at all. + origLayers, err := orig.Layers() + if err != nil { + return nil, fmt.Errorf("failed to get layers for original: %w", err) + } + oldBaseLayers, err := oldBase.Layers() + if err != nil { + return nil, err + } + if len(oldBaseLayers) > len(origLayers) { + return nil, fmt.Errorf("image %q is not based on %q (too few layers)", orig, oldBase) + } + for i, l := range oldBaseLayers { + oldLayerDigest, err := l.Digest() + if err != nil { + return nil, fmt.Errorf("failed to get digest of layer %d of %q: %w", i, oldBase, err) + } + origLayerDigest, err := origLayers[i].Digest() + if err != nil { + return nil, fmt.Errorf("failed to get digest of layer %d of %q: %w", i, orig, err) + } + if oldLayerDigest != origLayerDigest { + return nil, fmt.Errorf("image %q is not based on %q (layer %d mismatch)", orig, oldBase, i) + } + } + + oldConfig, err := oldBase.ConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to get config for old base: %w", err) + } + + origConfig, err := orig.ConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to get config for original: %w", err) + } + + newConfig, err := newBase.ConfigFile() + if err != nil { + return nil, fmt.Errorf("could not get config for new base: %w", err) + } + + // Stitch together an image that contains: + // - original image's config + // - new base image's os/arch properties + // - new base image's layers + top of original image's layers + // - new base image's history + top of original image's history + rebasedImage, err := Config(empty.Image, *origConfig.Config.DeepCopy()) + if err != nil { + return nil, fmt.Errorf("failed to create empty image with original config: %w", err) + } + + // Add new config properties from existing images. + rebasedConfig, err := rebasedImage.ConfigFile() + if err != nil { + return nil, fmt.Errorf("could not get config for rebased image: %w", err) + } + // OS/Arch properties from new base + rebasedConfig.Architecture = newConfig.Architecture + rebasedConfig.OS = newConfig.OS + rebasedConfig.OSVersion = newConfig.OSVersion + + // Apply config properties to rebased. + rebasedImage, err = ConfigFile(rebasedImage, rebasedConfig) + if err != nil { + return nil, fmt.Errorf("failed to replace config for rebased image: %w", err) + } + + // Get new base layers and config for history. + newBaseLayers, err := newBase.Layers() + if err != nil { + return nil, fmt.Errorf("could not get new base layers for new base: %w", err) + } + // Add new base layers. + rebasedImage, err = Append(rebasedImage, createAddendums(0, 0, newConfig.History, newBaseLayers)...) + if err != nil { + return nil, fmt.Errorf("failed to append new base image: %w", err) + } + + // Add original layers above the old base. + rebasedImage, err = Append(rebasedImage, createAddendums(len(oldConfig.History), len(oldBaseLayers)+1, origConfig.History, origLayers)...) + if err != nil { + return nil, fmt.Errorf("failed to append original image: %w", err) + } + + return rebasedImage, nil +} + +// createAddendums makes a list of addendums from a history and layers starting from a specific history and layer +// indexes. +func createAddendums(startHistory, startLayer int, history []v1.History, layers []v1.Layer) []Addendum { + var adds []Addendum + // History should be a superset of layers; empty layers (e.g. ENV statements) only exist in history. + // They cannot be iterated identically but must be walked independently, only advancing the iterator for layers + // when a history entry for a non-empty layer is seen. + layerIndex := 0 + for historyIndex := range history { + var layer v1.Layer + emptyLayer := history[historyIndex].EmptyLayer + if !emptyLayer { + layer = layers[layerIndex] + layerIndex++ + } + if historyIndex >= startHistory || layerIndex >= startLayer { + adds = append(adds, Addendum{ + Layer: layer, + History: history[historyIndex], + }) + } + } + // In the event history was malformed or non-existent, append the remaining layers. + for i := layerIndex; i < len(layers); i++ { + if i >= startLayer { + adds = append(adds, Addendum{Layer: layers[layerIndex]}) + } + } + + return adds +} diff --git a/pkg/v1/mutate/rebase_test.go b/pkg/v1/mutate/rebase_test.go new file mode 100644 index 0000000..250b6bf --- /dev/null +++ b/pkg/v1/mutate/rebase_test.go @@ -0,0 +1,179 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate_test + +import ( + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func layerDigests(t *testing.T, img v1.Image) []string { + layers, err := img.Layers() + if err != nil { + t.Fatalf("oldBase.Layers: %v", err) + } + layerDigests := make([]string, len(layers)) + for i, l := range layers { + dig, err := l.Digest() + if err != nil { + t.Fatalf("layer.Digest %d: %v", i, err) + } + t.Log(dig) + layerDigests[i] = dig.String() + } + return layerDigests +} + +// TestRebase tests that layer digests are expected when performing a rebase on +// random.Image layers. +func TestRebase(t *testing.T) { + // Create a random old base image of 5 layers and get those layers' digests. + const oldBaseLayerCount = 5 + oldBase, err := random.Image(100, oldBaseLayerCount) + if err != nil { + t.Fatalf("random.Image (oldBase): %v", err) + } + t.Log("Old base:") + _ = layerDigests(t, oldBase) + + // Construct an image with 2 layers on top of oldBase (an empty layer and a random layer). + top, err := random.Image(100, 1) + if err != nil { + t.Fatalf("random.Image (top): %v", err) + } + topLayers, err := top.Layers() + if err != nil { + t.Fatalf("top.Layers: %v", err) + } + orig, err := mutate.Append(oldBase, + mutate.Addendum{ + Layer: nil, + History: v1.History{ + Author: "me", + Created: v1.Time{Time: time.Now()}, + CreatedBy: "test-empty", + Comment: "this is an empty test", + EmptyLayer: true, + }, + }, + mutate.Addendum{ + Layer: topLayers[0], + History: v1.History{ + Author: "me", + Created: v1.Time{Time: time.Now()}, + CreatedBy: "test", + Comment: "this is a test", + }, + }, + ) + if err != nil { + t.Fatalf("Append: %v", err) + } + + t.Log("Original:") + origLayerDigests := layerDigests(t, orig) + + // Create a random new base image of 3 layers. + newBase, err := random.Image(100, 3) + if err != nil { + t.Fatalf("random.Image (newBase): %v", err) + } + t.Log("New base:") + newBaseLayerDigests := layerDigests(t, newBase) + + // Add config file os/arch property fields + newBaseConfigFile, err := newBase.ConfigFile() + if err != nil { + t.Fatalf("newBase.ConfigFile: %v", err) + } + newBaseConfigFile.Architecture = "arm" + newBaseConfigFile.OS = "windows" + newBaseConfigFile.OSVersion = "10.0.17763.1339" + + newBase, err = mutate.ConfigFile(newBase, newBaseConfigFile) + if err != nil { + t.Fatalf("ConfigFile (newBase): %v", err) + } + + // Rebase original image onto new base. + rebased, err := mutate.Rebase(orig, oldBase, newBase) + if err != nil { + t.Fatalf("Rebase: %v", err) + } + + rebasedBaseLayers, err := rebased.Layers() + if err != nil { + t.Fatalf("rebased.Layers: %v", err) + } + rebasedLayerDigests := make([]string, len(rebasedBaseLayers)) + t.Log("Rebased image layer digests:") + for i, l := range rebasedBaseLayers { + dig, err := l.Digest() + if err != nil { + t.Fatalf("layer.Digest (rebased base layer %d): %v", i, err) + } + t.Log(dig) + rebasedLayerDigests[i] = dig.String() + } + + // Compare rebased layers. + wantLayerDigests := append(newBaseLayerDigests, origLayerDigests[len(origLayerDigests)-1]) + if len(rebasedLayerDigests) != len(wantLayerDigests) { + t.Fatalf("Rebased image contained %d layers, want %d", len(rebasedLayerDigests), len(wantLayerDigests)) + } + for i, rl := range rebasedLayerDigests { + if got, want := rl, wantLayerDigests[i]; got != want { + t.Errorf("Layer %d mismatch, got %q, want %q", i, got, want) + } + } + + // Compare rebased history. + origConfig, err := orig.ConfigFile() + if err != nil { + t.Fatalf("orig.ConfigFile: %v", err) + } + newBaseConfig, err := newBase.ConfigFile() + if err != nil { + t.Fatalf("newBase.ConfigFile: %v", err) + } + rebasedConfig, err := rebased.ConfigFile() + if err != nil { + t.Fatalf("rebased.ConfigFile: %v", err) + } + wantHistories := append(newBaseConfig.History, origConfig.History[oldBaseLayerCount:]...) + if len(wantHistories) != len(rebasedConfig.History) { + t.Fatalf("Rebased image contained %d history, want %d", len(rebasedConfig.History), len(wantHistories)) + } + for i, rh := range rebasedConfig.History { + if got, want := rh.Comment, wantHistories[i].Comment; got != want { + t.Errorf("Layer %d mismatch, got %q, want %q", i, got, want) + } + } + + // Compare ConfigFile property fields copied from new base. + if rebasedConfig.Architecture != newBaseConfig.Architecture { + t.Errorf("ConfigFile property Architecture mismatch, got %q, want %q", rebasedConfig.Architecture, newBaseConfig.Architecture) + } + if rebasedConfig.OS != newBaseConfig.OS { + t.Errorf("ConfigFile property OS mismatch, got %q, want %q", rebasedConfig.OS, newBaseConfig.OS) + } + if rebasedConfig.OSVersion != newBaseConfig.OSVersion { + t.Errorf("ConfigFile property OSVersion mismatch, got %q, want %q", rebasedConfig.OSVersion, newBaseConfig.OSVersion) + } +} diff --git a/pkg/v1/mutate/testdata/README.md b/pkg/v1/mutate/testdata/README.md new file mode 100644 index 0000000..a35d433 --- /dev/null +++ b/pkg/v1/mutate/testdata/README.md @@ -0,0 +1,10 @@ +# whiteout\_image.tar + +Including whiteout files in our source caused [issues](https://github.com/google/go-containerregistry/issues/305) +when cloning this repo inside a docker build. Removing the whiteout file from +this test data doesn't break anything (since we checked in the tar), but if you +want to rebuild it for some reason: + +``` +touch whiteout/.wh.foo.txt +``` diff --git a/pkg/v1/mutate/testdata/bar b/pkg/v1/mutate/testdata/bar new file mode 100644 index 0000000..5716ca5 --- /dev/null +++ b/pkg/v1/mutate/testdata/bar @@ -0,0 +1 @@ +bar diff --git a/pkg/v1/mutate/testdata/foo b/pkg/v1/mutate/testdata/foo new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/pkg/v1/mutate/testdata/foo @@ -0,0 +1 @@ +foo diff --git a/pkg/v1/mutate/testdata/overwritten_file.tar b/pkg/v1/mutate/testdata/overwritten_file.tar new file mode 100755 index 0000000..7159556 Binary files /dev/null and b/pkg/v1/mutate/testdata/overwritten_file.tar differ diff --git a/pkg/v1/mutate/testdata/source_image.tar b/pkg/v1/mutate/testdata/source_image.tar new file mode 100755 index 0000000..7824a7b Binary files /dev/null and b/pkg/v1/mutate/testdata/source_image.tar differ diff --git a/pkg/v1/mutate/testdata/source_image_with_empty_layer_history.tar b/pkg/v1/mutate/testdata/source_image_with_empty_layer_history.tar new file mode 100755 index 0000000..541cb37 Binary files /dev/null and b/pkg/v1/mutate/testdata/source_image_with_empty_layer_history.tar differ diff --git a/pkg/v1/mutate/testdata/whiteout/bar.txt b/pkg/v1/mutate/testdata/whiteout/bar.txt new file mode 100644 index 0000000..5716ca5 --- /dev/null +++ b/pkg/v1/mutate/testdata/whiteout/bar.txt @@ -0,0 +1 @@ +bar diff --git a/pkg/v1/mutate/testdata/whiteout/foo.txt b/pkg/v1/mutate/testdata/whiteout/foo.txt new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/pkg/v1/mutate/testdata/whiteout/foo.txt @@ -0,0 +1 @@ +foo diff --git a/pkg/v1/mutate/testdata/whiteout_image.tar b/pkg/v1/mutate/testdata/whiteout_image.tar new file mode 100755 index 0000000..748621e Binary files /dev/null and b/pkg/v1/mutate/testdata/whiteout_image.tar differ diff --git a/pkg/v1/mutate/whiteout_test.go b/pkg/v1/mutate/whiteout_test.go new file mode 100644 index 0000000..d3e7a86 --- /dev/null +++ b/pkg/v1/mutate/whiteout_test.go @@ -0,0 +1,43 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "testing" +) + +func TestWhiteoutDir(t *testing.T) { + fsMap := map[string]bool{ + "baz": true, + "red/blue": true, + } + var tests = []struct { + path string + whiteout bool + }{ + {"usr/bin", false}, + {"baz/foo.txt", true}, + {"baz/bar/foo.txt", true}, + {"red/green", false}, + {"red/yellow.txt", false}, + } + + for _, tt := range tests { + whiteout := inWhiteoutDir(fsMap, tt.path) + if whiteout != tt.whiteout { + t.Errorf("Whiteout %s: expected %v, but got %v", tt.path, tt.whiteout, whiteout) + } + } +} diff --git a/pkg/v1/partial/README.md b/pkg/v1/partial/README.md new file mode 100644 index 0000000..53ebbc6 --- /dev/null +++ b/pkg/v1/partial/README.md @@ -0,0 +1,82 @@ +# `partial` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial) + +## Partial Implementations + +There are roughly two kinds of image representations: compressed and uncompressed. + +The implementations for these kinds of images are almost identical, with the only +major difference being how blobs (config and layers) are fetched. This common +code lives in this package, where you provide a _partial_ implementation of a +compressed or uncompressed image, and you get back a full `v1.Image` implementation. + +### Examples + +In a registry, blobs are compressed, so it's easiest to implement a `v1.Image` in terms +of compressed layers. `remote.remoteImage` does this by implementing `CompressedImageCore`: + +```go +type CompressedImageCore interface { + RawConfigFile() ([]byte, error) + MediaType() (types.MediaType, error) + RawManifest() ([]byte, error) + LayerByDigest(v1.Hash) (CompressedLayer, error) +} +``` + +In a tarball, blobs are (often) uncompressed, so it's easiest to implement a `v1.Image` in terms +of uncompressed layers. `tarball.uncompressedImage` does this by implementing `UncompressedImageCore`: + +```go +type UncompressedImageCore interface { + RawConfigFile() ([]byte, error) + MediaType() (types.MediaType, error) + LayerByDiffID(v1.Hash) (UncompressedLayer, error) +} +``` + +## Optional Methods + +Where possible, we access some information via optional methods as an optimization. + +### [`partial.Descriptor`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Descriptor) + +There are some properties of a [`Descriptor`](https://github.com/opencontainers/image-spec/blob/master/descriptor.md#properties) that aren't derivable from just image data: + +* `MediaType` +* `Platform` +* `URLs` +* `Annotations` + +For example, in a `tarball.Image`, there is a `LayerSources` field that contains +an entire layer descriptor with `URLs` information for foreign layers. This +information can be passed through to callers by implementing this optional +`Descriptor` method. + +See [`#654`](https://github.com/google/go-containerregistry/pull/654). + +### [`partial.UncompressedSize`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#UncompressedSize) + +Usually, you don't need to know the uncompressed size of a layer, since that +information isn't stored in a config file (just he sha256 is needed); however, +there are cases where it is very helpful to know the layer size, e.g. when +writing the uncompressed layer into a tarball. + +See [`#655`](https://github.com/google/go-containerregistry/pull/655). + +### [`partial.Exists`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Exists) + +We generally don't care about the existence of something as granular as a +layer, and would rather ensure all the invariants of an image are upheld via +the `validate` package. However, there are situations where we want to do a +quick smoke test to ensure that the underlying storage engine hasn't been +corrupted by something e.g. deleting files or blobs. Thus, we've exposed an +optional `Exists` method that does an existence check without actually reading +any bytes. + +The `remote` package implements this via `HEAD` requests. + +The `layout` package implements this via `os.Stat`. + +See [`#838`](https://github.com/google/go-containerregistry/pull/838). diff --git a/pkg/v1/partial/compressed.go b/pkg/v1/partial/compressed.go new file mode 100644 index 0000000..44989ac --- /dev/null +++ b/pkg/v1/partial/compressed.go @@ -0,0 +1,188 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "io" + + "github.com/google/go-containerregistry/internal/and" + "github.com/google/go-containerregistry/internal/compression" + "github.com/google/go-containerregistry/internal/gzip" + "github.com/google/go-containerregistry/internal/zstd" + comp "github.com/google/go-containerregistry/pkg/compression" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// CompressedLayer represents the bare minimum interface a natively +// compressed layer must implement for us to produce a v1.Layer +type CompressedLayer interface { + // Digest returns the Hash of the compressed layer. + Digest() (v1.Hash, error) + + // Compressed returns an io.ReadCloser for the compressed layer contents. + Compressed() (io.ReadCloser, error) + + // Size returns the compressed size of the Layer. + Size() (int64, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) +} + +// compressedLayerExtender implements v1.Image using the compressed base properties. +type compressedLayerExtender struct { + CompressedLayer +} + +// Uncompressed implements v1.Layer +func (cle *compressedLayerExtender) Uncompressed() (io.ReadCloser, error) { + rc, err := cle.Compressed() + if err != nil { + return nil, err + } + + // Often, the "compressed" bytes are not actually-compressed. + // Peek at the first two bytes to determine whether it's correct to + // wrap this with gzip.UnzipReadCloser or zstd.UnzipReadCloser. + cp, pr, err := compression.PeekCompression(rc) + if err != nil { + return nil, err + } + + prc := &and.ReadCloser{ + Reader: pr, + CloseFunc: rc.Close, + } + + switch cp { + case comp.GZip: + return gzip.UnzipReadCloser(prc) + case comp.ZStd: + return zstd.UnzipReadCloser(prc) + default: + return prc, nil + } +} + +// DiffID implements v1.Layer +func (cle *compressedLayerExtender) DiffID() (v1.Hash, error) { + // If our nested CompressedLayer implements DiffID, + // then delegate to it instead. + if wdi, ok := cle.CompressedLayer.(WithDiffID); ok { + return wdi.DiffID() + } + r, err := cle.Uncompressed() + if err != nil { + return v1.Hash{}, err + } + defer r.Close() + h, _, err := v1.SHA256(r) + return h, err +} + +// CompressedToLayer fills in the missing methods from a CompressedLayer so that it implements v1.Layer +func CompressedToLayer(ul CompressedLayer) (v1.Layer, error) { + return &compressedLayerExtender{ul}, nil +} + +// CompressedImageCore represents the base minimum interface a natively +// compressed image must implement for us to produce a v1.Image. +type CompressedImageCore interface { + ImageCore + + // RawManifest returns the serialized bytes of the manifest. + RawManifest() ([]byte, error) + + // LayerByDigest is a variation on the v1.Image method, which returns + // a CompressedLayer instead. + LayerByDigest(v1.Hash) (CompressedLayer, error) +} + +// compressedImageExtender implements v1.Image by extending CompressedImageCore with the +// appropriate methods computed from the minimal core. +type compressedImageExtender struct { + CompressedImageCore +} + +// Assert that our extender type completes the v1.Image interface +var _ v1.Image = (*compressedImageExtender)(nil) + +// Digest implements v1.Image +func (i *compressedImageExtender) Digest() (v1.Hash, error) { + return Digest(i) +} + +// ConfigName implements v1.Image +func (i *compressedImageExtender) ConfigName() (v1.Hash, error) { + return ConfigName(i) +} + +// Layers implements v1.Image +func (i *compressedImageExtender) Layers() ([]v1.Layer, error) { + hs, err := FSLayers(i) + if err != nil { + return nil, err + } + ls := make([]v1.Layer, 0, len(hs)) + for _, h := range hs { + l, err := i.LayerByDigest(h) + if err != nil { + return nil, err + } + ls = append(ls, l) + } + return ls, nil +} + +// LayerByDigest implements v1.Image +func (i *compressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) { + cl, err := i.CompressedImageCore.LayerByDigest(h) + if err != nil { + return nil, err + } + return CompressedToLayer(cl) +} + +// LayerByDiffID implements v1.Image +func (i *compressedImageExtender) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + h, err := DiffIDToBlob(i, h) + if err != nil { + return nil, err + } + return i.LayerByDigest(h) +} + +// ConfigFile implements v1.Image +func (i *compressedImageExtender) ConfigFile() (*v1.ConfigFile, error) { + return ConfigFile(i) +} + +// Manifest implements v1.Image +func (i *compressedImageExtender) Manifest() (*v1.Manifest, error) { + return Manifest(i) +} + +// Size implements v1.Image +func (i *compressedImageExtender) Size() (int64, error) { + return Size(i) +} + +// CompressedToImage fills in the missing methods from a CompressedImageCore so that it implements v1.Image +func CompressedToImage(cic CompressedImageCore) (v1.Image, error) { + return &compressedImageExtender{ + CompressedImageCore: cic, + }, nil +} diff --git a/pkg/v1/partial/compressed_test.go b/pkg/v1/partial/compressed_test.go new file mode 100644 index 0000000..bf6bff4 --- /dev/null +++ b/pkg/v1/partial/compressed_test.go @@ -0,0 +1,193 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial_test + +import ( + "io" + "net/http/httptest" + "net/url" + "path" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +// Remote leverages a lot of compressed partials. +func TestRemote(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + rnd, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + + src := path.Join(u.Host, "test/compressed") + ref, err := name.ParseReference(src) + if err != nil { + t.Fatal(err) + } + if err := remote.Write(ref, rnd); err != nil { + t.Fatal(err) + } + + img, err := remote.Image(ref) + if err != nil { + t.Fatal(err) + } + if err := validate.Image(img); err != nil { + t.Fatal(err) + } + + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + m, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + layer, err := img.LayerByDiffID(cf.RootFS.DiffIDs[0]) + if err != nil { + t.Fatal(err) + } + d, err := layer.Digest() + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(d, m.Layers[0].Digest); diff != "" { + t.Errorf("mismatched digest: %v", diff) + } + + ok, err := partial.Exists(layer) + if err != nil { + t.Fatal(err) + } + if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } + + cl, err := partial.ConfigLayer(img) + if err != nil { + t.Fatal(err) + } + if _, ok := cl.(*remote.MountableLayer); !ok { + t.Errorf("ConfigLayer() expected to be MountableLayer, got %T", cl) + } +} + +type noDiffID struct { + l v1.Layer +} + +func (l *noDiffID) Digest() (v1.Hash, error) { + return l.l.Digest() +} +func (l *noDiffID) Compressed() (io.ReadCloser, error) { + return l.l.Compressed() +} +func (l *noDiffID) Size() (int64, error) { + return l.l.Size() +} +func (l *noDiffID) MediaType() (types.MediaType, error) { + return l.l.MediaType() +} +func (l *noDiffID) Descriptor() (*v1.Descriptor, error) { + return partial.Descriptor(l.l) +} +func (l *noDiffID) UncompressedSize() (int64, error) { + return partial.UncompressedSize(l.l) +} + +func TestCompressedLayerExtender(t *testing.T) { + rnd, err := random.Layer(1000, types.OCILayer) + if err != nil { + t.Fatal(err) + } + l, err := partial.CompressedToLayer(&noDiffID{rnd}) + if err != nil { + t.Fatal(err) + } + + if err := compare.Layers(rnd, l); err != nil { + t.Fatalf("compare.Layers: %v", err) + } + if _, err := partial.Descriptor(l); err != nil { + t.Fatalf("partial.Descriptor: %v", err) + } + if _, err := partial.UncompressedSize(l); err != nil { + t.Fatalf("partial.UncompressedSize: %v", err) + } +} + +type compressedImage struct { + img v1.Image +} + +func (i *compressedImage) RawConfigFile() ([]byte, error) { + return i.img.RawConfigFile() +} + +func (i *compressedImage) MediaType() (types.MediaType, error) { + return i.img.MediaType() +} + +func (i *compressedImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + return i.img.LayerByDigest(h) +} + +func (i *compressedImage) RawManifest() ([]byte, error) { + return i.img.RawManifest() +} + +func (i *compressedImage) Descriptor() (*v1.Descriptor, error) { + return partial.Descriptor(i.img) +} + +func TestCompressed(t *testing.T) { + rnd, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + core := &compressedImage{rnd} + + img, err := partial.CompressedToImage(core) + if err != nil { + t.Fatal(err) + } + + if err := validate.Image(img); err != nil { + t.Fatalf("validate.Image: %v", err) + } + if _, err := partial.Descriptor(img); err != nil { + t.Fatalf("partial.Descriptor: %v", err) + } +} diff --git a/pkg/v1/partial/configlayer_test.go b/pkg/v1/partial/configlayer_test.go new file mode 100644 index 0000000..decdcab --- /dev/null +++ b/pkg/v1/partial/configlayer_test.go @@ -0,0 +1,139 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "fmt" + "io" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type testUIC struct { + UncompressedImageCore + configFile []byte +} + +func (t testUIC) RawConfigFile() ([]byte, error) { + return t.configFile, nil +} + +type testCIC struct { + CompressedImageCore + configFile []byte +} + +func (t testCIC) LayerByDigest(h v1.Hash) (CompressedLayer, error) { + return nil, fmt.Errorf("no layer by diff ID %v", h) +} + +func (t testCIC) RawConfigFile() ([]byte, error) { + return t.configFile, nil +} + +func TestConfigLayer(t *testing.T) { + cases := []v1.Image{ + &compressedImageExtender{ + CompressedImageCore: testCIC{ + configFile: []byte("{}"), + }, + }, + &uncompressedImageExtender{ + UncompressedImageCore: testUIC{ + configFile: []byte("{}"), + }, + }, + } + + for _, image := range cases { + hash, err := image.ConfigName() + if err != nil { + t.Fatalf("Error getting config name: %v", err) + } + + if _, err := image.LayerByDigest(hash); err == nil { + t.Error("LayerByDigest(config hash) returned nil error, wanted error") + } + + layer, err := ConfigLayer(image) + if err != nil { + t.Fatalf("ConfigLayer: %v", err) + } + lr, err := layer.Uncompressed() + if err != nil { + t.Fatalf("Error getting uncompressed layer: %v", err) + } + zr, err := layer.Compressed() + if err != nil { + t.Fatalf("Error getting compressed layer: %v", err) + } + + cfgLayerBytes, err := io.ReadAll(lr) + if err != nil { + t.Fatalf("Error reading config layer bytes: %v", err) + } + zcfgLayerBytes, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("Error reading config layer bytes: %v", err) + } + + cfgFile, err := image.RawConfigFile() + if err != nil { + t.Fatalf("Error getting raw config file: %v", err) + } + + if string(cfgFile) != string(cfgLayerBytes) { + t.Errorf("Config file layer doesn't match raw config file") + } + if string(cfgFile) != string(zcfgLayerBytes) { + t.Errorf("Config file layer doesn't match raw config file") + } + + size, err := layer.Size() + if err != nil { + t.Fatalf("Error getting config layer size: %v", err) + } + if size != int64(len(cfgFile)) { + t.Errorf("Size() = %d, want %d", size, len(cfgFile)) + } + + digest, err := layer.Digest() + if err != nil { + t.Fatalf("Digest() = %v", err) + } + if digest != hash { + t.Errorf("ConfigLayer().Digest() != ConfigName(); %v, %v", digest, hash) + } + + diffid, err := layer.DiffID() + if err != nil { + t.Fatalf("DiffId() = %v", err) + } + if diffid != hash { + t.Errorf("ConfigLayer().DiffID() != ConfigName(); %v, %v", diffid, hash) + } + + mt, err := layer.MediaType() + if err != nil { + t.Fatalf("Error getting config layer media type: %v", err) + } + + if mt != types.OCIConfigJSON { + t.Errorf("MediaType() = %v, want %v", mt, types.OCIConfigJSON) + } + } +} diff --git a/pkg/v1/partial/doc.go b/pkg/v1/partial/doc.go new file mode 100644 index 0000000..153dfe4 --- /dev/null +++ b/pkg/v1/partial/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package partial defines methods for building up a v1.Image from +// minimal subsets that are sufficient for defining a v1.Image. +package partial diff --git a/pkg/v1/partial/image.go b/pkg/v1/partial/image.go new file mode 100644 index 0000000..c65f45e --- /dev/null +++ b/pkg/v1/partial/image.go @@ -0,0 +1,28 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// ImageCore is the core set of properties without which we cannot build a v1.Image +type ImageCore interface { + // RawConfigFile returns the serialized bytes of this image's config file. + RawConfigFile() ([]byte, error) + + // MediaType of this image's manifest. + MediaType() (types.MediaType, error) +} diff --git a/pkg/v1/partial/index.go b/pkg/v1/partial/index.go new file mode 100644 index 0000000..f17f274 --- /dev/null +++ b/pkg/v1/partial/index.go @@ -0,0 +1,85 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/match" +) + +// FindManifests given a v1.ImageIndex, find the manifests that fit the matcher. +func FindManifests(index v1.ImageIndex, matcher match.Matcher) ([]v1.Descriptor, error) { + // get the actual manifest list + indexManifest, err := index.IndexManifest() + if err != nil { + return nil, fmt.Errorf("unable to get raw index: %w", err) + } + manifests := []v1.Descriptor{} + // try to get the root of our image + for _, manifest := range indexManifest.Manifests { + if matcher(manifest) { + manifests = append(manifests, manifest) + } + } + return manifests, nil +} + +// FindImages given a v1.ImageIndex, find the images that fit the matcher. If a Descriptor +// matches the provider Matcher, but the referenced item is not an Image, ignores it. +// Only returns those that match the Matcher and are images. +func FindImages(index v1.ImageIndex, matcher match.Matcher) ([]v1.Image, error) { + matches := []v1.Image{} + manifests, err := FindManifests(index, matcher) + if err != nil { + return nil, err + } + for _, desc := range manifests { + // if it is not an image, ignore it + if !desc.MediaType.IsImage() { + continue + } + img, err := index.Image(desc.Digest) + if err != nil { + return nil, err + } + matches = append(matches, img) + } + return matches, nil +} + +// FindIndexes given a v1.ImageIndex, find the indexes that fit the matcher. If a Descriptor +// matches the provider Matcher, but the referenced item is not an Index, ignores it. +// Only returns those that match the Matcher and are indexes. +func FindIndexes(index v1.ImageIndex, matcher match.Matcher) ([]v1.ImageIndex, error) { + matches := []v1.ImageIndex{} + manifests, err := FindManifests(index, matcher) + if err != nil { + return nil, err + } + for _, desc := range manifests { + if !desc.MediaType.IsIndex() { + continue + } + // if it is not an index, ignore it + idx, err := index.ImageIndex(desc.Digest) + if err != nil { + return nil, err + } + matches = append(matches, idx) + } + return matches, nil +} diff --git a/pkg/v1/partial/index_test.go b/pkg/v1/partial/index_test.go new file mode 100644 index 0000000..c289758 --- /dev/null +++ b/pkg/v1/partial/index_test.go @@ -0,0 +1,119 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial_test + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestFindManifests(t *testing.T) { + ii, err := random.Index(100, 5, 6) // random image of 6 manifests, each having 5 layers of size 100 + if err != nil { + t.Fatal("could not create random index:", err) + } + m, _ := ii.IndexManifest() + digest := m.Manifests[0].Digest + + matcher := func(desc v1.Descriptor) bool { + return desc.Digest != digest + } + + descriptors, err := partial.FindManifests(ii, matcher) + expected := len(m.Manifests) - 1 + switch { + case err != nil: + t.Error("unexpected error:", err) + case len(descriptors) != expected: + t.Errorf("failed on manifests, actual %d, expected %d", len(descriptors), expected) + } +} + +func TestFindImages(t *testing.T) { + // create our imageindex with which to work + ii, err := random.Index(100, 5, 6) // random image of 6 manifests, each having 5 layers of size 100 + if err != nil { + t.Fatal("could not create random index:", err) + } + m, _ := ii.IndexManifest() + digest := m.Manifests[0].Digest + + matcher := func(desc v1.Descriptor) bool { + return desc.Digest != digest + } + images, err := partial.FindImages(ii, matcher) + expected := len(m.Manifests) - 1 + switch { + case err != nil: + t.Error("unexpected error:", err) + case len(images) != expected: + t.Errorf("failed on images, actual %d, expected %d", len(images), expected) + } +} + +func TestFindIndexes(t *testing.T) { + // there is no utility to generate an index of indexes, so we need to create one + // base index + var ( + indexCount = 5 + imageCount = 7 + ) + base := empty.Index + // we now have 5 indexes and 5 images, so wrap them into a single index + adds := []mutate.IndexAddendum{} + for i := 0; i < indexCount; i++ { + ii, err := random.Index(100, 1, 1) + if err != nil { + t.Fatalf("%d: unable to create random index: %v", i, err) + } + adds = append(adds, mutate.IndexAddendum{ + Add: ii, + Descriptor: v1.Descriptor{ + MediaType: types.OCIImageIndex, + }, + }) + } + for i := 0; i < imageCount; i++ { + img, err := random.Image(100, 1) + if err != nil { + t.Fatalf("%d: unable to create random image: %v", i, err) + } + adds = append(adds, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{ + MediaType: types.OCIManifestSchema1, + }, + }) + } + + // just see if it finds all of the indexes + matcher := func(desc v1.Descriptor) bool { + return true + } + index := mutate.AppendManifests(base, adds...) + idxes, err := partial.FindIndexes(index, matcher) + switch { + case err != nil: + t.Error("unexpected error:", err) + case len(idxes) != indexCount: + t.Errorf("failed on index, actual %d, expected %d", len(idxes), indexCount) + } +} diff --git a/pkg/v1/partial/uncompressed.go b/pkg/v1/partial/uncompressed.go new file mode 100644 index 0000000..df20d3a --- /dev/null +++ b/pkg/v1/partial/uncompressed.go @@ -0,0 +1,223 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "bytes" + "io" + "sync" + + "github.com/google/go-containerregistry/internal/gzip" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// UncompressedLayer represents the bare minimum interface a natively +// uncompressed layer must implement for us to produce a v1.Layer +type UncompressedLayer interface { + // DiffID returns the Hash of the uncompressed layer. + DiffID() (v1.Hash, error) + + // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. + Uncompressed() (io.ReadCloser, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) +} + +// uncompressedLayerExtender implements v1.Image using the uncompressed base properties. +type uncompressedLayerExtender struct { + UncompressedLayer + // Memoize size/hash so that the methods aren't twice as + // expensive as doing this manually. + hash v1.Hash + size int64 + hashSizeError error + once sync.Once +} + +// Compressed implements v1.Layer +func (ule *uncompressedLayerExtender) Compressed() (io.ReadCloser, error) { + u, err := ule.Uncompressed() + if err != nil { + return nil, err + } + return gzip.ReadCloser(u), nil +} + +// Digest implements v1.Layer +func (ule *uncompressedLayerExtender) Digest() (v1.Hash, error) { + ule.calcSizeHash() + return ule.hash, ule.hashSizeError +} + +// Size implements v1.Layer +func (ule *uncompressedLayerExtender) Size() (int64, error) { + ule.calcSizeHash() + return ule.size, ule.hashSizeError +} + +func (ule *uncompressedLayerExtender) calcSizeHash() { + ule.once.Do(func() { + var r io.ReadCloser + r, ule.hashSizeError = ule.Compressed() + if ule.hashSizeError != nil { + return + } + defer r.Close() + ule.hash, ule.size, ule.hashSizeError = v1.SHA256(r) + }) +} + +// UncompressedToLayer fills in the missing methods from an UncompressedLayer so that it implements v1.Layer +func UncompressedToLayer(ul UncompressedLayer) (v1.Layer, error) { + return &uncompressedLayerExtender{UncompressedLayer: ul}, nil +} + +// UncompressedImageCore represents the bare minimum interface a natively +// uncompressed image must implement for us to produce a v1.Image +type UncompressedImageCore interface { + ImageCore + + // LayerByDiffID is a variation on the v1.Image method, which returns + // an UncompressedLayer instead. + LayerByDiffID(v1.Hash) (UncompressedLayer, error) +} + +// UncompressedToImage fills in the missing methods from an UncompressedImageCore so that it implements v1.Image. +func UncompressedToImage(uic UncompressedImageCore) (v1.Image, error) { + return &uncompressedImageExtender{ + UncompressedImageCore: uic, + }, nil +} + +// uncompressedImageExtender implements v1.Image by extending UncompressedImageCore with the +// appropriate methods computed from the minimal core. +type uncompressedImageExtender struct { + UncompressedImageCore + + lock sync.Mutex + manifest *v1.Manifest +} + +// Assert that our extender type completes the v1.Image interface +var _ v1.Image = (*uncompressedImageExtender)(nil) + +// Digest implements v1.Image +func (i *uncompressedImageExtender) Digest() (v1.Hash, error) { + return Digest(i) +} + +// Manifest implements v1.Image +func (i *uncompressedImageExtender) Manifest() (*v1.Manifest, error) { + i.lock.Lock() + defer i.lock.Unlock() + if i.manifest != nil { + return i.manifest, nil + } + + b, err := i.RawConfigFile() + if err != nil { + return nil, err + } + + cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + m := &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.DockerManifestSchema2, + Config: v1.Descriptor{ + MediaType: types.DockerConfigJSON, + Size: cfgSize, + Digest: cfgHash, + }, + } + + ls, err := i.Layers() + if err != nil { + return nil, err + } + + m.Layers = make([]v1.Descriptor, len(ls)) + for i, l := range ls { + desc, err := Descriptor(l) + if err != nil { + return nil, err + } + + m.Layers[i] = *desc + } + + i.manifest = m + return i.manifest, nil +} + +// RawManifest implements v1.Image +func (i *uncompressedImageExtender) RawManifest() ([]byte, error) { + return RawManifest(i) +} + +// Size implements v1.Image +func (i *uncompressedImageExtender) Size() (int64, error) { + return Size(i) +} + +// ConfigName implements v1.Image +func (i *uncompressedImageExtender) ConfigName() (v1.Hash, error) { + return ConfigName(i) +} + +// ConfigFile implements v1.Image +func (i *uncompressedImageExtender) ConfigFile() (*v1.ConfigFile, error) { + return ConfigFile(i) +} + +// Layers implements v1.Image +func (i *uncompressedImageExtender) Layers() ([]v1.Layer, error) { + diffIDs, err := DiffIDs(i) + if err != nil { + return nil, err + } + ls := make([]v1.Layer, 0, len(diffIDs)) + for _, h := range diffIDs { + l, err := i.LayerByDiffID(h) + if err != nil { + return nil, err + } + ls = append(ls, l) + } + return ls, nil +} + +// LayerByDiffID implements v1.Image +func (i *uncompressedImageExtender) LayerByDiffID(diffID v1.Hash) (v1.Layer, error) { + ul, err := i.UncompressedImageCore.LayerByDiffID(diffID) + if err != nil { + return nil, err + } + return UncompressedToLayer(ul) +} + +// LayerByDigest implements v1.Image +func (i *uncompressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) { + diffID, err := BlobToDiffID(i, h) + if err != nil { + return nil, err + } + return i.LayerByDiffID(diffID) +} diff --git a/pkg/v1/partial/uncompressed_test.go b/pkg/v1/partial/uncompressed_test.go new file mode 100644 index 0000000..ce5a6fa --- /dev/null +++ b/pkg/v1/partial/uncompressed_test.go @@ -0,0 +1,233 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial_test + +import ( + "io" + "os" + "testing" + + "github.com/google/go-containerregistry/internal/compare" + legacy "github.com/google/go-containerregistry/pkg/legacy/tarball" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +// foreignLayer implements both partial.Describable and partial.UncompressedLayer. +type foreignLayer struct { + wrapped v1.Layer +} + +func (l *foreignLayer) Digest() (v1.Hash, error) { + return l.wrapped.Digest() +} + +func (l *foreignLayer) Size() (int64, error) { + return l.wrapped.Size() +} + +func (l *foreignLayer) MediaType() (types.MediaType, error) { + return types.DockerForeignLayer, nil +} + +func (l *foreignLayer) Uncompressed() (io.ReadCloser, error) { + return l.wrapped.Uncompressed() +} + +func (l *foreignLayer) DiffID() (v1.Hash, error) { + return l.wrapped.DiffID() +} + +func (l *foreignLayer) Descriptor() (*v1.Descriptor, error) { + r, err := l.wrapped.Compressed() + if err != nil { + return nil, err + } + h, sz, err := v1.SHA256(r) + if err != nil { + return nil, err + } + return &v1.Descriptor{ + Digest: h, + Size: sz, + MediaType: types.DockerForeignLayer, + URLs: []string{"http://example.com"}, + }, nil +} + +func (l *foreignLayer) UncompressedSize() (int64, error) { + return partial.UncompressedSize(l.wrapped) +} + +func TestUncompressedLayer(t *testing.T) { + randLayer, err := random.Layer(1024, types.DockerForeignLayer) + if err != nil { + t.Fatal(err) + } + l := &foreignLayer{randLayer} + + desc, err := partial.Descriptor(l) + if err != nil { + t.Fatal(err) + } + + if want, got := desc.URLs[0], "http://example.com"; want != got { + t.Errorf("URLs[0] = %s != %s", got, want) + } + + layer, err := partial.UncompressedToLayer(l) + if err != nil { + t.Fatal(err) + } + + if err := validate.Layer(layer); err != nil { + t.Errorf("validate.Layer: %v", err) + } + if _, err := partial.UncompressedSize(layer); err != nil { + t.Errorf("partial.UncompressedSize: %v", err) + } +} + +// legacy/tarball.Write + tarball.Image leverages a lot of uncompressed partials. +// +// This is cribbed from pkg/legacy/tarball just to get intra-package coverage. +func TestLegacyWrite(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + layer with Descriptor(). + randImage, err := random.Image(256, 2) + if err != nil { + t.Fatalf("Error creating random image: %v", err) + } + randLayer, err := random.Layer(1024, types.DockerForeignLayer) + if err != nil { + t.Fatal(err) + } + l, err := partial.UncompressedToLayer(&foreignLayer{randLayer}) + if err != nil { + t.Fatal(err) + } + img, err := mutate.AppendLayers(randImage, l) + if err != nil { + t.Fatal(err) + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag: %v", err) + } + o, err := os.Create(fp.Name()) + if err != nil { + t.Fatalf("Error creating %q to write image tarball: %v", fp.Name(), err) + } + defer o.Close() + if err := legacy.Write(tag, img, o); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + + // Make sure the image is valid and can be loaded. + // Load it both by nil and by its name. + for _, it := range []*name.Tag{nil, &tag} { + tarImage, err := tarball.ImageFromPath(fp.Name(), it) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + if err := compare.Images(img, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } + + // Try loading a different tag, it should error. + fakeTag, err := name.NewTag("gcr.io/notthistag:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error generating tag: %v", err) + } + if _, err := tarball.ImageFromPath(fp.Name(), &fakeTag); err == nil { + t.Errorf("Expected error loading tag %v from image", fakeTag) + } +} + +type uncompressedImage struct { + img v1.Image +} + +func (i *uncompressedImage) RawConfigFile() ([]byte, error) { + return i.img.RawConfigFile() +} + +func (i *uncompressedImage) MediaType() (types.MediaType, error) { + return i.img.MediaType() +} + +func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { + return i.img.LayerByDiffID(h) +} + +func (i *uncompressedImage) Descriptor() (*v1.Descriptor, error) { + return partial.Descriptor(i.img) +} + +func TestUncompressed(t *testing.T) { + rnd, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + core := &uncompressedImage{rnd} + + img, err := partial.UncompressedToImage(core) + if err != nil { + t.Fatal(err) + } + + if err := validate.Image(img); err != nil { + t.Fatalf("validate.Image: %v", err) + } + if _, err := partial.Descriptor(img); err != nil { + t.Fatalf("partial.Descriptor: %v", err) + } + + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + layer, err := partial.UncompressedToLayer(&fastpathLayer{layers[0]}) + if err != nil { + t.Fatal(err) + } + + ok, err := partial.Exists(layer) + if err != nil { + t.Fatal(err) + } + if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } +} diff --git a/pkg/v1/partial/with.go b/pkg/v1/partial/with.go new file mode 100644 index 0000000..c8b22b3 --- /dev/null +++ b/pkg/v1/partial/with.go @@ -0,0 +1,436 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// WithRawConfigFile defines the subset of v1.Image used by these helper methods +type WithRawConfigFile interface { + // RawConfigFile returns the serialized bytes of this image's config file. + RawConfigFile() ([]byte, error) +} + +// ConfigFile is a helper for implementing v1.Image +func ConfigFile(i WithRawConfigFile) (*v1.ConfigFile, error) { + b, err := i.RawConfigFile() + if err != nil { + return nil, err + } + return v1.ParseConfigFile(bytes.NewReader(b)) +} + +// ConfigName is a helper for implementing v1.Image +func ConfigName(i WithRawConfigFile) (v1.Hash, error) { + b, err := i.RawConfigFile() + if err != nil { + return v1.Hash{}, err + } + h, _, err := v1.SHA256(bytes.NewReader(b)) + return h, err +} + +type configLayer struct { + hash v1.Hash + content []byte +} + +// Digest implements v1.Layer +func (cl *configLayer) Digest() (v1.Hash, error) { + return cl.hash, nil +} + +// DiffID implements v1.Layer +func (cl *configLayer) DiffID() (v1.Hash, error) { + return cl.hash, nil +} + +// Uncompressed implements v1.Layer +func (cl *configLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(cl.content)), nil +} + +// Compressed implements v1.Layer +func (cl *configLayer) Compressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(cl.content)), nil +} + +// Size implements v1.Layer +func (cl *configLayer) Size() (int64, error) { + return int64(len(cl.content)), nil +} + +func (cl *configLayer) MediaType() (types.MediaType, error) { + // Defaulting this to OCIConfigJSON as it should remain + // backwards compatible with DockerConfigJSON + return types.OCIConfigJSON, nil +} + +var _ v1.Layer = (*configLayer)(nil) + +// withConfigLayer allows partial image implementations to provide a layer +// for their config file. +type withConfigLayer interface { + ConfigLayer() (v1.Layer, error) +} + +// ConfigLayer implements v1.Layer from the raw config bytes. +// This is so that clients (e.g. remote) can access the config as a blob. +// +// Images that want to return a specific layer implementation can implement +// withConfigLayer. +func ConfigLayer(i WithRawConfigFile) (v1.Layer, error) { + if wcl, ok := unwrap(i).(withConfigLayer); ok { + return wcl.ConfigLayer() + } + + h, err := ConfigName(i) + if err != nil { + return nil, err + } + rcfg, err := i.RawConfigFile() + if err != nil { + return nil, err + } + return &configLayer{ + hash: h, + content: rcfg, + }, nil +} + +// WithConfigFile defines the subset of v1.Image used by these helper methods +type WithConfigFile interface { + // ConfigFile returns this image's config file. + ConfigFile() (*v1.ConfigFile, error) +} + +// DiffIDs is a helper for implementing v1.Image +func DiffIDs(i WithConfigFile) ([]v1.Hash, error) { + cfg, err := i.ConfigFile() + if err != nil { + return nil, err + } + return cfg.RootFS.DiffIDs, nil +} + +// RawConfigFile is a helper for implementing v1.Image +func RawConfigFile(i WithConfigFile) ([]byte, error) { + cfg, err := i.ConfigFile() + if err != nil { + return nil, err + } + return json.Marshal(cfg) +} + +// WithRawManifest defines the subset of v1.Image used by these helper methods +type WithRawManifest interface { + // RawManifest returns the serialized bytes of this image's config file. + RawManifest() ([]byte, error) +} + +// Digest is a helper for implementing v1.Image +func Digest(i WithRawManifest) (v1.Hash, error) { + mb, err := i.RawManifest() + if err != nil { + return v1.Hash{}, err + } + digest, _, err := v1.SHA256(bytes.NewReader(mb)) + return digest, err +} + +// Manifest is a helper for implementing v1.Image +func Manifest(i WithRawManifest) (*v1.Manifest, error) { + b, err := i.RawManifest() + if err != nil { + return nil, err + } + return v1.ParseManifest(bytes.NewReader(b)) +} + +// WithManifest defines the subset of v1.Image used by these helper methods +type WithManifest interface { + // Manifest returns this image's Manifest object. + Manifest() (*v1.Manifest, error) +} + +// RawManifest is a helper for implementing v1.Image +func RawManifest(i WithManifest) ([]byte, error) { + m, err := i.Manifest() + if err != nil { + return nil, err + } + return json.Marshal(m) +} + +// Size is a helper for implementing v1.Image +func Size(i WithRawManifest) (int64, error) { + b, err := i.RawManifest() + if err != nil { + return -1, err + } + return int64(len(b)), nil +} + +// FSLayers is a helper for implementing v1.Image +func FSLayers(i WithManifest) ([]v1.Hash, error) { + m, err := i.Manifest() + if err != nil { + return nil, err + } + fsl := make([]v1.Hash, len(m.Layers)) + for i, l := range m.Layers { + fsl[i] = l.Digest + } + return fsl, nil +} + +// BlobSize is a helper for implementing v1.Image +func BlobSize(i WithManifest, h v1.Hash) (int64, error) { + d, err := BlobDescriptor(i, h) + if err != nil { + return -1, err + } + return d.Size, nil +} + +// BlobDescriptor is a helper for implementing v1.Image +func BlobDescriptor(i WithManifest, h v1.Hash) (*v1.Descriptor, error) { + m, err := i.Manifest() + if err != nil { + return nil, err + } + + if m.Config.Digest == h { + return &m.Config, nil + } + + for _, l := range m.Layers { + if l.Digest == h { + return &l, nil + } + } + return nil, fmt.Errorf("blob %v not found", h) +} + +// WithManifestAndConfigFile defines the subset of v1.Image used by these helper methods +type WithManifestAndConfigFile interface { + WithConfigFile + + // Manifest returns this image's Manifest object. + Manifest() (*v1.Manifest, error) +} + +// BlobToDiffID is a helper for mapping between compressed +// and uncompressed blob hashes. +func BlobToDiffID(i WithManifestAndConfigFile, h v1.Hash) (v1.Hash, error) { + blobs, err := FSLayers(i) + if err != nil { + return v1.Hash{}, err + } + diffIDs, err := DiffIDs(i) + if err != nil { + return v1.Hash{}, err + } + if len(blobs) != len(diffIDs) { + return v1.Hash{}, fmt.Errorf("mismatched fs layers (%d) and diff ids (%d)", len(blobs), len(diffIDs)) + } + for i, blob := range blobs { + if blob == h { + return diffIDs[i], nil + } + } + return v1.Hash{}, fmt.Errorf("unknown blob %v", h) +} + +// DiffIDToBlob is a helper for mapping between uncompressed +// and compressed blob hashes. +func DiffIDToBlob(wm WithManifestAndConfigFile, h v1.Hash) (v1.Hash, error) { + blobs, err := FSLayers(wm) + if err != nil { + return v1.Hash{}, err + } + diffIDs, err := DiffIDs(wm) + if err != nil { + return v1.Hash{}, err + } + if len(blobs) != len(diffIDs) { + return v1.Hash{}, fmt.Errorf("mismatched fs layers (%d) and diff ids (%d)", len(blobs), len(diffIDs)) + } + for i, diffID := range diffIDs { + if diffID == h { + return blobs[i], nil + } + } + return v1.Hash{}, fmt.Errorf("unknown diffID %v", h) +} + +// WithDiffID defines the subset of v1.Layer for exposing the DiffID method. +type WithDiffID interface { + DiffID() (v1.Hash, error) +} + +// withDescriptor allows partial layer implementations to provide a layer +// descriptor to the partial image manifest builder. This allows partial +// uncompressed layers to provide foreign layer metadata like URLs to the +// uncompressed image manifest. +type withDescriptor interface { + Descriptor() (*v1.Descriptor, error) +} + +// Describable represents something for which we can produce a v1.Descriptor. +type Describable interface { + Digest() (v1.Hash, error) + MediaType() (types.MediaType, error) + Size() (int64, error) +} + +// Descriptor returns a v1.Descriptor given a Describable. It also encodes +// some logic for unwrapping things that have been wrapped by +// CompressedToLayer, UncompressedToLayer, CompressedToImage, or +// UncompressedToImage. +func Descriptor(d Describable) (*v1.Descriptor, error) { + // If Describable implements Descriptor itself, return that. + if wd, ok := unwrap(d).(withDescriptor); ok { + return wd.Descriptor() + } + + // If all else fails, compute the descriptor from the individual methods. + var ( + desc v1.Descriptor + err error + ) + + if desc.Size, err = d.Size(); err != nil { + return nil, err + } + if desc.Digest, err = d.Digest(); err != nil { + return nil, err + } + if desc.MediaType, err = d.MediaType(); err != nil { + return nil, err + } + if wat, ok := d.(withArtifactType); ok { + if desc.ArtifactType, err = wat.ArtifactType(); err != nil { + return nil, err + } + } else { + if wrm, ok := d.(WithRawManifest); ok && desc.MediaType.IsImage() { + mf, _ := Manifest(wrm) + // Failing to parse as a manifest should just be ignored. + // The manifest might not be valid, and that's okay. + if mf != nil && !mf.Config.MediaType.IsConfig() { + desc.ArtifactType = string(mf.Config.MediaType) + } + } + } + + return &desc, nil +} + +type withArtifactType interface { + ArtifactType() (string, error) +} + +type withUncompressedSize interface { + UncompressedSize() (int64, error) +} + +// UncompressedSize returns the size of the Uncompressed layer. If the +// underlying implementation doesn't implement UncompressedSize directly, +// this will compute the uncompressedSize by reading everything returned +// by Compressed(). This is potentially expensive and may consume the contents +// for streaming layers. +func UncompressedSize(l v1.Layer) (int64, error) { + // If the layer implements UncompressedSize itself, return that. + if wus, ok := unwrap(l).(withUncompressedSize); ok { + return wus.UncompressedSize() + } + + // The layer doesn't implement UncompressedSize, we need to compute it. + rc, err := l.Uncompressed() + if err != nil { + return -1, err + } + defer rc.Close() + + return io.Copy(io.Discard, rc) +} + +type withExists interface { + Exists() (bool, error) +} + +// Exists checks to see if a layer exists. This is a hack to work around the +// mistakes of the partial package. Don't use this. +func Exists(l v1.Layer) (bool, error) { + // If the layer implements Exists itself, return that. + if we, ok := unwrap(l).(withExists); ok { + return we.Exists() + } + + // The layer doesn't implement Exists, so we hope that calling Compressed() + // is enough to trigger an error if the layer does not exist. + rc, err := l.Compressed() + if err != nil { + return false, err + } + defer rc.Close() + + // We may want to try actually reading a single byte, but if we need to do + // that, we should just fix this hack. + return true, nil +} + +// Recursively unwrap our wrappers so that we can check for the original implementation. +// We might want to expose this? +func unwrap(i any) any { + if ule, ok := i.(*uncompressedLayerExtender); ok { + return unwrap(ule.UncompressedLayer) + } + if cle, ok := i.(*compressedLayerExtender); ok { + return unwrap(cle.CompressedLayer) + } + if uie, ok := i.(*uncompressedImageExtender); ok { + return unwrap(uie.UncompressedImageCore) + } + if cie, ok := i.(*compressedImageExtender); ok { + return unwrap(cie.CompressedImageCore) + } + return i +} + +// ArtifactType returns the artifact type for the given manifest. +// +// If the manifest reports its own artifact type, that's returned, otherwise +// the manifest is parsed and, if successful, its config.mediaType is returned. +func ArtifactType(w WithManifest) (string, error) { + if wat, ok := w.(withArtifactType); ok { + return wat.ArtifactType() + } + mf, _ := w.Manifest() + // Failing to parse as a manifest should just be ignored. + // The manifest might not be valid, and that's okay. + if mf != nil && !mf.Config.MediaType.IsConfig() { + return string(mf.Config.MediaType), nil + } + return "", nil +} diff --git a/pkg/v1/partial/with_test.go b/pkg/v1/partial/with_test.go new file mode 100644 index 0000000..7796bd9 --- /dev/null +++ b/pkg/v1/partial/with_test.go @@ -0,0 +1,246 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestRawConfigFile(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + part, err := partial.RawConfigFile(img) + if err != nil { + t.Fatal(err) + } + + method, err := img.RawConfigFile() + if err != nil { + t.Fatal(err) + } + + if string(part) != string(method) { + t.Errorf("mismatched config file: %s vs %s", part, method) + } +} + +func TestDigest(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + part, err := partial.Digest(img) + if err != nil { + t.Fatal(err) + } + + method, err := img.Digest() + if err != nil { + t.Fatal(err) + } + + if part != method { + t.Errorf("mismatched digest: %s vs %s", part, method) + } +} + +func TestManifest(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + part, err := partial.Manifest(img) + if err != nil { + t.Fatal(err) + } + + method, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(part, method); diff != "" { + t.Errorf("mismatched manifest: %v", diff) + } +} + +func TestSize(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + + part, err := partial.Size(img) + if err != nil { + t.Fatal(err) + } + + method, err := img.Size() + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(part, method); diff != "" { + t.Errorf("mismatched size: %v", diff) + } +} + +func TestDiffIDToBlob(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + want, err := layers[0].Digest() + if err != nil { + t.Fatal(err) + } + got, err := partial.DiffIDToBlob(img, cf.RootFS.DiffIDs[0]) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("mismatched digest: %v", diff) + } + + if _, err := partial.DiffIDToBlob(img, want); err == nil { + t.Errorf("expected err, got nil") + } +} + +func TestBlobToDiffID(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + d, err := layers[0].Digest() + if err != nil { + t.Fatal(err) + } + want := cf.RootFS.DiffIDs[0] + got, err := partial.BlobToDiffID(img, d) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("mismatched digest: %v", diff) + } + + if _, err := partial.BlobToDiffID(img, want); err == nil { + t.Errorf("expected err, got nil") + } +} + +func TestBlobSize(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatal(err) + } + m, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + want := m.Layers[0].Size + got, err := partial.BlobSize(img, m.Layers[0].Digest) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("mismatched blob size: %v", diff) + } + + if _, err := partial.BlobSize(img, v1.Hash{}); err == nil { + t.Errorf("expected err, got nil") + } +} + +type fastpathLayer struct { + v1.Layer +} + +func (l *fastpathLayer) UncompressedSize() (int64, error) { + return 100, nil +} + +func (l *fastpathLayer) Exists() (bool, error) { + return true, nil +} + +func TestUncompressedSize(t *testing.T) { + randLayer, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + fpl := &fastpathLayer{randLayer} + us, err := partial.UncompressedSize(fpl) + if err != nil { + t.Fatal(err) + } + if got, want := us, int64(100); got != want { + t.Errorf("UncompressedSize() = %d != %d", got, want) + } +} + +func TestExists(t *testing.T) { + randLayer, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + fpl := &fastpathLayer{randLayer} + ok, err := partial.Exists(fpl) + if err != nil { + t.Fatal(err) + } + if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } + + ok, err = partial.Exists(randLayer) + if err != nil { + t.Fatal(err) + } + if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } +} diff --git a/pkg/v1/platform.go b/pkg/v1/platform.go new file mode 100644 index 0000000..59ca402 --- /dev/null +++ b/pkg/v1/platform.go @@ -0,0 +1,149 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + "sort" + "strings" +) + +// Platform represents the target os/arch for an image. +type Platform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` + Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` +} + +func (p Platform) String() string { + if p.OS == "" { + return "" + } + var b strings.Builder + b.WriteString(p.OS) + if p.Architecture != "" { + b.WriteString("/") + b.WriteString(p.Architecture) + } + if p.Variant != "" { + b.WriteString("/") + b.WriteString(p.Variant) + } + if p.OSVersion != "" { + b.WriteString(":") + b.WriteString(p.OSVersion) + } + return b.String() +} + +// ParsePlatform parses a string representing a Platform, if possible. +func ParsePlatform(s string) (*Platform, error) { + var p Platform + parts := strings.Split(strings.TrimSpace(s), ":") + if len(parts) == 2 { + p.OSVersion = parts[1] + } + parts = strings.Split(parts[0], "/") + if len(parts) > 0 { + p.OS = parts[0] + } + if len(parts) > 1 { + p.Architecture = parts[1] + } + if len(parts) > 2 { + p.Variant = parts[2] + } + if len(parts) > 3 { + return nil, fmt.Errorf("too many slashes in platform spec: %s", s) + } + return &p, nil +} + +// Equals returns true if the given platform is semantically equivalent to this one. +// The order of Features and OSFeatures is not important. +func (p Platform) Equals(o Platform) bool { + return p.OS == o.OS && + p.Architecture == o.Architecture && + p.Variant == o.Variant && + p.OSVersion == o.OSVersion && + stringSliceEqualIgnoreOrder(p.OSFeatures, o.OSFeatures) && + stringSliceEqualIgnoreOrder(p.Features, o.Features) +} + +// Satisfies returns true if this Platform "satisfies" the given spec Platform. +// +// Note that this is different from Equals and that Satisfies is not reflexive. +// +// The given spec represents "requirements" such that any missing values in the +// spec are not compared. +// +// For OSFeatures and Features, Satisfies will return true if this Platform's +// fields contain a superset of the values in the spec's fields (order ignored). +func (p Platform) Satisfies(spec Platform) bool { + return satisfies(spec.OS, p.OS) && + satisfies(spec.Architecture, p.Architecture) && + satisfies(spec.Variant, p.Variant) && + satisfies(spec.OSVersion, p.OSVersion) && + satisfiesList(spec.OSFeatures, p.OSFeatures) && + satisfiesList(spec.Features, p.Features) +} + +func satisfies(want, have string) bool { + return want == "" || want == have +} + +func satisfiesList(want, have []string) bool { + if len(want) == 0 { + return true + } + + set := map[string]struct{}{} + for _, h := range have { + set[h] = struct{}{} + } + + for _, w := range want { + if _, ok := set[w]; !ok { + return false + } + } + + return true +} + +// stringSliceEqual compares 2 string slices and returns if their contents are identical. +func stringSliceEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, elm := range a { + if elm != b[i] { + return false + } + } + return true +} + +// stringSliceEqualIgnoreOrder compares 2 string slices and returns if their contents are identical, ignoring order +func stringSliceEqualIgnoreOrder(a, b []string) bool { + if a != nil && b != nil { + sort.Strings(a) + sort.Strings(b) + } + return stringSliceEqual(a, b) +} diff --git a/pkg/v1/platform_test.go b/pkg/v1/platform_test.go new file mode 100644 index 0000000..80c67ed --- /dev/null +++ b/pkg/v1/platform_test.go @@ -0,0 +1,235 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func TestPlatformString(t *testing.T) { + for _, c := range []struct { + plat v1.Platform + want string + }{{ + v1.Platform{}, + "", + }, { + v1.Platform{OS: "linux"}, + "linux", + }, { + v1.Platform{OS: "linux", Architecture: "amd64"}, + "linux/amd64", + }, { + v1.Platform{OS: "linux", Architecture: "amd64", Variant: "v7"}, + "linux/amd64/v7", + }, { + v1.Platform{OS: "linux", Architecture: "amd64", OSVersion: "1.2.3.4"}, + "linux/amd64:1.2.3.4", + }, { + v1.Platform{OS: "linux", Architecture: "amd64", OSVersion: "1.2.3.4", OSFeatures: []string{"a", "b"}, Features: []string{"c", "d"}}, + "linux/amd64:1.2.3.4", + }} { + if got := c.plat.String(); got != c.want { + t.Errorf("got %q, want %q", got, c.want) + } + + if len(c.plat.OSFeatures) > 0 || len(c.plat.Features) > 0 { + // If these values are set, roundtripping back to the + // Platform will be lossy, and we expect that. + continue + } + + back, err := v1.ParsePlatform(c.plat.String()) + if err != nil { + t.Errorf("ParsePlatform(%q): %v", c.plat, err) + } + if d := cmp.Diff(&c.plat, back); d != "" { + t.Errorf("ParsePlatform(%q) diff:\n%s", c.plat.String(), d) + } + } + + // Known bad examples. + for _, s := range []string{ + "linux/amd64/v7/s9", // too many slashes + } { + got, err := v1.ParsePlatform(s) + if err == nil { + t.Errorf("ParsePlatform(%q) wanted error; got %v", s, got) + } + } +} + +func TestPlatformEquals(t *testing.T) { + tests := []struct { + a, b v1.Platform + equal bool + }{{ + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "arm64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "darwin"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "5.0"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "5.0"}, + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "3.6"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "ubuntu"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"ac", "bd"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"b", "a"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"ac", "bd"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"b", "a"}}, + true, + }} + for i, tt := range tests { + if equal := tt.a.Equals(tt.b); equal != tt.equal { + t.Errorf("%d: mismatched was %v expected %v; original (-want +got) %s", i, equal, tt.equal, cmp.Diff(tt.a, tt.b)) + } + } +} + +func TestPlatformSatisfies(t *testing.T) { + tests := []struct { + have, spec v1.Platform + sat bool + }{{ + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "arm64", OS: "linux"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "darwin"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "5.0"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "5.0"}, + v1.Platform{Architecture: "amd64", OS: "linux", OSVersion: "3.6"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "ubuntu"}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + v1.Platform{Architecture: "amd64", OS: "linux", Variant: "pios"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"ac", "bd"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", OSFeatures: []string{"b", "a"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux"}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux"}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + true, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"ac", "bd"}}, + false, + }, { + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"a", "b"}}, + v1.Platform{Architecture: "amd64", OS: "linux", Features: []string{"b", "a"}}, + true, + }} + for i, tt := range tests { + if sat := tt.have.Satisfies(tt.spec); sat != tt.sat { + t.Errorf("%d: mismatched was %v expected %v; original (-want +got) %s", i, sat, tt.sat, cmp.Diff(tt.have, tt.spec)) + } + } +} diff --git a/pkg/v1/progress.go b/pkg/v1/progress.go new file mode 100644 index 0000000..844f04d --- /dev/null +++ b/pkg/v1/progress.go @@ -0,0 +1,25 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +// Update representation of an update of transfer progress. Some functions +// in this module can take a channel to which updates will be sent while a +// transfer is in progress. +// +k8s:deepcopy-gen=false +type Update struct { + Total int64 + Complete int64 + Error error +} diff --git a/pkg/v1/random/doc.go b/pkg/v1/random/doc.go new file mode 100644 index 0000000..d371276 --- /dev/null +++ b/pkg/v1/random/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package random provides a facility for synthesizing pseudo-random images. +package random diff --git a/pkg/v1/random/image.go b/pkg/v1/random/image.go new file mode 100644 index 0000000..4b28913 --- /dev/null +++ b/pkg/v1/random/image.go @@ -0,0 +1,116 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package random + +import ( + "archive/tar" + "bytes" + "crypto" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + mrand "math/rand" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// uncompressedLayer implements partial.UncompressedLayer from raw bytes. +type uncompressedLayer struct { + diffID v1.Hash + mediaType types.MediaType + content []byte +} + +// DiffID implements partial.UncompressedLayer +func (ul *uncompressedLayer) DiffID() (v1.Hash, error) { + return ul.diffID, nil +} + +// Uncompressed implements partial.UncompressedLayer +func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(ul.content)), nil +} + +// MediaType returns the media type of the layer +func (ul *uncompressedLayer) MediaType() (types.MediaType, error) { + return ul.mediaType, nil +} + +var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) + +// Image returns a pseudo-randomly generated Image. +func Image(byteSize, layers int64) (v1.Image, error) { + adds := make([]mutate.Addendum, 0, 5) + for i := int64(0); i < layers; i++ { + layer, err := Layer(byteSize, types.DockerLayer) + if err != nil { + return nil, err + } + adds = append(adds, mutate.Addendum{ + Layer: layer, + History: v1.History{ + Author: "random.Image", + Comment: fmt.Sprintf("this is a random history %d of %d", i, layers), + CreatedBy: "random", + Created: v1.Time{Time: time.Now()}, + }, + }) + } + + return mutate.Append(empty.Image, adds...) +} + +// Layer returns a layer with pseudo-randomly generated content. +func Layer(byteSize int64, mt types.MediaType) (v1.Layer, error) { + fileName := fmt.Sprintf("random_file_%d.txt", mrand.Int()) //nolint: gosec + + // Hash the contents as we write it out to the buffer. + var b bytes.Buffer + hasher := crypto.SHA256.New() + mw := io.MultiWriter(&b, hasher) + + // Write a single file with a random name and random contents. + tw := tar.NewWriter(mw) + if err := tw.WriteHeader(&tar.Header{ + Name: fileName, + Size: byteSize, + Typeflag: tar.TypeReg, + }); err != nil { + return nil, err + } + if _, err := io.CopyN(tw, rand.Reader, byteSize); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + + h := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + } + + return partial.UncompressedToLayer(&uncompressedLayer{ + diffID: h, + mediaType: mt, + content: b.Bytes(), + }) +} diff --git a/pkg/v1/random/image_test.go b/pkg/v1/random/image_test.go new file mode 100644 index 0000000..8f30bc7 --- /dev/null +++ b/pkg/v1/random/image_test.go @@ -0,0 +1,129 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package random + +import ( + "archive/tar" + "errors" + "io" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestManifestAndConfig(t *testing.T) { + want := int64(12) + img, err := Image(1024, want) + if err != nil { + t.Fatalf("Error loading image: %v", err) + } + manifest, err := img.Manifest() + if err != nil { + t.Fatalf("Error loading manifest: %v", err) + } + if got := int64(len(manifest.Layers)); got != want { + t.Fatalf("num layers; got %v, want %v", got, want) + } + + config, err := img.ConfigFile() + if err != nil { + t.Fatalf("Error loading config file: %v", err) + } + if got := int64(len(config.RootFS.DiffIDs)); got != want { + t.Fatalf("num diff ids; got %v, want %v", got, want) + } + + if err := validate.Image(img); err != nil { + t.Errorf("failed to validate: %v", err) + } +} + +func TestTarLayer(t *testing.T) { + img, err := Image(1024, 5) + if err != nil { + t.Fatalf("Image: %v", err) + } + layers, err := img.Layers() + if err != nil { + t.Fatalf("Layers: %v", err) + } + if len(layers) != 5 { + t.Errorf("Got %d layers, want 5", len(layers)) + } + for i, l := range layers { + mediaType, err := l.MediaType() + if err != nil { + t.Fatalf("MediaType: %v", err) + } + if got, want := mediaType, types.DockerLayer; got != want { + t.Fatalf("MediaType(); got %q, want %q", got, want) + } + + rc, err := l.Uncompressed() + if err != nil { + t.Errorf("Uncompressed(%d): %v", i, err) + } + defer rc.Close() + tr := tar.NewReader(rc) + if _, err := tr.Next(); err != nil { + t.Errorf("tar.Next: %v", err) + } + + if n, err := io.Copy(io.Discard, tr); err != nil { + t.Errorf("Reading tar layer: %v", err) + } else if n != 1024 { + t.Errorf("Layer %d was %d bytes, want 1024", i, n) + } + + if _, err := tr.Next(); !errors.Is(err, io.EOF) { + t.Errorf("Layer contained more files; got %v, want EOF", err) + } + } +} + +func TestRandomLayer(t *testing.T) { + l, err := Layer(1024, types.DockerLayer) + if err != nil { + t.Fatalf("Layer: %v", err) + } + mediaType, err := l.MediaType() + if err != nil { + t.Fatalf("MediaType: %v", err) + } + if got, want := mediaType, types.DockerLayer; got != want { + t.Errorf("MediaType(); got %q, want %q", got, want) + } + + rc, err := l.Uncompressed() + if err != nil { + t.Fatalf("Uncompressed(): %v", err) + } + defer rc.Close() + tr := tar.NewReader(rc) + if _, err := tr.Next(); err != nil { + t.Fatalf("tar.Next: %v", err) + } + + if n, err := io.Copy(io.Discard, tr); err != nil { + t.Errorf("Reading tar layer: %v", err) + } else if n != 1024 { + t.Errorf("Layer was %d bytes, want 1024", n) + } + + if _, err := tr.Next(); !errors.Is(err, io.EOF) { + t.Errorf("Layer contained more files; got %v, want EOF", err) + } +} diff --git a/pkg/v1/random/index.go b/pkg/v1/random/index.go new file mode 100644 index 0000000..89a8843 --- /dev/null +++ b/pkg/v1/random/index.go @@ -0,0 +1,111 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package random + +import ( + "bytes" + "encoding/json" + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type randomIndex struct { + images map[v1.Hash]v1.Image + manifest *v1.IndexManifest +} + +// Index returns a pseudo-randomly generated ImageIndex with count images, each +// having the given number of layers of size byteSize. +func Index(byteSize, layers, count int64) (v1.ImageIndex, error) { + manifest := v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + Manifests: []v1.Descriptor{}, + } + + images := make(map[v1.Hash]v1.Image) + for i := int64(0); i < count; i++ { + img, err := Image(byteSize, layers) + if err != nil { + return nil, err + } + + rawManifest, err := img.RawManifest() + if err != nil { + return nil, err + } + digest, size, err := v1.SHA256(bytes.NewReader(rawManifest)) + if err != nil { + return nil, err + } + mediaType, err := img.MediaType() + if err != nil { + return nil, err + } + + manifest.Manifests = append(manifest.Manifests, v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + }) + + images[digest] = img + } + + return &randomIndex{ + images: images, + manifest: &manifest, + }, nil +} + +func (i *randomIndex) MediaType() (types.MediaType, error) { + return i.manifest.MediaType, nil +} + +func (i *randomIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *randomIndex) Size() (int64, error) { + return partial.Size(i) +} + +func (i *randomIndex) IndexManifest() (*v1.IndexManifest, error) { + return i.manifest, nil +} + +func (i *randomIndex) RawManifest() ([]byte, error) { + m, err := i.IndexManifest() + if err != nil { + return nil, err + } + return json.Marshal(m) +} + +func (i *randomIndex) Image(h v1.Hash) (v1.Image, error) { + if img, ok := i.images[h]; ok { + return img, nil + } + + return nil, fmt.Errorf("image not found: %v", h) +} + +func (i *randomIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // This is a single level index (for now?). + return nil, fmt.Errorf("image not found: %v", h) +} diff --git a/pkg/v1/random/index_test.go b/pkg/v1/random/index_test.go new file mode 100644 index 0000000..73e744b --- /dev/null +++ b/pkg/v1/random/index_test.go @@ -0,0 +1,64 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package random + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestRandomIndex(t *testing.T) { + ii, err := Index(1024, 5, 3) + if err != nil { + t.Fatalf("Error loading index: %v", err) + } + + if err := validate.Index(ii); err != nil { + t.Errorf("validate.Index() = %v", err) + } + + digest, err := ii.Digest() + if err != nil { + t.Fatalf("Digest(): unexpected err: %v", err) + } + + if _, err := ii.Image(digest); err == nil { + t.Errorf("Image(%s): expected err, got nil", digest) + } + + if _, err := ii.ImageIndex(digest); err == nil { + t.Errorf("ImageIndex(%s): expected err, got nil", digest) + } + + mt, err := ii.MediaType() + if err != nil { + t.Errorf("MediaType(): unexpected err: %v", err) + } + + if got, want := mt, types.OCIImageIndex; got != want { + t.Errorf("MediaType(): got: %v, want: %v", got, want) + } + + man, err := ii.IndexManifest() + if err != nil { + t.Errorf("IndexManifest(): unexpected err: %v", err) + } + + if got, want := man.MediaType, types.OCIImageIndex; got != want { + t.Errorf("MediaType: got: %v, want: %v", got, want) + } +} diff --git a/pkg/v1/remote/README.md b/pkg/v1/remote/README.md new file mode 100644 index 0000000..c1e81b3 --- /dev/null +++ b/pkg/v1/remote/README.md @@ -0,0 +1,117 @@ +# `remote` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote) + +The `remote` package implements a client for accessing a registry, +per the [OCI distribution spec](https://github.com/opencontainers/distribution-spec/blob/master/spec.md). + +It leans heavily on the lower level [`transport`](/pkg/v1/remote/transport) package, which handles the +authentication handshake and structured errors. + +## Usage + +```go +package main + +import ( + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func main() { + ref, err := name.ParseReference("gcr.io/google-containers/pause") + if err != nil { + panic(err) + } + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + panic(err) + } + + // do stuff with img +} +``` + +## Structure + +

+ +

+ + +## Background + +There are a lot of confusingly similar terms that come up when talking about images in registries. + +### Anatomy of an image + +In general... + +* A tag refers to an image manifest. +* An image manifest references a config file and an orderered list of _compressed_ layers by sha256 digest. +* A config file references an ordered list of _uncompressed_ layers by sha256 digest and contains runtime configuration. +* The sha256 digest of the config file is the [image id](https://github.com/opencontainers/image-spec/blob/master/config.md#imageid) for the image. + +For example, an image with two layers would look something like this: + +![image anatomy](/images/image-anatomy.dot.svg) + +### Anatomy of an index + +In the normal case, an [index](https://github.com/opencontainers/image-spec/blob/master/image-index.md) is used to represent a multi-platform image. +This was the original use case for a [manifest +list](https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list). + +![image index anatomy](/images/index-anatomy.dot.svg) + +It is possible for an index to reference another index, per the OCI +[image-spec](https://github.com/opencontainers/image-spec/blob/master/media-types.md#compatibility-matrix). +In theory, both an image and image index can reference arbitrary things via +[descriptors](https://github.com/opencontainers/image-spec/blob/master/descriptor.md), +e.g. see the [image layout +example](https://github.com/opencontainers/image-spec/blob/master/image-layout.md#index-example), +which references an application/xml file from an image index. + +That could look something like this: + +![strange image index anatomy](/images/index-anatomy-strange.dot.svg) + +Using a recursive index like this might not be possible with all registries, +but this flexibility allows for some interesting applications, e.g. the +[OCI Artifacts](https://github.com/opencontainers/artifacts) effort. + +### Anatomy of an image upload + +The structure of an image requires a delicate ordering when uploading an image to a registry. +Below is a (slightly simplified) figure that describes how an image is prepared for upload +to a registry and how the data flows between various artifacts: + +![upload](/images/upload.dot.svg) + +Note that: + +* A config file references the uncompressed layer contents by sha256. +* A manifest references the compressed layer contents by sha256 and the size of the layer. +* A manifest references the config file contents by sha256 and the size of the file. + +It follows that during an upload, we need to upload layers before the config file, +and we need to upload the config file before the manifest. + +Sometimes, we know all of this information ahead of time, (e.g. when copying from remote.Image), +so the ordering is less important. + +In other cases, e.g. when using a [`stream.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream#Layer), +we can't compute anything until we have already uploaded the layer, so we need to be careful about ordering. + +## Caveats + +### schema 1 + +This package does not support schema 1 images, see [`#377`](https://github.com/google/go-containerregistry/issues/377), +however, it's possible to do _something_ useful with them via [`remote.Get`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Get), +which doesn't try to interpret what is returned by the registry. + +[`crane.Copy`](https://godoc.org/github.com/google/go-containerregistry/pkg/crane#Copy) takes advantage of this to implement support for copying schema 1 images, +see [here](https://github.com/google/go-containerregistry/blob/main/pkg/internal/legacy/copy.go). diff --git a/pkg/v1/remote/catalog.go b/pkg/v1/remote/catalog.go new file mode 100644 index 0000000..eb4306f --- /dev/null +++ b/pkg/v1/remote/catalog.go @@ -0,0 +1,154 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +type catalog struct { + Repos []string `json:"repositories"` +} + +// CatalogPage calls /_catalog, returning the list of repositories on the registry. +func CatalogPage(target name.Registry, last string, n int, options ...Option) ([]string, error) { + o, err := makeOptions(target, options...) + if err != nil { + return nil, err + } + + scopes := []string{target.Scope(transport.PullScope)} + tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes) + if err != nil { + return nil, err + } + + query := fmt.Sprintf("last=%s&n=%d", url.QueryEscape(last), n) + + uri := url.URL{ + Scheme: target.Scheme(), + Host: target.RegistryStr(), + Path: "/v2/_catalog", + RawQuery: query, + } + + client := http.Client{Transport: tr} + req, err := http.NewRequest(http.MethodGet, uri.String(), nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req.WithContext(o.context)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, err + } + + var parsed catalog + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, err + } + + return parsed.Repos, nil +} + +// Catalog calls /_catalog, returning the list of repositories on the registry. +func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]string, error) { + o, err := makeOptions(target, options...) + if err != nil { + return nil, err + } + + scopes := []string{target.Scope(transport.PullScope)} + tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes) + if err != nil { + return nil, err + } + + uri := &url.URL{ + Scheme: target.Scheme(), + Host: target.RegistryStr(), + Path: "/v2/_catalog", + } + + if o.pageSize > 0 { + uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize) + } + + client := http.Client{Transport: tr} + + // WithContext overrides the ctx passed directly. + if o.context != context.Background() { + ctx = o.context + } + + var ( + parsed catalog + repoList []string + ) + + // get responses until there is no next page + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + req, err := http.NewRequest("GET", uri.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, err + } + + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, err + } + if err := resp.Body.Close(); err != nil { + return nil, err + } + + repoList = append(repoList, parsed.Repos...) + + uri, err = getNextPageURL(resp) + if err != nil { + return nil, err + } + // no next page + if uri == nil { + break + } + } + return repoList, nil +} diff --git a/pkg/v1/remote/catalog_test.go b/pkg/v1/remote/catalog_test.go new file mode 100644 index 0000000..0a90bf6 --- /dev/null +++ b/pkg/v1/remote/catalog_test.go @@ -0,0 +1,183 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/name" +) + +func TestCatalogPage(t *testing.T) { + cases := []struct { + name string + responseBody []byte + wantErr bool + wantRepos []string + }{{ + name: "success", + responseBody: []byte(`{"repositories":["test/test","foo/bar"]}`), + wantErr: false, + wantRepos: []string{"test/test", "foo/bar"}, + }, { + name: "not json", + responseBody: []byte("notjson"), + wantErr: true, + }} + // TODO: add test cases for pagination + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + catalogPath := "/v2/_catalog" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case catalogPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Write(tc.responseBody) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + reg, err := name.NewRegistry(u.Host) + if err != nil { + t.Fatalf("name.NewRegistry(%v) = %v", u.Host, err) + } + + repos, err := CatalogPage(reg, "", 100) + if (err != nil) != tc.wantErr { + t.Errorf("CatalogPage() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + + if diff := cmp.Diff(tc.wantRepos, repos); diff != "" { + t.Errorf("CatalogPage() wrong repos (-want +got) = %s", diff) + } + }) + } +} + +func TestCatalog(t *testing.T) { + cases := []struct { + name string + pages [][]byte + wantErr bool + wantRepos []string + }{{ + name: "success", + pages: [][]byte{ + []byte(`{"repositories":["test/one","test/two"]}`), + []byte(`{"repositories":["test/three","test/four"]}`), + }, + wantErr: false, + wantRepos: []string{"test/one", "test/two", "test/three", "test/four"}, + }, { + name: "not json", + pages: [][]byte{[]byte("notjson")}, + wantErr: true, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + catalogPath := "/v2/_catalog" + pageTwo := "/v2/_catalog_two" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := 0 + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case pageTwo: + page = 1 + fallthrough + case catalogPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + if page == 0 { + w.Header().Set("Link", fmt.Sprintf("<%s>", pageTwo)) + } + w.Write(tc.pages[page]) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + reg, err := name.NewRegistry(u.Host) + if err != nil { + t.Fatalf("name.NewRegistry(%v) = %v", u.Host, err) + } + + repos, err := Catalog(context.Background(), reg) + if (err != nil) != tc.wantErr { + t.Errorf("Catalog() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + + if diff := cmp.Diff(tc.wantRepos, repos); diff != "" { + t.Errorf("Catalog() wrong repos (-want +got) = %s", diff) + } + }) + } +} + +func TestCancelledCatalog(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + reg, err := name.NewRegistry(u.Host) + if err != nil { + t.Fatalf("name.NewRegistry(%v) = %v", u.Host, err) + } + + _, err = Catalog(ctx, reg) + if want, got := context.Canceled, err; !errors.Is(got, want) { + t.Errorf("wanted %v got %v", want, got) + } +} diff --git a/pkg/v1/remote/check.go b/pkg/v1/remote/check.go new file mode 100644 index 0000000..b4395c2 --- /dev/null +++ b/pkg/v1/remote/check.go @@ -0,0 +1,72 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "fmt" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// CheckPushPermission returns an error if the given keychain cannot authorize +// a push operation to the given ref. +// +// This can be useful to check whether the caller has permission to push an +// image before doing work to construct the image. +// +// TODO(#412): Remove the need for this method. +func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { + auth, err := kc.Resolve(ref.Context().Registry) + if err != nil { + return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context().Registry, err) + } + + scopes := []string{ref.Scope(transport.PushScope)} + tr, err := transport.NewWithContext(context.TODO(), ref.Context().Registry, auth, t, scopes) + if err != nil { + return fmt.Errorf("creating push check transport for %v failed: %w", ref.Context().Registry, err) + } + // TODO(jasonhall): Against GCR, just doing the token handshake is + // enough, but this doesn't extend to Dockerhub + // (https://github.com/docker/hub-feedback/issues/1771), so we actually + // need to initiate an upload to tell whether the credentials can + // authorize a push. Figure out how to return early here when we can, + // to avoid a roundtrip for spec-compliant registries. + w := writer{ + repo: ref.Context(), + client: &http.Client{Transport: tr}, + } + loc, _, err := w.initiateUpload(context.Background(), "", "", "") + if loc != "" { + // Since we're only initiating the upload to check whether we + // can, we should attempt to cancel it, in case initiating + // reserves some resources on the server. We shouldn't wait for + // cancelling to complete, and we don't care if it fails. + go w.cancelUpload(loc) + } + return err +} + +func (w *writer) cancelUpload(loc string) { + req, err := http.NewRequest(http.MethodDelete, loc, nil) + if err != nil { + return + } + _, _ = w.client.Do(req) +} diff --git a/pkg/v1/remote/check_e2e_test.go b/pkg/v1/remote/check_e2e_test.go new file mode 100644 index 0000000..a302230 --- /dev/null +++ b/pkg/v1/remote/check_e2e_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration +// +build integration + +package remote + +import ( + "net/http" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" +) + +func TestCheckPushPermission_Real(t *testing.T) { + // Tests should not run in an environment where these registries can + // be pushed to. + for _, r := range []name.Reference{ + name.MustParseReference("ubuntu"), + name.MustParseReference("google/cloud-sdk"), + name.MustParseReference("microsoft/dotnet:sdk"), + name.MustParseReference("gcr.io/non-existent-project/made-up"), + name.MustParseReference("gcr.io/google-containers/foo"), + name.MustParseReference("quay.io/username/reponame"), + } { + t.Run(r.String(), func(t *testing.T) { + t.Parallel() + if err := CheckPushPermission(r, authn.DefaultKeychain, http.DefaultTransport); err == nil { + t.Errorf("CheckPushPermission(%s) returned nil", r) + } + }) + } +} diff --git a/pkg/v1/remote/check_test.go b/pkg/v1/remote/check_test.go new file mode 100644 index 0000000..2f76e12 --- /dev/null +++ b/pkg/v1/remote/check_test.go @@ -0,0 +1,76 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" +) + +func TestCheckPushPermission(t *testing.T) { + for _, c := range []struct { + status int + wantErr bool + }{{ + http.StatusCreated, + false, + }, { + http.StatusAccepted, + false, + }, { + http.StatusForbidden, + true, + }, { + http.StatusBadRequest, + true, + }} { + expectedRepo := "write/time" + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + somewhereElse := fmt.Sprintf("/v2/%s/blobs/uploads/somewhere/else", expectedRepo) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + w.Header().Set("Location", "somewhere/else") + http.Error(w, "", c.status) + case somewhereElse: + if r.Method != http.MethodDelete { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodDelete) + } + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + ref := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + if err := CheckPushPermission(ref, authn.DefaultKeychain, http.DefaultTransport); (err != nil) != c.wantErr { + t.Errorf("CheckPermission(%d): got error = %v, want err = %t", c.status, err, c.wantErr) + } + } +} diff --git a/pkg/v1/remote/delete.go b/pkg/v1/remote/delete.go new file mode 100644 index 0000000..74a06fd --- /dev/null +++ b/pkg/v1/remote/delete.go @@ -0,0 +1,61 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// Delete removes the specified image reference from the remote registry. +func Delete(ref name.Reference, options ...Option) error { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + scopes := []string{ref.Scope(transport.DeleteScope)} + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + c := &http.Client{Transport: tr} + + u := url.URL{ + Scheme: ref.Context().Registry.Scheme(), + Host: ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/manifests/%s", ref.Context().RepositoryStr(), ref.Identifier()), + } + + req, err := http.NewRequest(http.MethodDelete, u.String(), nil) + if err != nil { + return err + } + + resp, err := c.Do(req.WithContext(o.context)) + if err != nil { + return err + } + defer resp.Body.Close() + + return transport.CheckError(resp, http.StatusOK, http.StatusAccepted) + + // TODO(jason): If the manifest had a `subject`, and if the registry + // doesn't support Referrers, update the index pointed to by the + // subject's fallback tag to remove the descriptor for this manifest. +} diff --git a/pkg/v1/remote/delete_test.go b/pkg/v1/remote/delete_test.go new file mode 100644 index 0000000..4918e16 --- /dev/null +++ b/pkg/v1/remote/delete_test.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-containerregistry/pkg/name" +) + +func TestDelete(t *testing.T) { + expectedRepo := "write/time" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodDelete { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodDelete) + } + http.Error(w, "Deleted", http.StatusOK) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := Delete(tag); err != nil { + t.Errorf("Delete() = %v", err) + } +} + +func TestDeleteBadStatus(t *testing.T) { + expectedRepo := "write/time" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodDelete { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodDelete) + } + http.Error(w, "Boom Goes Server", http.StatusInternalServerError) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := Delete(tag); err == nil { + t.Error("Delete() = nil; wanted error") + } +} diff --git a/pkg/v1/remote/descriptor.go b/pkg/v1/remote/descriptor.go new file mode 100644 index 0000000..78919d7 --- /dev/null +++ b/pkg/v1/remote/descriptor.go @@ -0,0 +1,511 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/verify" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +type ErrSchema1 struct { + schema string +} + +// newErrSchema1 returns an ErrSchema1 with the unexpected MediaType. +func newErrSchema1(schema types.MediaType) error { + return &ErrSchema1{ + schema: string(schema), + } +} + +// Error implements error. +func (e *ErrSchema1) Error() string { + return fmt.Sprintf("unsupported MediaType: %q, see https://github.com/google/go-containerregistry/issues/377", e.schema) +} + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image. + platform v1.Platform +} + +// RawManifest exists to satisfy the Taggable interface. +func (d *Descriptor) RawManifest() ([]byte, error) { + return d.Manifest, nil +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +// +// See Head if you don't need the response body. +func Get(ref name.Reference, options ...Option) (*Descriptor, error) { + acceptable := []types.MediaType{ + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + acceptable = append(acceptable, acceptableImageMediaTypes...) + acceptable = append(acceptable, acceptableIndexMediaTypes...) + return get(ref, acceptable, options...) +} + +// Head returns a v1.Descriptor for the given reference by issuing a HEAD +// request. +// +// Note that the server response will not have a body, so any errors encountered +// should be retried with Get to get more details. +func Head(ref name.Reference, options ...Option) (*v1.Descriptor, error) { + acceptable := []types.MediaType{ + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + acceptable = append(acceptable, acceptableImageMediaTypes...) + acceptable = append(acceptable, acceptableIndexMediaTypes...) + + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return nil, err + } + + f, err := makeFetcher(ref, o) + if err != nil { + return nil, err + } + + return f.headManifest(ref, acceptable) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return nil, err + } + f, err := makeFetcher(ref, o) + if err != nil { + return nil, err + } + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + return &Descriptor{ + fetcher: *f, + Manifest: b, + Descriptor: *desc, + platform: o.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, newErrSchema1(d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + logs.Warn.Printf("Unexpected media type for Image(): %s", d.MediaType) + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, newErrSchema1(d.MediaType) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + logs.Warn.Printf("Unexpected media type for ImageIndex(): %s", d.MediaType) + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: d.fetcher, + manifest: d.Manifest, + mediaType: d.MediaType, + descriptor: &d.Descriptor, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: d.fetcher, + manifest: d.Manifest, + mediaType: d.MediaType, + descriptor: &d.Descriptor, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client + context context.Context +} + +func makeFetcher(ref name.Reference, o *options) (*fetcher, error) { + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + return &fetcher{ + Ref: ref, + Client: &http.Client{Transport: tr}, + context: o.context, + }, nil +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#referrers-tag-schema +func fallbackTag(d name.Digest) name.Tag { + return d.Context().Tag(strings.Replace(d.DigestStr(), ":", "-", 1)) +} + +func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, d name.Digest) (*v1.IndexManifest, error) { + // Check the Referrers API endpoint first. + u := f.url("referrers", d.DigestStr()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", string(types.OCIImageIndex)) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound, http.StatusBadRequest); err != nil { + return nil, err + } + if resp.StatusCode == http.StatusOK { + var im v1.IndexManifest + if err := json.NewDecoder(resp.Body).Decode(&im); err != nil { + return nil, err + } + return filterReferrersResponse(filter, &im), nil + } + + // The registry doesn't support the Referrers API endpoint, so we'll use the fallback tag scheme. + b, _, err := f.fetchManifest(fallbackTag(d), []types.MediaType{types.OCIImageIndex}) + if err != nil { + return nil, err + } + var terr *transport.Error + if ok := errors.As(err, &terr); ok && terr.StatusCode == http.StatusNotFound { + // Not found just means there are no attachments yet. Start with an empty manifest. + return &v1.IndexManifest{MediaType: types.OCIImageIndex}, nil + } + + var im v1.IndexManifest + if err := json.Unmarshal(b, &im); err != nil { + return nil, err + } + + return filterReferrersResponse(filter, &im), nil +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req.WithContext(f.context)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + if err == nil && mediaType == types.DockerManifestSchema1Signed { + // If we can parse the digest from the header, and it's a signed schema 1 + // manifest, let's use that for the digest to appease older registries. + digest = contentDigest + } + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } + + var artifactType string + mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) + // Failing to parse as a manifest should just be ignored. + // The manifest might not be valid, and that's okay. + if mf != nil && !mf.Config.MediaType.IsConfig() { + artifactType = string(mf.Config.MediaType) + } + + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/GoogleContainerTools/kaniko/issues/298 + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + ArtifactType: artifactType, + } + + return manifest, &desc, nil +} + +func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType) (*v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodHead, u.String(), nil) + if err != nil { + return nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req.WithContext(f.context)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, err + } + + mth := resp.Header.Get("Content-Type") + if mth == "" { + return nil, fmt.Errorf("HEAD %s: response did not include Content-Type header", u.String()) + } + mediaType := types.MediaType(mth) + + size := resp.ContentLength + if size == -1 { + return nil, fmt.Errorf("GET %s: response did not include Content-Length header", u.String()) + } + + dh := resp.Header.Get("Docker-Content-Digest") + if dh == "" { + return nil, fmt.Errorf("HEAD %s: response did not include Docker-Content-Digest header", u.String()) + } + digest, err := v1.NewHash(dh) + if err != nil { + return nil, err + } + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if digest.String() != dgst.DigestStr() { + return nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } + + // Return all this info since we have to calculate it anyway. + return &v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + }, nil +} + +func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) { + u := f.url("blobs", h.String()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := f.Client.Do(req.WithContext(ctx)) + if err != nil { + return nil, redact.Error(err) + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + resp.Body.Close() + return nil, err + } + + // Do whatever we can. + // If we have an expected size and Content-Length doesn't match, return an error. + // If we don't have an expected size and we do have a Content-Length, use Content-Length. + if hsize := resp.ContentLength; hsize != -1 { + if size == verify.SizeUnknown { + size = hsize + } else if hsize != size { + return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size) + } + } + + return verify.ReadCloser(resp.Body, size, h) +} + +func (f *fetcher) headBlob(h v1.Hash) (*http.Response, error) { + u := f.url("blobs", h.String()) + req, err := http.NewRequest(http.MethodHead, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := f.Client.Do(req.WithContext(f.context)) + if err != nil { + return nil, redact.Error(err) + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + resp.Body.Close() + return nil, err + } + + return resp, nil +} + +func (f *fetcher) blobExists(h v1.Hash) (bool, error) { + u := f.url("blobs", h.String()) + req, err := http.NewRequest(http.MethodHead, u.String(), nil) + if err != nil { + return false, err + } + + resp, err := f.Client.Do(req.WithContext(f.context)) + if err != nil { + return false, redact.Error(err) + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil { + return false, err + } + + return resp.StatusCode == http.StatusOK, nil +} + +// If filter applied, filter out by artifactType. +// See https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +func filterReferrersResponse(filter map[string]string, origIndex *v1.IndexManifest) *v1.IndexManifest { + newIndex := origIndex + if filter == nil { + return newIndex + } + if v, ok := filter["artifactType"]; ok { + tmp := []v1.Descriptor{} + for _, desc := range newIndex.Manifests { + if desc.ArtifactType == v { + tmp = append(tmp, desc) + } + } + newIndex.Manifests = tmp + } + return newIndex +} diff --git a/pkg/v1/remote/descriptor_test.go b/pkg/v1/remote/descriptor_test.go new file mode 100644 index 0000000..1b77f80 --- /dev/null +++ b/pkg/v1/remote/descriptor_test.go @@ -0,0 +1,259 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestGetSchema1(t *testing.T) { + expectedRepo := "foo/bar" + fakeDigest := "sha256:0000000000000000000000000000000000000000000000000000000000000000" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Header().Set("Content-Type", string(types.DockerManifestSchema1Signed)) + w.Header().Set("Docker-Content-Digest", fakeDigest) + w.Write([]byte("doesn't matter")) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + + // Get should succeed even for invalid json. We don't parse the response. + desc, err := Get(tag) + if err != nil { + t.Fatalf("Get(%s) = %v", tag, err) + } + + if desc.Digest.String() != fakeDigest { + t.Errorf("Descriptor.Digest = %q, expected %q", desc.Digest, fakeDigest) + } + + want := `unsupported MediaType: "application/vnd.docker.distribution.manifest.v1+prettyjws", see https://github.com/google/go-containerregistry/issues/377` + // Should fail based on media type. + if _, err := desc.Image(); err != nil { + if errors.Is(err, &ErrSchema1{}) { + t.Errorf("Image() = %v, expected remote.ErrSchema1", err) + } + if diff := cmp.Diff(want, err.Error()); diff != "" { + t.Errorf("Image() error message (-want +got) = %v", diff) + } + } else { + t.Errorf("Image() = %v, expected err", err) + } + + // Should fail based on media type. + if _, err := desc.ImageIndex(); err != nil { + var s1err ErrSchema1 + if errors.Is(err, &s1err) { + t.Errorf("ImageImage() = %v, expected remote.ErrSchema1", err) + } + } else { + t.Errorf("ImageIndex() = %v, expected err", err) + } +} + +func TestGetImageAsIndex(t *testing.T) { + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Header().Set("Content-Type", string(types.DockerManifestSchema2)) + w.Write([]byte("doesn't matter")) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + + // Get should succeed even for invalid json. We don't parse the response. + desc, err := Get(tag) + if err != nil { + t.Fatalf("Get(%s) = %v", tag, err) + } + + // Should fail based on media type. + if _, err := desc.ImageIndex(); err == nil { + t.Errorf("ImageIndex() = %v, expected err", err) + } +} + +func TestHeadSchema1(t *testing.T) { + expectedRepo := "foo/bar" + mediaType := types.DockerManifestSchema1Signed + fakeDigest := "sha256:0000000000000000000000000000000000000000000000000000000000000000" + response := []byte("doesn't matter") + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodHead { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodHead) + } + w.Header().Set("Content-Type", string(mediaType)) + w.Header().Set("Content-Length", strconv.Itoa(len(response))) + w.Header().Set("Docker-Content-Digest", fakeDigest) + w.Write(response) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + + // Head should succeed even for invalid json. We don't parse the response. + desc, err := Head(tag) + if err != nil { + t.Fatalf("Head(%s) = %v", tag, err) + } + + if desc.MediaType != mediaType { + t.Errorf("Descriptor.MediaType = %q, expected %q", desc.MediaType, mediaType) + } + + if desc.Digest.String() != fakeDigest { + t.Errorf("Descriptor.Digest = %q, expected %q", desc.Digest, fakeDigest) + } + + if desc.Size != int64(len(response)) { + t.Errorf("Descriptor.Size = %q, expected %q", desc.Size, len(response)) + } +} + +// TestHead_MissingHeaders tests that HEAD responses missing necessary headers +// result in errors. +func TestHead_MissingHeaders(t *testing.T) { + missingType := "missing-type" + missingLength := "missing-length" + missingDigest := "missing-digest" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v2/" { + w.WriteHeader(http.StatusOK) + return + } + if r.Method != http.MethodHead { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodHead) + } + if !strings.Contains(r.URL.Path, missingType) { + w.Header().Set("Content-Type", "My-Media-Type") + } + if !strings.Contains(r.URL.Path, missingLength) { + w.Header().Set("Content-Length", "10") + } + if !strings.Contains(r.URL.Path, missingDigest) { + w.Header().Set("Docker-Content-Digest", "sha256:0000000000000000000000000000000000000000000000000000000000000000") + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + for _, repo := range []string{missingType, missingLength, missingDigest} { + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, repo)) + if _, err := Head(tag); err == nil { + t.Errorf("Head(%q): expected error, got nil", tag) + } + } +} + +// TestRedactFetchBlob tests that a request to fetchBlob that gets redirected +// to a URL that contains sensitive information has that information redacted +// if the subsequent request fails. +func TestRedactFetchBlob(t *testing.T) { + ctx := context.Background() + f := fetcher{ + Ref: mustNewTag(t, "original.com/repo:latest"), + Client: &http.Client{ + Transport: errTransport{}, + }, + context: ctx, + } + h, err := v1.NewHash("sha256:0000000000000000000000000000000000000000000000000000000000000000") + if err != nil { + t.Fatal("NewHash:", err) + } + if _, err := f.fetchBlob(ctx, 0, h); err == nil { + t.Fatalf("fetchBlob: expected error, got nil") + } else if !strings.Contains(err.Error(), "access_token=REDACTED") { + t.Fatalf("fetchBlob: expected error to contain redacted access token, got %v", err) + } +} + +type errTransport struct{} + +func (errTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // This simulates a registry that returns a redirect upon the first + // request, and then returns an error upon subsequent requests. This helps + // test whether error redaction takes into account URLs in error messasges + // that are not the original request URL. + if req.URL.Host == "original.com" { + return &http.Response{ + StatusCode: http.StatusSeeOther, + Header: http.Header{"Location": []string{"https://redirected.com?access_token=SECRET"}}, + }, nil + } + return nil, fmt.Errorf("error reaching %s", req.URL.String()) +} diff --git a/pkg/v1/remote/doc.go b/pkg/v1/remote/doc.go new file mode 100644 index 0000000..846ba07 --- /dev/null +++ b/pkg/v1/remote/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package remote provides facilities for reading/writing v1.Images from/to +// a remote image registry. +package remote diff --git a/pkg/v1/remote/error_roundtrip_test.go b/pkg/v1/remote/error_roundtrip_test.go new file mode 100644 index 0000000..5b81ee5 --- /dev/null +++ b/pkg/v1/remote/error_roundtrip_test.go @@ -0,0 +1,127 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote_test + +import ( + "errors" + "fmt" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +func TestStatusCodeReturned(t *testing.T) { + tcs := []struct { + Description string + Handler http.Handler + }{{ + Description: "Only returns teapot status", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTeapot) + }), + }, { + Description: "Handle v2, returns teapot status else", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Print(r.URL.Path) + if r.URL.Path == "/v2/" { + return + } + w.WriteHeader(http.StatusTeapot) + }), + }} + + for _, tc := range tcs { + t.Run(tc.Description, func(t *testing.T) { + o := httptest.NewServer(tc.Handler) + defer o.Close() + + ref, err := name.NewDigest(strings.TrimPrefix(o.URL+"/foo:@sha256:53b27244ffa2f585799adbfaf79fba5a5af104597751b289c8b235e7b8f7ebf5", "http://")) + + if err != nil { + t.Fatalf("Unable to parse digest: %v", err) + } + + _, err = remote.Image(ref) + var terr *transport.Error + if !errors.As(err, &terr) { + t.Fatalf("Unable to cast error to transport error: %v", err) + } + if terr.StatusCode != http.StatusTeapot { + t.Errorf("Incorrect status code received, got %v, wanted %v", terr.StatusCode, http.StatusTeapot) + } + }) + } +} + +func TestBlobStatusCodeReturned(t *testing.T) { + reg := registry.New() + rh := httptest.NewServer(reg) + defer rh.Close() + i, _ := random.Image(1024, 16) + tag := strings.TrimPrefix(fmt.Sprintf("%s/foo:bar", rh.URL), "http://") + d, _ := name.NewTag(tag) + if err := remote.Write(d, i); err != nil { + t.Fatalf("Unable to write empty image: %v", err) + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Print(r.URL.Path) + if strings.Contains(r.URL.Path, "blob") { + w.WriteHeader(http.StatusTeapot) + return + } + reg.ServeHTTP(w, r) + }) + + o := httptest.NewServer(handler) + defer o.Close() + + ref, err := name.NewTag(strings.TrimPrefix(fmt.Sprintf("%s/foo:bar", o.URL), "http://")) + if err != nil { + t.Fatalf("Unable to parse digest: %v", err) + } + + ri, err := remote.Image(ref) + if err != nil { + t.Fatalf("Unable to fetch manifest: %v", err) + } + l, err := ri.Layers() + if err != nil { + t.Fatalf("Unable to fetch layers: %v", err) + } + _, err = l[0].Compressed() + var terr *transport.Error + if !errors.As(err, &terr) { + t.Fatalf("Unable to cast error to transport error: %v", err) + } + if terr.StatusCode != http.StatusTeapot { + t.Errorf("Incorrect status code received, got %v, wanted %v", terr.StatusCode, http.StatusTeapot) + } + _, err = l[0].Uncompressed() + if !errors.As(err, &terr) { + t.Fatalf("Unable to cast error to transport error: %v", err) + } + if terr.StatusCode != http.StatusTeapot { + t.Errorf("Incorrect status code received, got %v, wanted %v", terr.StatusCode, http.StatusTeapot) + } +} diff --git a/pkg/v1/remote/image.go b/pkg/v1/remote/image.go new file mode 100644 index 0000000..fde6142 --- /dev/null +++ b/pkg/v1/remote/image.go @@ -0,0 +1,256 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "io" + "net/http" + "net/url" + "sync" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/verify" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var acceptableImageMediaTypes = []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, +} + +// remoteImage accesses an image from a remote registry +type remoteImage struct { + fetcher + manifestLock sync.Mutex // Protects manifest + manifest []byte + configLock sync.Mutex // Protects config + config []byte + mediaType types.MediaType + descriptor *v1.Descriptor +} + +func (r *remoteImage) ArtifactType() (string, error) { + // kind of a hack, but RawManifest does appropriate locking/memoization + // and makes sure r.descriptor is populated. + if _, err := r.RawManifest(); err != nil { + return "", err + } + return r.descriptor.ArtifactType, nil +} + +var _ partial.CompressedImageCore = (*remoteImage)(nil) + +// Image provides access to a remote image reference. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + desc, err := Get(ref, options...) + if err != nil { + return nil, err + } + + return desc.Image() +} + +func (r *remoteImage) MediaType() (types.MediaType, error) { + if string(r.mediaType) != "" { + return r.mediaType, nil + } + return types.DockerManifestSchema2, nil +} + +func (r *remoteImage) RawManifest() ([]byte, error) { + r.manifestLock.Lock() + defer r.manifestLock.Unlock() + if r.manifest != nil { + return r.manifest, nil + } + + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. + manifest, desc, err := r.fetchManifest(r.Ref, acceptableImageMediaTypes) + if err != nil { + return nil, err + } + + if r.descriptor == nil { + r.descriptor = desc + } + r.mediaType = desc.MediaType + r.manifest = manifest + return r.manifest, nil +} + +func (r *remoteImage) RawConfigFile() ([]byte, error) { + r.configLock.Lock() + defer r.configLock.Unlock() + if r.config != nil { + return r.config, nil + } + + m, err := partial.Manifest(r) + if err != nil { + return nil, err + } + + if m.Config.Data != nil { + if err := verify.Descriptor(m.Config); err != nil { + return nil, err + } + r.config = m.Config.Data + return r.config, nil + } + + body, err := r.fetchBlob(r.context, m.Config.Size, m.Config.Digest) + if err != nil { + return nil, err + } + defer body.Close() + + r.config, err = io.ReadAll(body) + if err != nil { + return nil, err + } + return r.config, nil +} + +// Descriptor retains the original descriptor from an index manifest. +// See partial.Descriptor. +func (r *remoteImage) Descriptor() (*v1.Descriptor, error) { + // kind of a hack, but RawManifest does appropriate locking/memoization + // and makes sure r.descriptor is populated. + _, err := r.RawManifest() + return r.descriptor, err +} + +// remoteImageLayer implements partial.CompressedLayer +type remoteImageLayer struct { + ri *remoteImage + digest v1.Hash +} + +// Digest implements partial.CompressedLayer +func (rl *remoteImageLayer) Digest() (v1.Hash, error) { + return rl.digest, nil +} + +// Compressed implements partial.CompressedLayer +func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { + urls := []url.URL{rl.ri.url("blobs", rl.digest.String())} + + // Add alternative layer sources from URLs (usually none). + d, err := partial.BlobDescriptor(rl, rl.digest) + if err != nil { + return nil, err + } + + if d.Data != nil { + return verify.ReadCloser(io.NopCloser(bytes.NewReader(d.Data)), d.Size, d.Digest) + } + + // We don't want to log binary layers -- this can break terminals. + ctx := redact.NewContext(rl.ri.context, "omitting binary blobs from logs") + + for _, s := range d.URLs { + u, err := url.Parse(s) + if err != nil { + return nil, err + } + urls = append(urls, *u) + } + + // The lastErr for most pulls will be the same (the first error), but for + // foreign layers we'll want to surface the last one, since we try to pull + // from the registry first, which would often fail. + // TODO: Maybe we don't want to try pulling from the registry first? + var lastErr error + for _, u := range urls { + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := rl.ri.Client.Do(req.WithContext(ctx)) + if err != nil { + lastErr = err + continue + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + resp.Body.Close() + lastErr = err + continue + } + + return verify.ReadCloser(resp.Body, d.Size, rl.digest) + } + + return nil, lastErr +} + +// Manifest implements partial.WithManifest so that we can use partial.BlobSize below. +func (rl *remoteImageLayer) Manifest() (*v1.Manifest, error) { + return partial.Manifest(rl.ri) +} + +// MediaType implements v1.Layer +func (rl *remoteImageLayer) MediaType() (types.MediaType, error) { + bd, err := partial.BlobDescriptor(rl, rl.digest) + if err != nil { + return "", err + } + + return bd.MediaType, nil +} + +// Size implements partial.CompressedLayer +func (rl *remoteImageLayer) Size() (int64, error) { + // Look up the size of this digest in the manifest to avoid a request. + return partial.BlobSize(rl, rl.digest) +} + +// ConfigFile implements partial.WithManifestAndConfigFile so that we can use partial.BlobToDiffID below. +func (rl *remoteImageLayer) ConfigFile() (*v1.ConfigFile, error) { + return partial.ConfigFile(rl.ri) +} + +// DiffID implements partial.WithDiffID so that we don't recompute a DiffID that we already have +// available in our ConfigFile. +func (rl *remoteImageLayer) DiffID() (v1.Hash, error) { + return partial.BlobToDiffID(rl, rl.digest) +} + +// Descriptor retains the original descriptor from an image manifest. +// See partial.Descriptor. +func (rl *remoteImageLayer) Descriptor() (*v1.Descriptor, error) { + return partial.BlobDescriptor(rl, rl.digest) +} + +// See partial.Exists. +func (rl *remoteImageLayer) Exists() (bool, error) { + return rl.ri.blobExists(rl.digest) +} + +// LayerByDigest implements partial.CompressedLayer +func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + return &remoteImageLayer{ + ri: r, + digest: h, + }, nil +} diff --git a/pkg/v1/remote/image_test.go b/pkg/v1/remote/image_test.go new file mode 100644 index 0000000..4a6c29d --- /dev/null +++ b/pkg/v1/remote/image_test.go @@ -0,0 +1,743 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "path" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +const bogusDigest = "sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +type withDigest interface { + Digest() (v1.Hash, error) +} + +func mustDigest(t *testing.T, img withDigest) v1.Hash { + h, err := img.Digest() + if err != nil { + t.Fatalf("Digest() = %v", err) + } + return h +} + +func mustManifest(t *testing.T, img v1.Image) *v1.Manifest { + m, err := img.Manifest() + if err != nil { + t.Fatalf("Manifest() = %v", err) + } + return m +} + +func mustRawManifest(t *testing.T, img Taggable) []byte { + m, err := img.RawManifest() + if err != nil { + t.Fatalf("RawManifest() = %v", err) + } + return m +} + +func mustRawConfigFile(t *testing.T, img v1.Image) []byte { + c, err := img.RawConfigFile() + if err != nil { + t.Fatalf("RawConfigFile() = %v", err) + } + return c +} + +func randomImage(t *testing.T) v1.Image { + rnd, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + return rnd +} + +func newReference(host, repo, ref string) (name.Reference, error) { + tag, err := name.NewTag(fmt.Sprintf("%s/%s:%s", host, repo, ref), name.WeakValidation) + if err == nil { + return tag, nil + } + return name.NewDigest(fmt.Sprintf("%s/%s@%s", host, repo, ref), name.WeakValidation) +} + +// TODO(jonjohnsonjr): Make this real. +func TestMediaType(t *testing.T) { + img := remoteImage{} + got, err := img.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + want := types.DockerManifestSchema2 + if got != want { + t.Errorf("MediaType() = %v, want %v", got, want) + } +} + +func TestRawManifestDigests(t *testing.T) { + img := randomImage(t) + expectedRepo := "foo/bar" + + cases := []struct { + name string + ref string + responseBody []byte + contentDigest string + wantErr bool + }{{ + name: "normal pull, by tag", + ref: "latest", + responseBody: mustRawManifest(t, img), + contentDigest: mustDigest(t, img).String(), + wantErr: false, + }, { + name: "normal pull, by digest", + ref: mustDigest(t, img).String(), + responseBody: mustRawManifest(t, img), + contentDigest: mustDigest(t, img).String(), + wantErr: false, + }, { + name: "right content-digest, wrong body, by digest", + ref: mustDigest(t, img).String(), + responseBody: []byte("not even json"), + contentDigest: mustDigest(t, img).String(), + wantErr: true, + }, { + name: "right body, wrong content-digest, by tag", + ref: "latest", + responseBody: mustRawManifest(t, img), + contentDigest: bogusDigest, + wantErr: false, + }, { + // NB: This succeeds! We don't care what the registry thinks. + name: "right body, wrong content-digest, by digest", + ref: mustDigest(t, img).String(), + responseBody: mustRawManifest(t, img), + contentDigest: bogusDigest, + wantErr: false, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + manifestPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, tc.ref) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Header().Set("Docker-Content-Digest", tc.contentDigest) + w.Write(tc.responseBody) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + ref, err := newReference(u.Host, expectedRepo, tc.ref) + if err != nil { + t.Fatalf("url.Parse(%v, %v, %v) = %v", u.Host, expectedRepo, tc.ref, err) + } + + rmt := remoteImage{ + fetcher: fetcher{ + Ref: ref, + Client: http.DefaultClient, + context: context.Background(), + }, + } + + if _, err := rmt.RawManifest(); (err != nil) != tc.wantErr { + t.Errorf("RawManifest() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + }) + } +} + +func TestRawManifestNotFound(t *testing.T) { + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.WriteHeader(http.StatusNotFound) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + img := remoteImage{ + fetcher: fetcher{ + Ref: mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)), + Client: http.DefaultClient, + context: context.Background(), + }, + } + + if _, err := img.RawManifest(); err == nil { + t.Error("RawManifest() = nil; wanted error") + } +} + +func TestRawConfigFileNotFound(t *testing.T) { + img := randomImage(t) + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, img)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.WriteHeader(http.StatusNotFound) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, img)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + rmt := remoteImage{ + fetcher: fetcher{ + Ref: mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)), + Client: http.DefaultClient, + context: context.Background(), + }, + } + + if _, err := rmt.RawConfigFile(); err == nil { + t.Error("RawConfigFile() = nil; wanted error") + } +} + +func TestAcceptHeaders(t *testing.T) { + img := randomImage(t) + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + wantAccept := strings.Join([]string{ + string(types.DockerManifestSchema2), + string(types.OCIManifestSchema1), + }, ",") + if got, want := r.Header.Get("Accept"), wantAccept; got != want { + t.Errorf("Accept header; got %v, want %v", got, want) + } + w.Write(mustRawManifest(t, img)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + rmt := &remoteImage{ + fetcher: fetcher{ + Ref: mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)), + Client: http.DefaultClient, + context: context.Background(), + }, + } + manifest, err := rmt.RawManifest() + if err != nil { + t.Errorf("RawManifest() = %v", err) + } + if got, want := manifest, mustRawManifest(t, img); !bytes.Equal(got, want) { + t.Errorf("RawManifest() = %v, want %v", got, want) + } +} + +func TestImage(t *testing.T) { + img := randomImage(t) + expectedRepo := "foo/bar" + layerDigest := mustManifest(t, img).Layers[0].Digest + layerSize := mustManifest(t, img).Layers[0].Size + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, img)) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + layerPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, layerDigest) + manifestReqCount := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawConfigFile(t, img)) + case manifestPath: + manifestReqCount++ + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, img)) + case layerPath: + t.Fatalf("BlobSize should not make any request: %v", r.URL.Path) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + rmt, err := Image(tag, WithTransport(http.DefaultTransport), WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + t.Errorf("Image() = %v", err) + } + + if got, want := mustRawManifest(t, rmt), mustRawManifest(t, img); !bytes.Equal(got, want) { + t.Errorf("RawManifest() = %v, want %v", got, want) + } + if got, want := mustRawConfigFile(t, rmt), mustRawConfigFile(t, img); !bytes.Equal(got, want) { + t.Errorf("RawConfigFile() = %v, want %v", got, want) + } + // Make sure caching the manifest works. + if manifestReqCount != 1 { + t.Errorf("RawManifest made %v requests, expected 1", manifestReqCount) + } + + l, err := rmt.LayerByDigest(layerDigest) + if err != nil { + t.Errorf("LayerByDigest() = %v", err) + } + // BlobSize should not HEAD. + size, err := l.Size() + if err != nil { + t.Errorf("BlobSize() = %v", err) + } + if got, want := size, layerSize; want != got { + t.Errorf("BlobSize() = %v want %v", got, want) + } +} + +func TestPullingManifestList(t *testing.T) { + idx := randomIndex(t) + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + childDigest := mustIndexManifest(t, idx).Manifests[1].Digest + child := mustChild(t, idx, childDigest) + childPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, childDigest) + fakePlatformChildDigest := mustIndexManifest(t, idx).Manifests[0].Digest + fakePlatformChild := mustChild(t, idx, fakePlatformChildDigest) + fakePlatformChildPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, fakePlatformChildDigest) + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, child)) + + fakePlatform := v1.Platform{ + Architecture: "not-real-arch", + OS: "not-real-os", + } + + // Rewrite the index to make sure the desired platform matches the second child. + manifest, err := idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + // Make sure the first manifest doesn't match. + manifest.Manifests[0].Platform = &fakePlatform + // Make sure the second manifest does. + manifest.Manifests[1].Platform = &defaultPlatform + // Do short-circuiting via Data. + manifest.Manifests[1].Data = mustRawManifest(t, child) + rawManifest, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Header().Set("Content-Type", string(mustMediaType(t, idx))) + w.Write(rawManifest) + case childPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, child)) + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawConfigFile(t, child)) + case fakePlatformChildPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, fakePlatformChild)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + rmtChild, err := Image(tag) + if err != nil { + t.Errorf("Image() = %v", err) + } + + // Test that child works as expected. + if got, want := mustRawManifest(t, rmtChild), mustRawManifest(t, child); !bytes.Equal(got, want) { + t.Errorf("RawManifest() = %v, want %v", string(got), string(want)) + } + if got, want := mustRawConfigFile(t, rmtChild), mustRawConfigFile(t, child); !bytes.Equal(got, want) { + t.Errorf("RawConfigFile() = %v, want %v", got, want) + } + + // Make sure we can roundtrip platform info via Descriptor. + img, err := Image(tag, WithPlatform(fakePlatform)) + if err != nil { + t.Fatal(err) + } + desc, err := partial.Descriptor(img) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(*desc.Platform, fakePlatform); diff != "" { + t.Errorf("Desciptor() (-want +got) = %v", diff) + } +} + +func TestPullingManifestListNoMatch(t *testing.T) { + idx := randomIndex(t) + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + childDigest := mustIndexManifest(t, idx).Manifests[1].Digest + child := mustChild(t, idx, childDigest) + childPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, childDigest) + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, child)) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Header().Set("Content-Type", string(mustMediaType(t, idx))) + w.Write(mustRawManifest(t, idx)) + case childPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, child)) + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawConfigFile(t, child)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + platform := v1.Platform{ + Architecture: "not-real-arch", + OS: "not-real-os", + } + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + if _, err := Image(tag, WithPlatform(platform)); err == nil { + t.Errorf("Image succeeded, wanted err") + } +} + +func TestValidate(t *testing.T) { + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + tag, err := name.NewTag(u.Host + "/foo/bar") + if err != nil { + t.Fatal(err) + } + + if err := Write(tag, img); err != nil { + t.Fatal(err) + } + + img, err = Image(tag) + if err != nil { + t.Fatal(err) + } + + if err := validate.Image(img); err != nil { + t.Errorf("failed to validate remote.Image: %v", err) + } +} + +func TestPullingForeignLayer(t *testing.T) { + // For that sweet, sweet coverage in options. + var b bytes.Buffer + logs.Debug.SetOutput(&b) + + img := randomImage(t) + expectedRepo := "foo/bar" + foreignPath := "/foreign/path" + + foreignLayer, err := random.Layer(1024, types.DockerForeignLayer) + if err != nil { + t.Fatal(err) + } + + foreignServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case foreignPath: + compressed, err := foreignLayer.Compressed() + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(w, compressed); err != nil { + t.Fatal(err) + } + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer foreignServer.Close() + fu, err := url.Parse(foreignServer.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", foreignServer.URL, err) + } + + img, err = mutate.Append(img, mutate.Addendum{ + Layer: foreignLayer, + URLs: []string{ + "http://" + path.Join(fu.Host, foreignPath), + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set up a fake registry that will respond 404 to the foreign layer, + // but serve everything else correctly. + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, img)) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + foreignLayerDigest := mustManifest(t, img).Layers[1].Digest + foreignLayerPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, foreignLayerDigest) + layerDigest := mustManifest(t, img).Layers[0].Digest + layerPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, layerDigest) + + layer, err := img.LayerByDigest(layerDigest) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawConfigFile(t, img)) + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, img)) + case layerPath: + compressed, err := layer.Compressed() + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(w, compressed); err != nil { + t.Fatal(err) + } + w.WriteHeader(http.StatusOK) + case foreignLayerPath: + // Not here! + w.WriteHeader(http.StatusNotFound) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + // Pull from the registry and ensure that everything Validates; i.e. that + // we pull the layer from the foreignServer. + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + rmt, err := Image(tag, WithTransport(http.DefaultTransport)) + if err != nil { + t.Errorf("Image() = %v", err) + } + + if err := validate.Image(rmt); err != nil { + t.Errorf("failed to validate foreign image: %v", err) + } + + // Set up a fake registry and write what we pulled to it. + // This ensures we get coverage for the remoteLayer.MediaType path. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err = url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/foreign/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, rmt); err != nil { + t.Errorf("failed to Write: %v", err) + } +} + +func TestData(t *testing.T) { + img := randomImage(t) + manifest, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + cb, err := img.RawConfigFile() + if err != nil { + t.Fatal(err) + } + + manifest.Config.Data = cb + rc, err := layers[0].Compressed() + if err != nil { + t.Fatal(err) + } + lb, err := io.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + manifest.Layers[0].Data = lb + rawManifest, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case "/v2/test/manifests/latest": + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(rawManifest) + default: + // explode if we try to read blob or config + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + ref, err := newReference(u.Host, "test", "latest") + if err != nil { + t.Fatal(err) + } + rmt, err := Image(ref) + if err != nil { + t.Fatal(err) + } + if err := validate.Image(rmt); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/v1/remote/index.go b/pkg/v1/remote/index.go new file mode 100644 index 0000000..0939947 --- /dev/null +++ b/pkg/v1/remote/index.go @@ -0,0 +1,319 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "fmt" + "sync" + + "github.com/google/go-containerregistry/internal/verify" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var acceptableIndexMediaTypes = []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, +} + +// remoteIndex accesses an index from a remote registry +type remoteIndex struct { + fetcher + manifestLock sync.Mutex // Protects manifest + manifest []byte + mediaType types.MediaType + descriptor *v1.Descriptor +} + +// Index provides access to a remote index reference. +func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) { + desc, err := get(ref, acceptableIndexMediaTypes, options...) + if err != nil { + return nil, err + } + + return desc.ImageIndex() +} + +func (r *remoteIndex) MediaType() (types.MediaType, error) { + if string(r.mediaType) != "" { + return r.mediaType, nil + } + return types.DockerManifestList, nil +} + +func (r *remoteIndex) Digest() (v1.Hash, error) { + return partial.Digest(r) +} + +func (r *remoteIndex) Size() (int64, error) { + return partial.Size(r) +} + +func (r *remoteIndex) RawManifest() ([]byte, error) { + r.manifestLock.Lock() + defer r.manifestLock.Unlock() + if r.manifest != nil { + return r.manifest, nil + } + + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. + manifest, desc, err := r.fetchManifest(r.Ref, acceptableIndexMediaTypes) + if err != nil { + return nil, err + } + + if r.descriptor == nil { + r.descriptor = desc + } + r.mediaType = desc.MediaType + r.manifest = manifest + return r.manifest, nil +} + +func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { + b, err := r.RawManifest() + if err != nil { + return nil, err + } + return v1.ParseIndexManifest(bytes.NewReader(b)) +} + +func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { + desc, err := r.childByHash(h) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// Descriptor retains the original descriptor from an index manifest. +// See partial.Descriptor. +func (r *remoteIndex) Descriptor() (*v1.Descriptor, error) { + // kind of a hack, but RawManifest does appropriate locking/memoization + // and makes sure r.descriptor is populated. + _, err := r.RawManifest() + return r.descriptor, err +} + +func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + desc, err := r.childByHash(h) + if err != nil { + return nil, err + } + return desc.ImageIndex() +} + +// Workaround for #819. +func (r *remoteIndex) Layer(h v1.Hash) (v1.Layer, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + l, err := partial.CompressedToLayer(&remoteLayer{ + fetcher: r.fetcher, + digest: h, + }) + if err != nil { + return nil, err + } + return &MountableLayer{ + Layer: l, + Reference: r.Ref.Context().Digest(h.String()), + }, nil + } + } + return nil, fmt.Errorf("layer not found: %s", h) +} + +// Experiment with a better API for v1.ImageIndex. We might want to move this +// to partial? +func (r *remoteIndex) Manifests() ([]partial.Describable, error) { + m, err := r.IndexManifest() + if err != nil { + return nil, err + } + manifests := []partial.Describable{} + for _, desc := range m.Manifests { + switch { + case desc.MediaType.IsImage(): + img, err := r.Image(desc.Digest) + if err != nil { + return nil, err + } + manifests = append(manifests, img) + case desc.MediaType.IsIndex(): + idx, err := r.ImageIndex(desc.Digest) + if err != nil { + return nil, err + } + manifests = append(manifests, idx) + default: + layer, err := r.Layer(desc.Digest) + if err != nil { + return nil, err + } + manifests = append(manifests, layer) + } + } + + return manifests, nil +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// This naively matches the first manifest with matching platform attributes. +// +// We should probably use this instead: +// +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if matchesPlatform(p, platform) { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %+v in index %s", platform, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref := r.Ref.Context().Digest(child.Digest.String()) + var ( + manifest []byte + err error + ) + if child.Data != nil { + if err := verify.Descriptor(child); err != nil { + return nil, err + } + manifest = child.Data + } else { + manifest, _, err = r.fetchManifest(ref, []types.MediaType{child.MediaType}) + if err != nil { + return nil, err + } + } + + if child.MediaType.IsImage() { + mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) + // Failing to parse as a manifest should just be ignored. + // The manifest might not be valid, and that's okay. + if mf != nil && !mf.Config.MediaType.IsConfig() { + child.ArtifactType = string(mf.Config.MediaType) + } + } + + return &Descriptor{ + fetcher: fetcher{ + Ref: ref, + Client: r.Client, + context: r.context, + }, + Manifest: manifest, + Descriptor: child, + platform: platform, + }, nil +} + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} diff --git a/pkg/v1/remote/index_test.go b/pkg/v1/remote/index_test.go new file mode 100644 index 0000000..4399b16 --- /dev/null +++ b/pkg/v1/remote/index_test.go @@ -0,0 +1,504 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func randomIndex(t *testing.T) v1.ImageIndex { + rnd, err := random.Index(1024, 1, 3) + if err != nil { + t.Fatalf("random.Index() = %v", err) + } + return rnd +} + +func mustIndexManifest(t *testing.T, idx v1.ImageIndex) *v1.IndexManifest { + m, err := idx.IndexManifest() + if err != nil { + t.Fatalf("IndexManifest() = %v", err) + } + return m +} + +func mustChild(t *testing.T, idx v1.ImageIndex, h v1.Hash) v1.Image { + img, err := idx.Image(h) + if err != nil { + t.Fatalf("Image(%s) = %v", h, err) + } + return img +} + +func mustMediaType(t *testing.T, tag withMediaType) types.MediaType { + mt, err := tag.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + return mt +} + +func mustHash(t *testing.T, s string) v1.Hash { + h, err := v1.NewHash(s) + if err != nil { + t.Fatalf("NewHash() = %v", err) + } + return h +} + +func TestIndexRawManifestDigests(t *testing.T) { + idx := randomIndex(t) + expectedRepo := "foo/bar" + + cases := []struct { + name string + ref string + responseBody []byte + contentDigest string + wantErr bool + }{{ + name: "normal pull, by tag", + ref: "latest", + responseBody: mustRawManifest(t, idx), + contentDigest: mustDigest(t, idx).String(), + wantErr: false, + }, { + name: "normal pull, by digest", + ref: mustDigest(t, idx).String(), + responseBody: mustRawManifest(t, idx), + contentDigest: mustDigest(t, idx).String(), + wantErr: false, + }, { + name: "right content-digest, wrong body, by digest", + ref: mustDigest(t, idx).String(), + responseBody: []byte("not even json"), + contentDigest: mustDigest(t, idx).String(), + wantErr: true, + }, { + name: "right body, wrong content-digest, by tag", + ref: "latest", + responseBody: mustRawManifest(t, idx), + contentDigest: bogusDigest, + wantErr: false, + }, { + // NB: This succeeds! We don't care what the registry thinks. + name: "right body, wrong content-digest, by digest", + ref: mustDigest(t, idx).String(), + responseBody: mustRawManifest(t, idx), + contentDigest: bogusDigest, + wantErr: false, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + manifestPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, tc.ref) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manifestPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Header().Set("Docker-Content-Digest", tc.contentDigest) + w.Write(tc.responseBody) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + ref, err := newReference(u.Host, expectedRepo, tc.ref) + if err != nil { + t.Fatalf("url.Parse(%v, %v, %v) = %v", u.Host, expectedRepo, tc.ref, err) + } + + rmt := remoteIndex{ + fetcher: fetcher{ + Ref: ref, + Client: http.DefaultClient, + context: context.Background(), + }, + } + + if _, err := rmt.RawManifest(); (err != nil) != tc.wantErr { + t.Errorf("RawManifest() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + }) + } +} + +func TestIndex(t *testing.T) { + idx := randomIndex(t) + expectedRepo := "foo/bar" + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + childDigest := mustIndexManifest(t, idx).Manifests[0].Digest + child := mustChild(t, idx, childDigest) + childPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, childDigest) + configPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, mustConfigName(t, child)) + manifestReqCount := 0 + childReqCount := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case manifestPath: + manifestReqCount++ + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Header().Set("Content-Type", string(mustMediaType(t, idx))) + w.Write(mustRawManifest(t, idx)) + case childPath: + childReqCount++ + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawManifest(t, child)) + case configPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + w.Write(mustRawConfigFile(t, child)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + tag := mustNewTag(t, fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo)) + rmt, err := Index(tag, WithTransport(http.DefaultTransport)) + if err != nil { + t.Errorf("Index() = %v", err) + } + rmtChild, err := rmt.Image(childDigest) + if err != nil { + t.Errorf("remoteIndex.Image(%s) = %v", childDigest, err) + } + + // Test that index works as expected. + if got, want := mustRawManifest(t, rmt), mustRawManifest(t, idx); !bytes.Equal(got, want) { + t.Errorf("RawManifest() = %v, want %v", got, want) + } + if diff := cmp.Diff(mustIndexManifest(t, idx), mustIndexManifest(t, rmt)); diff != "" { + t.Errorf("IndexManifest() (-want +got) = %v", diff) + } + if got, want := mustMediaType(t, rmt), mustMediaType(t, idx); got != want { + t.Errorf("MediaType() = %v, want %v", got, want) + } + if got, want := mustDigest(t, rmt), mustDigest(t, idx); got != want { + t.Errorf("Digest() = %v, want %v", got, want) + } + // Make sure caching the manifest works for index. + if manifestReqCount != 1 { + t.Errorf("RawManifest made %v requests, expected 1", manifestReqCount) + } + + // Test that child works as expected. + if got, want := mustRawManifest(t, rmtChild), mustRawManifest(t, child); !bytes.Equal(got, want) { + t.Errorf("RawManifest() = %v, want %v", got, want) + } + if got, want := mustRawConfigFile(t, rmtChild), mustRawConfigFile(t, child); !bytes.Equal(got, want) { + t.Errorf("RawConfigFile() = %v, want %v", got, want) + } + // Make sure caching the manifest works for child. + if childReqCount != 1 { + t.Errorf("RawManifest made %v requests, expected 1", childReqCount) + } + + // Try to fetch bogus children. + bogusHash := mustHash(t, bogusDigest) + + if _, err := rmt.Image(bogusHash); err == nil { + t.Errorf("remoteIndex.Image(bogusDigest) err = %v, wanted err", err) + } + if _, err := rmt.ImageIndex(bogusHash); err == nil { + t.Errorf("remoteIndex.ImageIndex(bogusDigest) err = %v, wanted err", err) + } +} + +// TestMatchesPlatform runs test cases on the matchesPlatform function which verifies +// whether the given platform can run on the required platform by checking the +// compatibility of architecture, OS, OS version, OS features, variant and features. +func TestMatchesPlatform(t *testing.T) { + t.Parallel() + tests := []struct { + // want is the expected return value from matchesPlatform + // when the given platform is 'given' and the required platform is 'required'. + given v1.Platform + required v1.Platform + want bool + }{{ // The given & required platforms are identical. matchesPlatform expected to return true. + given: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: true, + }, + { // OS and Architecture must exactly match. matchesPlatform expected to return false. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // OS version must exactly match + given: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10587", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // OS Features must exactly match. matchesPlatform expected to return false. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // Variant must exactly match. matchesPlatform expected to return false. + given: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv7l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // OS must exactly match, and is case sensative. matchesPlatform expected to return false. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "LinuX", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // OSVersion and Variant are specified in given but not in required. + // matchesPlatform expected to return true. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "", + OSFeatures: []string{"win64k"}, + Variant: "", + Features: []string{"sse4"}, + }, + want: true, + }, + { // Ensure the optional field OSVersion & Variant match exactly if specified as required. + given: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "", + OSFeatures: []string{}, + Variant: "", + Features: []string{}, + }, + required: v1.Platform{ + Architecture: "amd64", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: false, + }, + { // Checking subset validity when required less features than given features. + // matchesPlatform expected to return true. + given: v1.Platform{ + Architecture: "", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win32k"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "", + OS: "linux", + OSVersion: "", + OSFeatures: []string{}, + Variant: "", + Features: []string{}, + }, + want: true, + }, + { // Checking subset validity when required features are subset of given features. + // matchesPlatform expected to return true. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k", "f1", "f2"}, + Variant: "", + Features: []string{"sse4", "f1"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "", + Features: []string{"sse4"}, + }, + want: true, + }, + { // Checking subset validity when some required features is not subset of given features. + // matchesPlatform expected to return false. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k", "f1", "f2"}, + Variant: "", + Features: []string{"sse4", "f1"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k"}, + Variant: "", + Features: []string{"sse4", "f2"}, + }, + want: false, + }, + { // Checking subset validity when OS features not required, + // and required features is indeed a subset of given features. + // matchesPlatform expected to return true. + given: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{"win64k", "f1", "f2"}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + required: v1.Platform{ + Architecture: "arm", + OS: "linux", + OSVersion: "10.0.10586", + OSFeatures: []string{}, + Variant: "armv6l", + Features: []string{"sse4"}, + }, + want: true, + }, + } + + for _, test := range tests { + got := matchesPlatform(test.given, test.required) + if got != test.want { + t.Errorf("matchesPlatform(%v, %v); got %v, want %v", test.given, test.required, got, test.want) + } + } +} diff --git a/pkg/v1/remote/layer.go b/pkg/v1/remote/layer.go new file mode 100644 index 0000000..b2126f5 --- /dev/null +++ b/pkg/v1/remote/layer.go @@ -0,0 +1,94 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "io" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/verify" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// remoteImagelayer implements partial.CompressedLayer +type remoteLayer struct { + fetcher + digest v1.Hash +} + +// Compressed implements partial.CompressedLayer +func (rl *remoteLayer) Compressed() (io.ReadCloser, error) { + // We don't want to log binary layers -- this can break terminals. + ctx := redact.NewContext(rl.context, "omitting binary blobs from logs") + return rl.fetchBlob(ctx, verify.SizeUnknown, rl.digest) +} + +// Compressed implements partial.CompressedLayer +func (rl *remoteLayer) Size() (int64, error) { + resp, err := rl.headBlob(rl.digest) + if err != nil { + return -1, err + } + defer resp.Body.Close() + return resp.ContentLength, nil +} + +// Digest implements partial.CompressedLayer +func (rl *remoteLayer) Digest() (v1.Hash, error) { + return rl.digest, nil +} + +// MediaType implements v1.Layer +func (rl *remoteLayer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + +// See partial.Exists. +func (rl *remoteLayer) Exists() (bool, error) { + return rl.blobExists(rl.digest) +} + +// Layer reads the given blob reference from a registry as a Layer. A blob +// reference here is just a punned name.Digest where the digest portion is the +// digest of the blob to be read and the repository portion is the repo where +// that blob lives. +func Layer(ref name.Digest, options ...Option) (v1.Layer, error) { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return nil, err + } + f, err := makeFetcher(ref, o) + if err != nil { + return nil, err + } + h, err := v1.NewHash(ref.Identifier()) + if err != nil { + return nil, err + } + l, err := partial.CompressedToLayer(&remoteLayer{ + fetcher: *f, + digest: h, + }) + if err != nil { + return nil, err + } + return &MountableLayer{ + Layer: l, + Reference: ref, + }, nil +} diff --git a/pkg/v1/remote/layer_test.go b/pkg/v1/remote/layer_test.go new file mode 100644 index 0000000..a2f56bd --- /dev/null +++ b/pkg/v1/remote/layer_test.go @@ -0,0 +1,148 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "fmt" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestRemoteLayer(t *testing.T) { + layer, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + digest, err := layer.Digest() + if err != nil { + t.Fatal(err) + } + + // Set up a fake registry and write what we pulled to it. + // This ensures we get coverage for the remoteLayer.MediaType path. + s := httptest.NewServer(registry.New()) + defer s.Close() + t.Log(s.URL) + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + t.Log(u) + dst := fmt.Sprintf("%s/some/path@%s", u.Host, digest) + t.Log(dst) + ref, err := name.NewDigest(dst) + if err != nil { + t.Fatal(err) + } + + t.Log(ref) + if err := WriteLayer(ref.Context(), layer); err != nil { + t.Fatalf("failed to WriteLayer: %v", err) + } + + got, err := Layer(ref) + if err != nil { + t.Fatal(err) + } + + if _, err := got.MediaType(); err != nil { + t.Errorf("reading MediaType: %v", err) + } + + if err := compare.Layers(got, layer); err != nil { + t.Errorf("compare.Layers: %v", err) + } + if err := validate.Layer(got); err != nil { + t.Errorf("validate.Layer: %v", err) + } + + if ok, err := partial.Exists(got); err != nil { + t.Fatal(err) + } else if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } +} + +func TestRemoteLayerDescriptor(t *testing.T) { + layer, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + image, err := mutate.Append(empty.Image, mutate.Addendum{ + Layer: layer, + URLs: []string{"example.com"}, + }) + if err != nil { + t.Fatal(err) + } + + // Set up a fake registry and write what we pulled to it. + // This ensures we get coverage for the remoteLayer.MediaType path. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + dst := fmt.Sprintf("%s/some/path:tag", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, image); err != nil { + t.Fatalf("failed to WriteLayer: %v", err) + } + + pulled, err := Image(ref) + if err != nil { + t.Fatal(err) + } + + layers, err := pulled.Layers() + if err != nil { + t.Fatal(err) + } + + desc, err := partial.Descriptor(layers[0]) + if err != nil { + t.Fatal(err) + } + + if len(desc.URLs) != 1 { + t.Fatalf("expected url for layer[0]") + } + + if got, want := desc.URLs[0], "example.com"; got != want { + t.Errorf("layer[0].urls[0] = %s != %s", got, want) + } + if ok, err := partial.Exists(layers[0]); err != nil { + t.Fatal(err) + } else if got, want := ok, true; got != want { + t.Errorf("Exists() = %t != %t", got, want) + } +} diff --git a/pkg/v1/remote/list.go b/pkg/v1/remote/list.go new file mode 100644 index 0000000..e643c49 --- /dev/null +++ b/pkg/v1/remote/list.go @@ -0,0 +1,141 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +type tags struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +// ListWithContext calls List with the given context. +// +// Deprecated: Use List and WithContext. This will be removed in a future release. +func ListWithContext(ctx context.Context, repo name.Repository, options ...Option) ([]string, error) { + return List(repo, append(options, WithContext(ctx))...) +} + +// List calls /tags/list for the given repository, returning the list of tags +// in the "tags" property. +func List(repo name.Repository, options ...Option) ([]string, error) { + o, err := makeOptions(repo, options...) + if err != nil { + return nil, err + } + scopes := []string{repo.Scope(transport.PullScope)} + tr, err := transport.NewWithContext(o.context, repo.Registry, o.auth, o.transport, scopes) + if err != nil { + return nil, err + } + + uri := &url.URL{ + Scheme: repo.Registry.Scheme(), + Host: repo.Registry.RegistryStr(), + Path: fmt.Sprintf("/v2/%s/tags/list", repo.RepositoryStr()), + } + + if o.pageSize > 0 { + uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize) + } + + client := http.Client{Transport: tr} + tagList := []string{} + parsed := tags{} + + // get responses until there is no next page + for { + select { + case <-o.context.Done(): + return nil, o.context.Err() + default: + } + + req, err := http.NewRequestWithContext(o.context, "GET", uri.String(), nil) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, err + } + + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, err + } + + if err := resp.Body.Close(); err != nil { + return nil, err + } + + tagList = append(tagList, parsed.Tags...) + + uri, err = getNextPageURL(resp) + if err != nil { + return nil, err + } + // no next page + if uri == nil { + break + } + } + + return tagList, nil +} + +// getNextPageURL checks if there is a Link header in a http.Response which +// contains a link to the next page. If yes it returns the url.URL of the next +// page otherwise it returns nil. +func getNextPageURL(resp *http.Response) (*url.URL, error) { + link := resp.Header.Get("Link") + if link == "" { + return nil, nil + } + + if link[0] != '<' { + return nil, fmt.Errorf("failed to parse link header: missing '<' in: %s", link) + } + + end := strings.Index(link, ">") + if end == -1 { + return nil, fmt.Errorf("failed to parse link header: missing '>' in: %s", link) + } + link = link[1:end] + + linkURL, err := url.Parse(link) + if err != nil { + return nil, err + } + if resp.Request == nil || resp.Request.URL == nil { + return nil, nil + } + linkURL = resp.Request.URL.ResolveReference(linkURL) + return linkURL, nil +} diff --git a/pkg/v1/remote/list_test.go b/pkg/v1/remote/list_test.go new file mode 100644 index 0000000..89700b8 --- /dev/null +++ b/pkg/v1/remote/list_test.go @@ -0,0 +1,159 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/name" +) + +func TestList(t *testing.T) { + cases := []struct { + name string + responseBody []byte + wantErr bool + wantTags []string + }{{ + name: "success", + responseBody: []byte(`{"tags":["foo","bar"]}`), + wantErr: false, + wantTags: []string{"foo", "bar"}, + }, { + name: "not json", + responseBody: []byte("notjson"), + wantErr: true, + }} + + repoName := "ubuntu" + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tagsPath := fmt.Sprintf("/v2/%s/tags/list", repoName) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case tagsPath: + if r.Method != http.MethodGet { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet) + } + + w.Write(tc.responseBody) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + repo, err := name.NewRepository(fmt.Sprintf("%s/%s", u.Host, repoName), name.WeakValidation) + if err != nil { + t.Fatalf("name.NewRepository(%v) = %v", repoName, err) + } + + tags, err := List(repo) + if (err != nil) != tc.wantErr { + t.Errorf("List() wrong error: %v, want %v: %v\n", (err != nil), tc.wantErr, err) + } + + if diff := cmp.Diff(tc.wantTags, tags); diff != "" { + t.Errorf("List() wrong tags (-want +got) = %s", diff) + } + }) + } +} + +func TestCancelledList(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + repoName := "doesnotmatter" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + + repo, err := name.NewRepository(fmt.Sprintf("%s/%s", u.Host, repoName), name.WeakValidation) + if err != nil { + t.Fatalf("name.NewRepository(%v) = %v", repoName, err) + } + + _, err = ListWithContext(ctx, repo) + if err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Errorf(`unexpected error; want "context canceled", got %v`, err) + } +} + +func makeResp(hdr string) *http.Response { + return &http.Response{ + Header: http.Header{ + "Link": []string{hdr}, + }, + } +} + +func TestGetNextPageURL(t *testing.T) { + for _, hdr := range []string{ + "", + "<", + "><", + "<>", + fmt.Sprintf("<%c>", 0x7f), // makes url.Parse fail + } { + u, err := getNextPageURL(makeResp(hdr)) + if err == nil && u != nil { + t.Errorf("Expected err, got %+v", u) + } + } + + good := &http.Response{ + Header: http.Header{ + "Link": []string{""}, + }, + Request: &http.Request{ + URL: &url.URL{ + Scheme: "https", + }, + }, + } + u, err := getNextPageURL(good) + if err != nil { + t.Fatal(err) + } + + if u.Scheme != "https" { + t.Errorf("expected scheme to match request, got %s", u.Scheme) + } +} diff --git a/pkg/v1/remote/mount.go b/pkg/v1/remote/mount.go new file mode 100644 index 0000000..36d0885 --- /dev/null +++ b/pkg/v1/remote/mount.go @@ -0,0 +1,108 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +// MountableLayer wraps a v1.Layer in a shim that enables the layer to be +// "mounted" when published to another registry. +type MountableLayer struct { + v1.Layer + + Reference name.Reference +} + +// Descriptor retains the original descriptor from an image manifest. +// See partial.Descriptor. +func (ml *MountableLayer) Descriptor() (*v1.Descriptor, error) { + return partial.Descriptor(ml.Layer) +} + +// Exists is a hack. See partial.Exists. +func (ml *MountableLayer) Exists() (bool, error) { + return partial.Exists(ml.Layer) +} + +// mountableImage wraps the v1.Layer references returned by the embedded v1.Image +// in MountableLayer's so that remote.Write might attempt to mount them from their +// source repository. +type mountableImage struct { + v1.Image + + Reference name.Reference +} + +// Layers implements v1.Image +func (mi *mountableImage) Layers() ([]v1.Layer, error) { + ls, err := mi.Image.Layers() + if err != nil { + return nil, err + } + mls := make([]v1.Layer, 0, len(ls)) + for _, l := range ls { + mls = append(mls, &MountableLayer{ + Layer: l, + Reference: mi.Reference, + }) + } + return mls, nil +} + +// LayerByDigest implements v1.Image +func (mi *mountableImage) LayerByDigest(d v1.Hash) (v1.Layer, error) { + l, err := mi.Image.LayerByDigest(d) + if err != nil { + return nil, err + } + return &MountableLayer{ + Layer: l, + Reference: mi.Reference, + }, nil +} + +// LayerByDiffID implements v1.Image +func (mi *mountableImage) LayerByDiffID(d v1.Hash) (v1.Layer, error) { + l, err := mi.Image.LayerByDiffID(d) + if err != nil { + return nil, err + } + return &MountableLayer{ + Layer: l, + Reference: mi.Reference, + }, nil +} + +// Descriptor retains the original descriptor from an index manifest. +// See partial.Descriptor. +func (mi *mountableImage) Descriptor() (*v1.Descriptor, error) { + return partial.Descriptor(mi.Image) +} + +// ConfigLayer retains the original reference so that it can be mounted. +// See partial.ConfigLayer. +func (mi *mountableImage) ConfigLayer() (v1.Layer, error) { + l, err := partial.ConfigLayer(mi.Image) + if err != nil { + return nil, err + } + return &MountableLayer{ + Layer: l, + Reference: mi.Reference, + }, nil +} diff --git a/pkg/v1/remote/mount_test.go b/pkg/v1/remote/mount_test.go new file mode 100644 index 0000000..ad9b8f6 --- /dev/null +++ b/pkg/v1/remote/mount_test.go @@ -0,0 +1,55 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestMountableImage(t *testing.T) { + img, err := random.Image(1024, 5) + if err != nil { + t.Fatal(err) + } + + ref, err := name.ParseReference("ubuntu") + if err != nil { + t.Fatal(err) + } + + img = &mountableImage{ + Image: img, + Reference: ref, + } + + if err := validate.Image(img); err != nil { + t.Errorf("Validate() = %v", err) + } + + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + + for i, l := range layers { + if _, ok := l.(*MountableLayer); !ok { + t.Errorf("layers[%d] should be MountableLayer but isn't", i) + } + } +} diff --git a/pkg/v1/remote/multi_write.go b/pkg/v1/remote/multi_write.go new file mode 100644 index 0000000..7f32413 --- /dev/null +++ b/pkg/v1/remote/multi_write.go @@ -0,0 +1,302 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "fmt" + "net/http" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +// MultiWrite writes the given Images or ImageIndexes to the given refs, as +// efficiently as possible, by deduping shared layer blobs and uploading layers +// in parallel, then uploading all manifests in parallel. +// +// Current limitations: +// - All refs must share the same repository. +// - Images cannot consist of stream.Layers. +func MultiWrite(m map[name.Reference]Taggable, options ...Option) (rerr error) { + // Determine the repository being pushed to; if asked to push to + // multiple repositories, give up. + var repo, zero name.Repository + for ref := range m { + if repo == zero { + repo = ref.Context() + } else if ref.Context() != repo { + return fmt.Errorf("MultiWrite can only push to the same repository (saw %q and %q)", repo, ref.Context()) + } + } + + o, err := makeOptions(repo, options...) + if err != nil { + return err + } + + // Collect unique blobs (layers and config blobs). + blobs := map[v1.Hash]v1.Layer{} + newManifests := []map[name.Reference]Taggable{} + // Separate originally requested images and indexes, so we can push images first. + images, indexes := map[name.Reference]Taggable{}, map[name.Reference]Taggable{} + for ref, i := range m { + if img, ok := i.(v1.Image); ok { + images[ref] = i + if err := addImageBlobs(img, blobs, o.allowNondistributableArtifacts); err != nil { + return err + } + continue + } + if idx, ok := i.(v1.ImageIndex); ok { + indexes[ref] = i + newManifests, err = addIndexBlobs(idx, blobs, repo, newManifests, 0, o.allowNondistributableArtifacts) + if err != nil { + return err + } + continue + } + return fmt.Errorf("pushable resource was not Image or ImageIndex: %T", i) + } + + // Determine if any of the layers are Mountable, because if so we need + // to request Pull scope too. + ls := []v1.Layer{} + for _, l := range blobs { + ls = append(ls, l) + } + scopes := scopesForUploadingImage(repo, ls) + tr, err := transport.NewWithContext(o.context, repo.Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + w := writer{ + repo: repo, + client: &http.Client{Transport: tr}, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + } + + // Collect the total size of blobs and manifests we're about to write. + if o.updates != nil { + w.progress = &progress{updates: o.updates} + w.progress.lastUpdate = &v1.Update{} + defer close(o.updates) + defer func() { _ = w.progress.err(rerr) }() + for _, b := range blobs { + size, err := b.Size() + if err != nil { + return err + } + w.progress.total(size) + } + countManifest := func(t Taggable) error { + b, err := t.RawManifest() + if err != nil { + return err + } + w.progress.total(int64(len(b))) + return nil + } + for _, i := range images { + if err := countManifest(i); err != nil { + return err + } + } + for _, nm := range newManifests { + for _, i := range nm { + if err := countManifest(i); err != nil { + return err + } + } + } + for _, i := range indexes { + if err := countManifest(i); err != nil { + return err + } + } + } + + // Upload individual blobs and collect any errors. + blobChan := make(chan v1.Layer, 2*o.jobs) + ctx := o.context + g, gctx := errgroup.WithContext(o.context) + for i := 0; i < o.jobs; i++ { + // Start N workers consuming blobs to upload. + g.Go(func() error { + for b := range blobChan { + if err := w.uploadOne(gctx, b); err != nil { + return err + } + } + return nil + }) + } + g.Go(func() error { + defer close(blobChan) + for _, b := range blobs { + select { + case blobChan <- b: + case <-gctx.Done(): + return gctx.Err() + } + } + return nil + }) + if err := g.Wait(); err != nil { + return err + } + + commitMany := func(ctx context.Context, m map[name.Reference]Taggable) error { + g, ctx := errgroup.WithContext(ctx) + // With all of the constituent elements uploaded, upload the manifests + // to commit the images and indexes, and collect any errors. + type task struct { + i Taggable + ref name.Reference + } + taskChan := make(chan task, 2*o.jobs) + for i := 0; i < o.jobs; i++ { + // Start N workers consuming tasks to upload manifests. + g.Go(func() error { + for t := range taskChan { + if err := w.commitManifest(ctx, t.i, t.ref); err != nil { + return err + } + } + return nil + }) + } + go func() { + for ref, i := range m { + taskChan <- task{i, ref} + } + close(taskChan) + }() + return g.Wait() + } + // Push originally requested image manifests. These have no + // dependencies. + if err := commitMany(ctx, images); err != nil { + return err + } + // Push new manifests from lowest levels up. + for i := len(newManifests) - 1; i >= 0; i-- { + if err := commitMany(ctx, newManifests[i]); err != nil { + return err + } + } + // Push originally requested index manifests, which might depend on + // newly discovered manifests. + + return commitMany(ctx, indexes) +} + +// addIndexBlobs adds blobs to the set of blobs we intend to upload, and +// returns the latest copy of the ordered collection of manifests to upload. +func addIndexBlobs(idx v1.ImageIndex, blobs map[v1.Hash]v1.Layer, repo name.Repository, newManifests []map[name.Reference]Taggable, lvl int, allowNondistributableArtifacts bool) ([]map[name.Reference]Taggable, error) { + if lvl > len(newManifests)-1 { + newManifests = append(newManifests, map[name.Reference]Taggable{}) + } + + im, err := idx.IndexManifest() + if err != nil { + return nil, err + } + for _, desc := range im.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + idx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return nil, err + } + newManifests, err = addIndexBlobs(idx, blobs, repo, newManifests, lvl+1, allowNondistributableArtifacts) + if err != nil { + return nil, err + } + + // Also track the sub-index manifest to upload later by digest. + newManifests[lvl][repo.Digest(desc.Digest.String())] = idx + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := idx.Image(desc.Digest) + if err != nil { + return nil, err + } + if err := addImageBlobs(img, blobs, allowNondistributableArtifacts); err != nil { + return nil, err + } + + // Also track the sub-image manifest to upload later by digest. + newManifests[lvl][repo.Digest(desc.Digest.String())] = img + default: + // Workaround for #819. + if wl, ok := idx.(withLayer); ok { + layer, err := wl.Layer(desc.Digest) + if err != nil { + return nil, err + } + if err := addLayerBlob(layer, blobs, allowNondistributableArtifacts); err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("unknown media type: %v", desc.MediaType) + } + } + } + return newManifests, nil +} + +func addLayerBlob(l v1.Layer, blobs map[v1.Hash]v1.Layer, allowNondistributableArtifacts bool) error { + // Ignore foreign layers. + mt, err := l.MediaType() + if err != nil { + return err + } + + if mt.IsDistributable() || allowNondistributableArtifacts { + d, err := l.Digest() + if err != nil { + return err + } + + blobs[d] = l + } + + return nil +} + +func addImageBlobs(img v1.Image, blobs map[v1.Hash]v1.Layer, allowNondistributableArtifacts bool) error { + ls, err := img.Layers() + if err != nil { + return err + } + // Collect all layers. + for _, l := range ls { + if err := addLayerBlob(l, blobs, allowNondistributableArtifacts); err != nil { + return err + } + } + + // Collect config blob. + cl, err := partial.ConfigLayer(img) + if err != nil { + return err + } + return addLayerBlob(cl, blobs, allowNondistributableArtifacts) +} diff --git a/pkg/v1/remote/multi_write_test.go b/pkg/v1/remote/multi_write_test.go new file mode 100644 index 0000000..c2dd2f0 --- /dev/null +++ b/pkg/v1/remote/multi_write_test.go @@ -0,0 +1,351 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "io" + "log" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestMultiWrite(t *testing.T) { + // Create a random image. + img1, err := random.Image(1024, 2) + if err != nil { + t.Fatal("random.Image:", err) + } + + // Create another image that's based on the first. + rl, err := random.Layer(1024, types.OCIUncompressedLayer) + if err != nil { + t.Fatal("random.Layer:", err) + } + img2, err := mutate.AppendLayers(img1, rl) + if err != nil { + t.Fatal("mutate.AppendLayers:", err) + } + + // Also create a random index of images. + subidx, err := random.Index(1024, 2, 3) + if err != nil { + t.Fatal("random.Index:", err) + } + + // Add a sub-sub-index of random images. + subsubidx, err := random.Index(1024, 3, 4) + if err != nil { + t.Fatal("random.Index:", err) + } + subidx = mutate.AppendManifests(subidx, mutate.IndexAddendum{Add: subsubidx}) + + // Create an index containing both images and the index above. + idx := mutate.AppendManifests(empty.Index, + mutate.IndexAddendum{Add: img1}, + mutate.IndexAddendum{Add: img2}, + mutate.IndexAddendum{Add: subidx}, + mutate.IndexAddendum{Add: rl}, + ) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + // Write both images and the manifest list. + tag1, tag2, tag3 := mustNewTag(t, u.Host+"/repo:tag1"), mustNewTag(t, u.Host+"/repo:tag2"), mustNewTag(t, u.Host+"/repo:tag3") + if err := MultiWrite(map[name.Reference]Taggable{ + tag1: img1, + tag2: img2, + tag3: idx, + }); err != nil { + t.Error("Write:", err) + } + + // Check that tagged images are present. + for _, tag := range []name.Tag{tag1, tag2} { + got, err := Image(tag) + if err != nil { + t.Error(err) + continue + } + if err := validate.Image(got); err != nil { + t.Error("Validate() =", err) + } + } + + // Check that tagged manfest list is present and valid. + got, err := Index(tag3) + if err != nil { + t.Fatal(err) + } + if err := validate.Index(got); err != nil { + t.Error("Validate() =", err) + } +} + +func TestMultiWriteWithNondistributableLayer(t *testing.T) { + // Create a random image. + img1, err := random.Image(1024, 2) + if err != nil { + t.Fatal("random.Image:", err) + } + + // Create another image that's based on the first. + rl, err := random.Layer(1024, types.OCIRestrictedLayer) + if err != nil { + t.Fatal("random.Layer:", err) + } + img, err := mutate.AppendLayers(img1, rl) + if err != nil { + t.Fatal("mutate.AppendLayers:", err) + } + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + // Write the image. + tag1 := mustNewTag(t, u.Host+"/repo:tag1") + if err := MultiWrite(map[name.Reference]Taggable{tag1: img}, WithNondistributable); err != nil { + t.Error("Write:", err) + } + + // Check that tagged image is present. + got, err := Image(tag1) + if err != nil { + t.Error(err) + } + if err := validate.Image(got); err != nil { + t.Error("Validate() =", err) + } +} + +func TestMultiWrite_Retry(t *testing.T) { + // Create a random image. + img1, err := random.Image(1024, 2) + if err != nil { + t.Fatal("random.Image:", err) + } + + t.Run("retry http error 500", func(t *testing.T) { + // Set up a fake registry. + handler := registry.New() + + numOfInternalServerErrors := 0 + registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if strings.Contains(request.URL.Path, "/manifests/") && numOfInternalServerErrors < 1 { + numOfInternalServerErrors++ + responseWriter.WriteHeader(500) + return + } + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatFailsOnFirstUpload) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + tag1 := mustNewTag(t, u.Host+"/repo:tag1") + if err := MultiWrite(map[name.Reference]Taggable{ + tag1: img1, + }, WithRetryBackoff(fastBackoff)); err != nil { + t.Error("Write:", err) + } + }) + + t.Run("do not retry http error 401", func(t *testing.T) { + // Set up a fake registry. + handler := registry.New() + + numOf401HttpErrors := 0 + registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if strings.Contains(request.URL.Path, "/manifests/") { + numOf401HttpErrors++ + responseWriter.WriteHeader(401) + return + } + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatFailsOnFirstUpload) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + tag1 := mustNewTag(t, u.Host+"/repo:tag1") + if err := MultiWrite(map[name.Reference]Taggable{ + tag1: img1, + }); err == nil { + t.Fatal("Expected error:") + } + + if numOf401HttpErrors > 1 { + t.Fatal("Should not retry on 401 errors:") + } + }) + + t.Run("do not retry transport errors if transport.Wrapper is used", func(t *testing.T) { + // reference a http server that is not listening (used to pick a port that isn't listening) + onlyHandlesPing := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if strings.HasSuffix(request.URL.Path, "/v2/") { + responseWriter.WriteHeader(200) + return + } + }) + s := httptest.NewServer(onlyHandlesPing) + defer s.Close() + + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + tag1 := mustNewTag(t, u.Host+"/repo:tag1") + + // using a transport.Wrapper, meaning retry logic should not be wrapped + doesNotRetryTransport := &countTransport{inner: http.DefaultTransport} + transportWrapper, err := transport.NewWithContext(context.Background(), tag1.Repository.Registry, authn.Anonymous, doesNotRetryTransport, nil) + if err != nil { + t.Fatal(err) + } + + noRetry := func(error) bool { return false } + + if err := MultiWrite(map[name.Reference]Taggable{ + tag1: img1, + }, WithTransport(transportWrapper), WithJobs(1), WithRetryPredicate(noRetry)); err == nil { + t.Errorf("Expected an error, got nil") + } + + // expect count == 1 since jobs is set to 1 and we should not retry on transport eof error + if doesNotRetryTransport.count != 1 { + t.Errorf("Incorrect count, got %d, want %d", doesNotRetryTransport.count, 1) + } + }) + + t.Run("do not add UserAgent if transport.Wrapper is used", func(t *testing.T) { + expectedNotUsedUserAgent := "TEST_USER_AGENT" + + handler := registry.New() + + registryThatAssertsUserAgentIsCorrect := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if strings.Contains(request.Header.Get("User-Agent"), expectedNotUsedUserAgent) { + t.Fatalf("Should not contain User-Agent: %s, Got: %s", expectedNotUsedUserAgent, request.Header.Get("User-Agent")) + } + + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatAssertsUserAgentIsCorrect) + + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + tag1 := mustNewTag(t, u.Host+"/repo:tag1") + // using a transport.Wrapper, meaning retry logic should not be wrapped + transportWrapper, err := transport.NewWithContext(context.Background(), tag1.Repository.Registry, authn.Anonymous, http.DefaultTransport, nil) + if err != nil { + t.Fatal(err) + } + + if err := MultiWrite(map[name.Reference]Taggable{ + tag1: img1, + }, WithTransport(transportWrapper), WithUserAgent(expectedNotUsedUserAgent)); err != nil { + t.Fatal(err) + } + }) +} + +// TestMultiWrite_Deep tests that a deeply nested tree of manifest lists gets +// pushed in the correct order (i.e., each level in sequence). +func TestMultiWrite_Deep(t *testing.T) { + idx, err := random.Index(1024, 2, 2) + if err != nil { + t.Fatal("random.Image:", err) + } + for i := 0; i < 4; i++ { + idx = mutate.AppendManifests(idx, mutate.IndexAddendum{Add: idx}) + } + + // Set up a fake registry (with NOP logger to avoid spamming test logs). + nopLog := log.New(io.Discard, "", 0) + s := httptest.NewServer(registry.New(registry.Logger(nopLog))) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + // Write both images and the manifest list. + tag := mustNewTag(t, u.Host+"/repo:tag") + if err := MultiWrite(map[name.Reference]Taggable{ + tag: idx, + }); err != nil { + t.Error("Write:", err) + } + + // Check that tagged manfest list is present and valid. + got, err := Index(tag) + if err != nil { + t.Fatal(err) + } + if err := validate.Index(got); err != nil { + t.Error("Validate() =", err) + } +} + +type countTransport struct { + count int + inner http.RoundTripper +} + +func (t *countTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if strings.HasSuffix(req.URL.Path, "/v2/") { + return t.inner.RoundTrip(req) + } + + t.count++ + return nil, io.ErrUnexpectedEOF +} diff --git a/pkg/v1/remote/options.go b/pkg/v1/remote/options.go new file mode 100644 index 0000000..54a0af2 --- /dev/null +++ b/pkg/v1/remote/options.go @@ -0,0 +1,317 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "syscall" + "time" + + "github.com/google/go-containerregistry/internal/retry" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// Option is a functional option for remote operations. +type Option func(*options) error + +type options struct { + auth authn.Authenticator + keychain authn.Keychain + transport http.RoundTripper + platform v1.Platform + context context.Context + jobs int + userAgent string + allowNondistributableArtifacts bool + updates chan<- v1.Update + pageSize int + retryBackoff Backoff + retryPredicate retry.Predicate + filter map[string]string +} + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// Backoff is an alias of retry.Backoff to expose this configuration option to consumers of this lib +type Backoff = retry.Backoff + +var defaultRetryPredicate retry.Predicate = func(err error) bool { + // Various failure modes here, as we're often reading from and writing to + // the network. + if retry.IsTemporary(err) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) || errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET) { + logs.Warn.Printf("retrying %v", err) + return true + } + return false +} + +// Try this three times, waiting 1s after first failure, 3s after second. +var defaultRetryBackoff = Backoff{ + Duration: 1.0 * time.Second, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, +} + +// Useful for tests +var fastBackoff = Backoff{ + Duration: 1.0 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, +} + +var retryableStatusCodes = []int{ + http.StatusRequestTimeout, + http.StatusInternalServerError, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, +} + +const ( + defaultJobs = 4 + + // ECR returns an error if n > 1000: + // https://github.com/google/go-containerregistry/issues/1091 + defaultPageSize = 1000 +) + +// DefaultTransport is based on http.DefaultTransport with modifications +// documented inline below. +var DefaultTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +func makeOptions(target authn.Resource, opts ...Option) (*options, error) { + o := &options{ + transport: DefaultTransport, + platform: defaultPlatform, + context: context.Background(), + jobs: defaultJobs, + pageSize: defaultPageSize, + retryPredicate: defaultRetryPredicate, + retryBackoff: defaultRetryBackoff, + } + + for _, option := range opts { + if err := option(o); err != nil { + return nil, err + } + } + + switch { + case o.auth != nil && o.keychain != nil: + // It is a better experience to explicitly tell a caller their auth is misconfigured + // than potentially fail silently when the correct auth is overridden by option misuse. + return nil, errors.New("provide an option for either authn.Authenticator or authn.Keychain, not both") + case o.keychain != nil: + auth, err := o.keychain.Resolve(target) + if err != nil { + return nil, err + } + o.auth = auth + case o.auth == nil: + o.auth = authn.Anonymous + } + + // transport.Wrapper is a signal that consumers are opt-ing into providing their own transport without any additional wrapping. + // This is to allow consumers full control over the transports logic, such as providing retry logic. + if _, ok := o.transport.(*transport.Wrapper); !ok { + // Wrap the transport in something that logs requests and responses. + // It's expensive to generate the dumps, so skip it if we're writing + // to nothing. + if logs.Enabled(logs.Debug) { + o.transport = transport.NewLogger(o.transport) + } + + // Wrap the transport in something that can retry network flakes. + o.transport = transport.NewRetry(o.transport, transport.WithRetryPredicate(defaultRetryPredicate), transport.WithRetryStatusCodes(retryableStatusCodes...)) + + // Wrap this last to prevent transport.New from double-wrapping. + if o.userAgent != "" { + o.transport = transport.NewUserAgent(o.transport, o.userAgent) + } + } + + return o, nil +} + +// WithTransport is a functional option for overriding the default transport +// for remote operations. +// If transport.Wrapper is provided, this signals that the consumer does *not* want any further wrapping to occur. +// i.e. logging, retry and useragent +// +// The default transport is DefaultTransport. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) error { + o.transport = t + return nil + } +} + +// WithAuth is a functional option for overriding the default authenticator +// for remote operations. +// It is an error to use both WithAuth and WithAuthFromKeychain in the same Option set. +// +// The default authenticator is authn.Anonymous. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) error { + o.auth = auth + return nil + } +} + +// WithAuthFromKeychain is a functional option for overriding the default +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// It is an error to use both WithAuth and WithAuthFromKeychain in the same Option set. +// +// The default authenticator is authn.Anonymous. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *options) error { + o.keychain = keys + return nil + } +} + +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// +// The default platform is amd64/linux. +func WithPlatform(p v1.Platform) Option { + return func(o *options) error { + o.platform = p + return nil + } +} + +// WithContext is a functional option for setting the context in http requests +// performed by a given function. Note that this context is used for _all_ +// http requests, not just the initial volley. E.g., for remote.Image, the +// context will be set on http requests generated by subsequent calls to +// RawConfigFile() and even methods on layers returned by Layers(). +// +// The default context is context.Background(). +func WithContext(ctx context.Context) Option { + return func(o *options) error { + o.context = ctx + return nil + } +} + +// WithJobs is a functional option for setting the parallelism of remote +// operations performed by a given function. Note that not all remote +// operations support parallelism. +// +// The default value is 4. +func WithJobs(jobs int) Option { + return func(o *options) error { + if jobs <= 0 { + return errors.New("jobs must be greater than zero") + } + o.jobs = jobs + return nil + } +} + +// WithUserAgent adds the given string to the User-Agent header for any HTTP +// requests. This header will also include "go-containerregistry/${version}". +// +// If you want to completely overwrite the User-Agent header, use WithTransport. +func WithUserAgent(ua string) Option { + return func(o *options) error { + o.userAgent = ua + return nil + } +} + +// WithNondistributable includes non-distributable (foreign) layers +// when writing images, see: +// https://github.com/opencontainers/image-spec/blob/master/layer.md#non-distributable-layers +// +// The default behaviour is to skip these layers +func WithNondistributable(o *options) error { + o.allowNondistributableArtifacts = true + return nil +} + +// WithProgress takes a channel that will receive progress updates as bytes are written. +// +// Sending updates to an unbuffered channel will block writes, so callers +// should provide a buffered channel to avoid potential deadlocks. +func WithProgress(updates chan<- v1.Update) Option { + return func(o *options) error { + o.updates = updates + return nil + } +} + +// WithPageSize sets the given size as the value of parameter 'n' in the request. +// +// To omit the `n` parameter entirely, use WithPageSize(0). +// The default value is 1000. +func WithPageSize(size int) Option { + return func(o *options) error { + o.pageSize = size + return nil + } +} + +// WithRetryBackoff sets the httpBackoff for retry HTTP operations. +func WithRetryBackoff(backoff Backoff) Option { + return func(o *options) error { + o.retryBackoff = backoff + return nil + } +} + +// WithRetryPredicate sets the predicate for retry HTTP operations. +func WithRetryPredicate(predicate retry.Predicate) Option { + return func(o *options) error { + o.retryPredicate = predicate + return nil + } +} + +// WithFilter sets the filter querystring for HTTP operations. +func WithFilter(key string, value string) Option { + return func(o *options) error { + if o.filter == nil { + o.filter = map[string]string{} + } + o.filter[key] = value + return nil + } +} diff --git a/pkg/v1/remote/progress.go b/pkg/v1/remote/progress.go new file mode 100644 index 0000000..1f43963 --- /dev/null +++ b/pkg/v1/remote/progress.go @@ -0,0 +1,69 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "io" + "sync" + "sync/atomic" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type progress struct { + sync.Mutex + updates chan<- v1.Update + lastUpdate *v1.Update +} + +func (p *progress) total(delta int64) { + atomic.AddInt64(&p.lastUpdate.Total, delta) +} + +func (p *progress) complete(delta int64) { + p.Lock() + defer p.Unlock() + p.updates <- v1.Update{ + Total: p.lastUpdate.Total, + Complete: atomic.AddInt64(&p.lastUpdate.Complete, delta), + } +} + +func (p *progress) err(err error) error { + if err != nil && p.updates != nil { + p.updates <- v1.Update{Error: err} + } + return err +} + +type progressReader struct { + rc io.ReadCloser + + count *int64 // number of bytes this reader has read, to support resetting on retry. + progress *progress +} + +func (r *progressReader) Read(b []byte) (int, error) { + n, err := r.rc.Read(b) + if err != nil { + return n, err + } + atomic.AddInt64(r.count, int64(n)) + // TODO: warn/debug log if sending takes too long, or if sending is blocked while context is canceled. + r.progress.complete(int64(n)) + return n, nil +} + +func (r *progressReader) Close() error { return r.rc.Close() } diff --git a/pkg/v1/remote/progress_test.go b/pkg/v1/remote/progress_test.go new file mode 100644 index 0000000..759c8ca --- /dev/null +++ b/pkg/v1/remote/progress_test.go @@ -0,0 +1,463 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestWriteLayer_Progress(t *testing.T) { + l, err := random.Layer(1000, types.OCIUncompressedLayer) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := WriteLayer(ref.Context(), l, WithProgress(c)); err != nil { + t.Fatalf("WriteLayer: %v", err) + } + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +// TestWriteLayer_Progress_Exists tests progress reporting behavior when the +// layer already exists in the registry, so writes are skipped, but progress +// should still be reported in one update. +func TestWriteLayer_Progress_Exists(t *testing.T) { + l, err := random.Layer(1000, types.OCILayer) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + // Write the layer, so we can get updates when we write it again. + if err := WriteLayer(ref.Context(), l); err != nil { + t.Fatalf("WriteLayer: %v", err) + } + if err := WriteLayer(ref.Context(), l, WithProgress(c)); err != nil { + t.Fatalf("WriteLayer: %v", err) + } + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +func TestWrite_Progress(t *testing.T) { + img, err := random.Image(1000, 5) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, img, WithProgress(c)); err != nil { + t.Fatalf("Write: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +// An image with multiple identical layers is handled correctly. +func TestWrite_Progress_DedupeLayers(t *testing.T) { + img := empty.Image + for i := 0; i < 10; i++ { + l, err := random.Layer(1000, types.OCILayer) + if err != nil { + t.Fatal(err) + } + + img, err = mutate.AppendLayers(img, l) + if err != nil { + t.Fatal(err) + } + } + + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, img, WithProgress(c)); err != nil { + t.Fatalf("Write: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +func TestWriteIndex_Progress(t *testing.T) { + idx, err := random.Index(1000, 3, 3) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := WriteIndex(ref, idx, WithProgress(c)); err != nil { + t.Fatalf("WriteIndex: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +func TestMultiWrite_Progress(t *testing.T) { + idx, err := random.Index(1000, 3, 3) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 1000) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + ref, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload", u.Host)) + if err != nil { + t.Fatal(err) + } + ref2, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload:again", u.Host)) + if err != nil { + t.Fatal(err) + } + + if err := MultiWrite(map[name.Reference]Taggable{ + ref: idx, + ref2: idx, + }, WithProgress(c)); err != nil { + t.Fatalf("MultiWrite: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +func TestMultiWrite_Progress_Retry(t *testing.T) { + idx, err := random.Index(1000, 3, 3) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 1000) + + // Set up a fake registry. + handler := registry.New() + numOfInternalServerErrors := 0 + var mu sync.Mutex + registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + mu.Lock() + defer mu.Unlock() + if strings.Contains(request.URL.Path, "/manifests/") && numOfInternalServerErrors < 1 { + numOfInternalServerErrors++ + responseWriter.WriteHeader(500) + return + } + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatFailsOnFirstUpload) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + ref, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload", u.Host)) + if err != nil { + t.Fatal(err) + } + ref2, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload:again", u.Host)) + if err != nil { + t.Fatal(err) + } + + if err := MultiWrite(map[name.Reference]Taggable{ + ref: idx, + ref2: idx, + }, WithProgress(c), WithRetryBackoff(fastBackoff)); err != nil { + t.Fatalf("MultiWrite: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +func TestWriteLayer_Progress_Retry(t *testing.T) { + l, err := random.Layer(100000, types.OCIUncompressedLayer) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + handler := registry.New() + + numOfInternalServerErrors := 0 + registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPatch && strings.Contains(request.URL.Path, "upload/blobs/uploads") && numOfInternalServerErrors < 1 { + numOfInternalServerErrors++ + responseWriter.WriteHeader(500) + return + } + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatFailsOnFirstUpload) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := WriteLayer(ref.Context(), l, WithProgress(c), WithRetryBackoff(fastBackoff)); err != nil { + t.Fatalf("WriteLayer: %v", err) + } + + everyUpdate := []v1.Update{} + for update := range c { + everyUpdate = append(everyUpdate, update) + } + + if diff := cmp.Diff(everyUpdate, []v1.Update{ + {Total: 101921, Complete: 32768}, + {Total: 101921, Complete: 65536}, + {Total: 101921, Complete: 98304}, + {Total: 101921, Complete: 101921}, + // retry results in the same messages sent to the updates channel + {Total: 101921, Complete: 0}, + {Total: 101921, Complete: 32768}, + {Total: 101921, Complete: 65536}, + {Total: 101921, Complete: 98304}, + {Total: 101921, Complete: 101921}, + }); diff != "" { + t.Errorf("received updates (-want +got) = %s", diff) + } +} + +func TestWriteLayer_Progress_Error(t *testing.T) { + l, err := random.Layer(100000, types.OCIUncompressedLayer) + if err != nil { + t.Fatal(err) + } + c := make(chan v1.Update, 200) + + // Set up a fake registry. + handler := registry.New() + registryThatAlwaysFails := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPatch && strings.Contains(request.URL.Path, "blobs/uploads") { + responseWriter.WriteHeader(403) + } + handler.ServeHTTP(responseWriter, request) + }) + + s := httptest.NewServer(registryThatAlwaysFails) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := WriteLayer(ref.Context(), l, WithProgress(c)); err == nil { + t.Errorf("WriteLayer: wanted error, got nil") + } + + everyUpdate := []v1.Update{} + for update := range c { + everyUpdate = append(everyUpdate, update) + } + + if diff := cmp.Diff(everyUpdate[:len(everyUpdate)-1], []v1.Update{ + {Total: 101921, Complete: 32768}, + {Total: 101921, Complete: 65536}, + {Total: 101921, Complete: 98304}, + {Total: 101921, Complete: 101921}, + // retry results in the same messages sent to the updates channel + {Total: 101921, Complete: 0}, + }); diff != "" { + t.Errorf("received updates (-want +got) = %s", diff) + } + if everyUpdate[len(everyUpdate)-1].Error == nil { + t.Errorf("Last update had nil error") + } +} + +func TestWrite_Progress_WithNonDistributableLayer_AndIncludeNonDistributableLayersOption(t *testing.T) { + ociLayer, err := random.Layer(1000, types.OCILayer) + if err != nil { + t.Fatal(err) + } + + nonDistributableLayer, err := random.Layer(1000, types.OCIRestrictedLayer) + if err != nil { + t.Fatal(err) + } + + img, err := mutate.AppendLayers(empty.Image, ociLayer, nonDistributableLayer) + if err != nil { + t.Fatal(err) + } + + c := make(chan v1.Update, 200) + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/progress/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, img, WithProgress(c), WithNondistributable); err != nil { + t.Fatalf("Write: %v", err) + } + + if err := checkUpdates(c); err != nil { + t.Fatal(err) + } +} + +// checkUpdates checks that updates show steady progress toward a total, and +// don't describe errors. +func checkUpdates(updates <-chan v1.Update) error { + var high, total int64 + for u := range updates { + if u.Error != nil { + return u.Error + } + + if u.Total == 0 { + return errors.New("saw zero total") + } + + if total == 0 { + total = u.Total + } else if u.Total != total { + return fmt.Errorf("total changed: was %d, saw %d", total, u.Total) + } + + if u.Complete < high { + return fmt.Errorf("saw progress revert: was high of %d, saw %d", high, u.Complete) + } + high = u.Complete + } + + if high > total { + return fmt.Errorf("final progress (%d) exceeded total (%d) by %d", high, total, high-total) + } else if high < total { + return fmt.Errorf("final progress (%d) did not reach total (%d) by %d", high, total, total-high) + } + + return nil +} diff --git a/pkg/v1/remote/referrers.go b/pkg/v1/remote/referrers.go new file mode 100644 index 0000000..b3db863 --- /dev/null +++ b/pkg/v1/remote/referrers.go @@ -0,0 +1,35 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Referrers returns a list of descriptors that refer to the given manifest digest. +// +// The subject manifest doesn't have to exist in the registry for there to be descriptors that refer to it. +func Referrers(d name.Digest, options ...Option) (*v1.IndexManifest, error) { + o, err := makeOptions(d.Context(), options...) + if err != nil { + return nil, err + } + f, err := makeFetcher(d, o) + if err != nil { + return nil, err + } + return f.fetchReferrers(o.context, o.filter, d) +} diff --git a/pkg/v1/remote/referrers_test.go b/pkg/v1/remote/referrers_test.go new file mode 100644 index 0000000..91f9edc --- /dev/null +++ b/pkg/v1/remote/referrers_test.go @@ -0,0 +1,183 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote_test + +import ( + "fmt" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestReferrers(t *testing.T) { + // Run all tests against: + // + // (1) A OCI 1.0 registry (without referrers API) + // (2) An OCI 1.1+ registry (with referrers API) + // + for _, leg := range []struct { + server *httptest.Server + tryFallback bool + }{ + { + server: httptest.NewServer(registry.New(registry.WithReferrersSupport(false))), + tryFallback: true, + }, + { + server: httptest.NewServer(registry.New(registry.WithReferrersSupport(true))), + tryFallback: false, + }, + } { + s := leg.server + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + descriptor := func(img v1.Image) v1.Descriptor { + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + sz, err := img.Size() + if err != nil { + t.Fatal(err) + } + mt, err := img.MediaType() + if err != nil { + t.Fatal(err) + } + return v1.Descriptor{ + Digest: d, + Size: sz, + MediaType: mt, + ArtifactType: "application/testing123", + } + } + + // Push an image we'll attach things to. + // We'll copy from src to dst. + rootRef, err := name.ParseReference(fmt.Sprintf("%s/repo:root", u.Host)) + if err != nil { + t.Fatal(err) + } + rootImg, err := random.Image(10, 10) + if err != nil { + t.Fatal(err) + } + rootImg = mutate.ConfigMediaType(rootImg, types.MediaType("application/testing123")) + if err := remote.Write(rootRef, rootImg); err != nil { + t.Fatal(err) + } + rootDesc := descriptor(rootImg) + t.Logf("root image is %s", rootDesc.Digest) + + // Push an image that refers to the root image as its subject. + leafRef, err := name.ParseReference(fmt.Sprintf("%s/repo:leaf", u.Host)) + if err != nil { + t.Fatal(err) + } + leafImg, err := random.Image(20, 20) + if err != nil { + t.Fatal(err) + } + leafImg = mutate.ConfigMediaType(leafImg, types.MediaType("application/testing123")) + leafImg = mutate.Subject(leafImg, rootDesc).(v1.Image) + if err := remote.Write(leafRef, leafImg); err != nil { + t.Fatal(err) + } + leafDesc := descriptor(leafImg) + t.Logf("leaf image is %s", leafDesc.Digest) + + // Get the referrers of the root image, by digest. + rootRefDigest := rootRef.Context().Digest(rootDesc.Digest.String()) + index, err := remote.Referrers(rootRefDigest) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff([]v1.Descriptor{leafDesc}, index.Manifests); d != "" { + t.Fatalf("referrers diff (-want,+got): %s", d) + } + + if leg.tryFallback { + // Get the referrers by querying the root image's fallback tag directly. + tag, err := name.ParseReference(fmt.Sprintf("%s/repo:sha256-%s", u.Host, rootDesc.Digest.Hex)) + if err != nil { + t.Fatal(err) + } + idx, err := remote.Index(tag) + if err != nil { + t.Fatal(err) + } + mf, err := idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(index.Manifests, mf.Manifests); d != "" { + t.Fatalf("fallback tag diff (-want,+got): %s", d) + } + } + + // Push the leaf image again, this time with a different tag. + // This shouldn't add another item to the root image's referrers, + // because it's the same digest. + // Push an image that refers to the root image as its subject. + leaf2Ref, err := name.ParseReference(fmt.Sprintf("%s/repo:leaf2", u.Host)) + if err != nil { + t.Fatal(err) + } + if err := remote.Write(leaf2Ref, leafImg); err != nil { + t.Fatal(err) + } + // Get the referrers of the root image again, which should only have one entry. + rootRefDigest = rootRef.Context().Digest(rootDesc.Digest.String()) + index, err = remote.Referrers(rootRefDigest) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff([]v1.Descriptor{leafDesc}, index.Manifests); d != "" { + t.Fatalf("referrers diff after second push (-want,+got): %s", d) + } + + // Try applying filters and verify number of manifests and and annotations + index, err = remote.Referrers(rootRefDigest, + remote.WithFilter("artifactType", "application/testing123")) + if err != nil { + t.Fatal(err) + } + if numManifests := len(index.Manifests); numManifests == 0 { + t.Fatal("index contained 0 manifests") + } + + index, err = remote.Referrers(rootRefDigest, + remote.WithFilter("artifactType", "application/testing123BADDDD")) + if err != nil { + t.Fatal(err) + } + if numManifests := len(index.Manifests); numManifests != 0 { + t.Fatalf("expected index to contain 0 manifests, but had %d", numManifests) + } + } +} diff --git a/pkg/v1/remote/transport/README.md b/pkg/v1/remote/transport/README.md new file mode 100644 index 0000000..bd4d957 --- /dev/null +++ b/pkg/v1/remote/transport/README.md @@ -0,0 +1,129 @@ +# `transport` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/transport?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/transport) + +The [distribution protocol](https://github.com/opencontainers/distribution-spec) is fairly simple, but correctly [implementing authentication](../../../authn/README.md) is **hard**. + +This package [implements](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote/transport#New) an [`http.RoundTripper`](https://godoc.org/net/http#RoundTripper) +that transparently performs: +* [Token +Authentication](https://docs.docker.com/registry/spec/auth/token/) and +* [OAuth2 +Authentication](https://docs.docker.com/registry/spec/auth/oauth/) + +for registry clients. + +## Raison d'être + +> Why not just use the [`docker/distribution`](https://godoc.org/github.com/docker/distribution/registry/client/auth) client? + +Great question! Mostly, because I don't want to depend on [`prometheus/client_golang`](https://github.com/prometheus/client_golang). + +As a performance optimization, that client uses [a cache](https://github.com/docker/distribution/blob/a8371794149d1d95f1e846744b05c87f2f825e5a/registry/client/repository.go#L173) to keep track of a mapping between blob digests and their [descriptors](https://github.com/docker/distribution/blob/a8371794149d1d95f1e846744b05c87f2f825e5a/blobs.go#L57-L86). Unfortunately, the cache [uses prometheus](https://github.com/docker/distribution/blob/a8371794149d1d95f1e846744b05c87f2f825e5a/registry/storage/cache/cachedblobdescriptorstore.go#L44) to track hits and misses, so if you want to use that client you have to pull in all of prometheus, which is pretty large. + +![docker/distribution](../../../../images/docker.dot.svg) + +> Why does it matter if you depend on prometheus? Who cares? + +It's generally polite to your downstream to reduce the number of dependencies your package requires: + +* Downloading your package is faster, which helps our Australian friends and people on airplanes. +* There is less code to compile, which speeds up builds and saves the planet from global warming. +* You reduce the likelihood of inflicting dependency hell upon your consumers. +* [Tim Hockin](https://twitter.com/thockin/status/958606077456654336) prefers it based on his experience working on Kubernetes, and he's a pretty smart guy. + +> Okay, what about [`containerd/containerd`](https://godoc.org/github.com/containerd/containerd/remotes/docker)? + +Similar reasons! That ends up pulling in grpc, protobuf, and logrus. + +![containerd/containerd](../../../../images/containerd.dot.svg) + +> Well... what about [`containers/image`](https://godoc.org/github.com/containers/image/docker)? + +That just uses the the `docker/distribution` client... and more! + +![containers/image](../../../../images/containers.dot.svg) + +> Wow, what about this package? + +Of course, this package isn't perfect either. `transport` depends on `authn`, +which in turn depends on docker's config file parsing and handling package, +which you don't strictly need but almost certainly want if you're going to be +interacting with a registry. + +![google/go-containerregistry](../../../../images/ggcr.dot.svg) + +*These graphs were generated by +[`kisielk/godepgraph`](https://github.com/kisielk/godepgraph).* + +## Usage + +This is heavily used by the +[`remote`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote) +package, which implements higher level image-centric functionality, but this +package is useful if you want to interact directly with the registry to do +something that `remote` doesn't support, e.g. [to handle with schema 1 +images](https://github.com/google/go-containerregistry/pull/509). + +This package also includes some [error +handling](https://github.com/opencontainers/distribution-spec/blob/60be706c34ee7805bdd1d3d11affec53b0dfb8fb/spec.md#errors) +facilities in the form of +[`CheckError`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote/transport#CheckError), +which will parse the response body into a structured error for unexpected http +status codes. + +Here's a "simple" program that writes the result of +[listing tags](https://github.com/opencontainers/distribution-spec/blob/60be706c34ee7805bdd1d3d11affec53b0dfb8fb/spec.md#tags) +for [`gcr.io/google-containers/pause`](https://gcr.io/google-containers/pause) +to stdout. + +```go +package main + +import ( + "io" + "net/http" + "os" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +func main() { + repo, err := name.NewRepository("gcr.io/google-containers/pause") + if err != nil { + panic(err) + } + + // Fetch credentials based on your docker config file, which is $HOME/.docker/config.json or $DOCKER_CONFIG. + auth, err := authn.DefaultKeychain.Resolve(repo.Registry) + if err != nil { + panic(err) + } + + // Construct an http.Client that is authorized to pull from gcr.io/google-containers/pause. + scopes := []string{repo.Scope(transport.PullScope)} + t, err := transport.New(repo.Registry, auth, http.DefaultTransport, scopes) + if err != nil { + panic(err) + } + client := &http.Client{Transport: t} + + // Make the actual request. + resp, err := client.Get("https://gcr.io/v2/google-containers/pause/tags/list") + if err != nil { + panic(err) + } + + // Assert that we get a 200, otherwise attempt to parse body as a structured error. + if err := transport.CheckError(resp, http.StatusOK); err != nil { + panic(err) + } + + // Write the response to stdout. + if _, err := io.Copy(os.Stdout, resp.Body); err != nil { + panic(err) + } +} +``` diff --git a/pkg/v1/remote/transport/basic.go b/pkg/v1/remote/transport/basic.go new file mode 100644 index 0000000..fdb362b --- /dev/null +++ b/pkg/v1/remote/transport/basic.go @@ -0,0 +1,62 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "encoding/base64" + "fmt" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" +) + +type basicTransport struct { + inner http.RoundTripper + auth authn.Authenticator + target string +} + +var _ http.RoundTripper = (*basicTransport)(nil) + +// RoundTrip implements http.RoundTripper +func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) { + if bt.auth != authn.Anonymous { + auth, err := bt.auth.Authorization() + if err != nil { + return nil, err + } + + // http.Client handles redirects at a layer above the http.RoundTripper + // abstraction, so to avoid forwarding Authorization headers to places + // we are redirected, only set it when the authorization header matches + // the host with which we are interacting. + // In case of redirect http.Client can use an empty Host, check URL too. + if in.Host == bt.target || in.URL.Host == bt.target { + if bearer := auth.RegistryToken; bearer != "" { + hdr := fmt.Sprintf("Bearer %s", bearer) + in.Header.Set("Authorization", hdr) + } else if user, pass := auth.Username, auth.Password; user != "" && pass != "" { + delimited := fmt.Sprintf("%s:%s", user, pass) + encoded := base64.StdEncoding.EncodeToString([]byte(delimited)) + hdr := fmt.Sprintf("Basic %s", encoded) + in.Header.Set("Authorization", hdr) + } else if token := auth.Auth; token != "" { + hdr := fmt.Sprintf("Basic %s", token) + in.Header.Set("Authorization", hdr) + } + } + } + return bt.inner.RoundTrip(in) +} diff --git a/pkg/v1/remote/transport/basic_test.go b/pkg/v1/remote/transport/basic_test.go new file mode 100644 index 0000000..68dd90e --- /dev/null +++ b/pkg/v1/remote/transport/basic_test.go @@ -0,0 +1,138 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" +) + +func TestBasicTransport(t *testing.T) { + username := "foo" + password := "bar" + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hdr := r.Header.Get("Authorization") + if !strings.HasPrefix(hdr, "Basic ") { + t.Errorf("Header.Get(Authorization); got %v, want Basic prefix", hdr) + } + user, pass, _ := r.BasicAuth() + if user != username || pass != password { + t.Error("Invalid credentials.") + } + if r.URL.Path == "/v2/auth" { + http.Redirect(w, r, "/redirect", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + inner := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: username, Password: password} + client := http.Client{Transport: &basicTransport{inner: inner, auth: basic, target: "gcr.io"}} + + _, err := client.Get("http://gcr.io/v2/auth") + if err != nil { + t.Errorf("Unexpected error during Get: %v", err) + } +} + +func TestBasicTransportRegistryToken(t *testing.T) { + token := "mytoken" + for _, tc := range []struct { + auth authn.Authenticator + hdr string + wantErr bool + }{{ + auth: authn.FromConfig(authn.AuthConfig{RegistryToken: token}), + hdr: "Bearer mytoken", + }, { + auth: authn.FromConfig(authn.AuthConfig{Auth: token}), + hdr: "Basic mytoken", + }, { + auth: authn.Anonymous, + hdr: "", + }, { + auth: &badAuth{}, + hdr: "", + wantErr: true, + }} { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hdr := r.Header.Get("Authorization") + want := tc.hdr + if hdr != want { + t.Errorf("Header.Get(Authorization); got %v, want %s", hdr, want) + } + if r.URL.Path == "/v2/auth" { + http.Redirect(w, r, "/redirect", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + inner := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + client := http.Client{Transport: &basicTransport{inner: inner, auth: tc.auth, target: "gcr.io"}} + + _, err := client.Get("http://gcr.io/v2/auth") + if err != nil && !tc.wantErr { + t.Errorf("Unexpected error during Get: %v", err) + } + } +} + +func TestBasicTransportWithEmptyAuthnCred(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if c, ok := r.Header["Authorization"]; ok && c[0] == "" { + t.Error("got empty Authorization header") + } + if r.URL.Path == "/v2/auth" { + http.Redirect(w, r, "/redirect", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + inner := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + client := http.Client{Transport: &basicTransport{inner: inner, auth: authn.Anonymous, target: "gcr.io"}} + _, err := client.Get("http://gcr.io/v2/auth") + if err != nil { + t.Errorf("Unexpected error during Get: %v", err) + } +} diff --git a/pkg/v1/remote/transport/bearer.go b/pkg/v1/remote/transport/bearer.go new file mode 100644 index 0000000..ea07ff6 --- /dev/null +++ b/pkg/v1/remote/transport/bearer.go @@ -0,0 +1,320 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + + authchallenge "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" +) + +type bearerTransport struct { + // Wrapped by bearerTransport. + inner http.RoundTripper + // Basic credentials that we exchange for bearer tokens. + basic authn.Authenticator + // Holds the bearer response from the token service. + bearer authn.AuthConfig + // Registry to which we send bearer tokens. + registry name.Registry + // See https://tools.ietf.org/html/rfc6750#section-3 + realm string + // See https://docs.docker.com/registry/spec/auth/token/ + service string + scopes []string + // Scheme we should use, determined by ping response. + scheme string +} + +var _ http.RoundTripper = (*bearerTransport)(nil) + +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + +func stringSet(ss []string) map[string]struct{} { + set := make(map[string]struct{}) + for _, s := range ss { + set[s] = struct{}{} + } + return set +} + +// RoundTrip implements http.RoundTripper +func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { + sendRequest := func() (*http.Response, error) { + // http.Client handles redirects at a layer above the http.RoundTripper + // abstraction, so to avoid forwarding Authorization headers to places + // we are redirected, only set it when the authorization header matches + // the registry with which we are interacting. + // In case of redirect http.Client can use an empty Host, check URL too. + if matchesHost(bt.registry, in, bt.scheme) { + hdr := fmt.Sprintf("Bearer %s", bt.bearer.RegistryToken) + in.Header.Set("Authorization", hdr) + } + return bt.inner.RoundTrip(in) + } + + res, err := sendRequest() + if err != nil { + return nil, err + } + + // If we hit a WWW-Authenticate challenge, it might be due to expired tokens or insufficient scope. + if challenges := authchallenge.ResponseChallenges(res); len(challenges) != 0 { + // close out old response, since we will not return it. + res.Body.Close() + + newScopes := []string{} + for _, wac := range challenges { + // TODO(jonjohnsonjr): Should we also update "realm" or "service"? + if want, ok := wac.Parameters["scope"]; ok { + // Add any scopes that we don't already request. + got := stringSet(bt.scopes) + if _, ok := got[want]; !ok { + newScopes = append(newScopes, want) + } + } + } + + // Some registries seem to only look at the first scope parameter during a token exchange. + // If a request fails because it's missing a scope, we should put those at the beginning, + // otherwise the registry might just ignore it :/ + newScopes = append(newScopes, bt.scopes...) + bt.scopes = newScopes + + // TODO(jonjohnsonjr): Teach transport.Error about "error" and "error_description" from challenge. + + // Retry the request to attempt to get a valid token. + if err = bt.refresh(in.Context()); err != nil { + return nil, err + } + return sendRequest() + } + + return res, err +} + +// It's unclear which authentication flow to use based purely on the protocol, +// so we rely on heuristics and fallbacks to support as many registries as possible. +// The basic token exchange is attempted first, falling back to the oauth flow. +// If the IdentityToken is set, this indicates that we should start with the oauth flow. +func (bt *bearerTransport) refresh(ctx context.Context) error { + auth, err := bt.basic.Authorization() + if err != nil { + return err + } + + if auth.RegistryToken != "" { + bt.bearer.RegistryToken = auth.RegistryToken + return nil + } + + var content []byte + if auth.IdentityToken != "" { + // If the secret being stored is an identity token, + // the Username should be set to , which indicates + // we are using an oauth flow. + content, err = bt.refreshOauth(ctx) + var terr *Error + if errors.As(err, &terr) && terr.StatusCode == http.StatusNotFound { + // Note: Not all token servers implement oauth2. + // If the request to the endpoint returns 404 using the HTTP POST method, + // refer to Token Documentation for using the HTTP GET method supported by all token servers. + content, err = bt.refreshBasic(ctx) + } + } else { + content, err = bt.refreshBasic(ctx) + } + if err != nil { + return err + } + + // Some registries don't have "token" in the response. See #54. + type tokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + // TODO: handle expiry? + } + + var response tokenResponse + if err := json.Unmarshal(content, &response); err != nil { + return err + } + + // Some registries set access_token instead of token. + if response.AccessToken != "" { + response.Token = response.AccessToken + } + + // Find a token to turn into a Bearer authenticator + if response.Token != "" { + bt.bearer.RegistryToken = response.Token + } else { + return fmt.Errorf("no token in bearer response:\n%s", content) + } + + // If we obtained a refresh token from the oauth flow, use that for refresh() now. + if response.RefreshToken != "" { + bt.basic = authn.FromConfig(authn.AuthConfig{ + IdentityToken: response.RefreshToken, + }) + } + + return nil +} + +func matchesHost(reg name.Registry, in *http.Request, scheme string) bool { + canonicalHeaderHost := canonicalAddress(in.Host, scheme) + canonicalURLHost := canonicalAddress(in.URL.Host, scheme) + canonicalRegistryHost := canonicalAddress(reg.RegistryStr(), scheme) + return canonicalHeaderHost == canonicalRegistryHost || canonicalURLHost == canonicalRegistryHost +} + +func canonicalAddress(host, scheme string) (address string) { + // The host may be any one of: + // - hostname + // - hostname:port + // - ipv4 + // - ipv4:port + // - ipv6 + // - [ipv6]:port + // As net.SplitHostPort returns an error if the host does not contain a port, we should only attempt + // to call it when we know that the address contains a port + if strings.Count(host, ":") == 1 || (strings.Count(host, ":") >= 2 && strings.Contains(host, "]:")) { + hostname, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if port == "" { + port = portMap[scheme] + } + + return net.JoinHostPort(hostname, port) + } + + return net.JoinHostPort(host, portMap[scheme]) +} + +// https://docs.docker.com/registry/spec/auth/oauth/ +func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { + auth, err := bt.basic.Authorization() + if err != nil { + return nil, err + } + + u, err := url.Parse(bt.realm) + if err != nil { + return nil, err + } + + v := url.Values{} + v.Set("scope", strings.Join(bt.scopes, " ")) + if bt.service != "" { + v.Set("service", bt.service) + } + v.Set("client_id", defaultUserAgent) + if auth.IdentityToken != "" { + v.Set("grant_type", "refresh_token") + v.Set("refresh_token", auth.IdentityToken) + } else if auth.Username != "" && auth.Password != "" { + // TODO(#629): This is unreachable. + v.Set("grant_type", "password") + v.Set("username", auth.Username) + v.Set("password", auth.Password) + v.Set("access_type", "offline") + } + + client := http.Client{Transport: bt.inner} + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + // We don't want to log credentials. + ctx = redact.NewContext(ctx, "oauth token response contains credentials") + + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if err := CheckError(resp, http.StatusOK); err != nil { + if bt.basic == authn.Anonymous { + logs.Warn.Printf("No matching credentials were found for %q", bt.registry) + } + return nil, err + } + + return io.ReadAll(resp.Body) +} + +// https://docs.docker.com/registry/spec/auth/token/ +func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { + u, err := url.Parse(bt.realm) + if err != nil { + return nil, err + } + b := &basicTransport{ + inner: bt.inner, + auth: bt.basic, + target: u.Host, + } + client := http.Client{Transport: b} + + v := u.Query() + v["scope"] = bt.scopes + v.Set("service", bt.service) + u.RawQuery = v.Encode() + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + // We don't want to log credentials. + ctx = redact.NewContext(ctx, "basic token response contains credentials") + + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if err := CheckError(resp, http.StatusOK); err != nil { + if bt.basic == authn.Anonymous { + logs.Warn.Printf("No matching credentials were found for %q", bt.registry) + } + return nil, err + } + + return io.ReadAll(resp.Body) +} diff --git a/pkg/v1/remote/transport/bearer_test.go b/pkg/v1/remote/transport/bearer_test.go new file mode 100644 index 0000000..a03b1f9 --- /dev/null +++ b/pkg/v1/remote/transport/bearer_test.go @@ -0,0 +1,561 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" +) + +func TestBearerRefresh(t *testing.T) { + expectedToken := "Sup3rDup3rS3cr3tz" + expectedScope := "this-is-your-scope" + expectedService := "my-service.io" + + cases := []struct { + tokenKey string + wantErr bool + }{{ + tokenKey: "token", + wantErr: false, + }, { + tokenKey: "access_token", + wantErr: false, + }, { + tokenKey: "tolkien", + wantErr: true, + }} + + for _, tc := range cases { + t.Run(tc.tokenKey, func(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hdr := r.Header.Get("Authorization") + if !strings.HasPrefix(hdr, "Basic ") { + t.Errorf("Header.Get(Authorization); got %v, want Basic prefix", hdr) + } + if got, want := r.FormValue("scope"), expectedScope; got != want { + t.Errorf("FormValue(scope); got %v, want %v", got, want) + } + if got, want := r.FormValue("service"), expectedService; got != want { + t.Errorf("FormValue(service); got %v, want %v", got, want) + } + w.Write([]byte(fmt.Sprintf(`{%q: %q}`, tc.tokenKey, expectedToken))) + })) + defer server.Close() + + basic := &authn.Basic{Username: "foo", Password: "bar"} + registry, err := name.NewRegistry(expectedService, name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + bt := &bearerTransport{ + inner: http.DefaultTransport, + basic: basic, + registry: registry, + realm: server.URL, + scopes: []string{expectedScope}, + service: expectedService, + scheme: "http", + } + + if err := bt.refresh(context.Background()); (err != nil) != tc.wantErr { + t.Errorf("refresh() = %v", err) + } + }) + } +} + +func TestBearerTransport(t *testing.T) { + expectedToken := "sdkjhfskjdhfkjshdf" + + blobServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // We don't expect the blobServer to receive bearer tokens. + if got := r.Header.Get("Authorization"); got != "" { + t.Errorf("Header.Get(Authorization); got %v, want empty string", got) + } + w.WriteHeader(http.StatusOK) + })) + defer blobServer.Close() + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("Authorization"), "Bearer "+expectedToken; got != want { + t.Errorf("Header.Get(Authorization); got %v, want %v", got, want) + } + if r.URL.Path == "/v2/auth" { + http.Redirect(w, r, "/redirect", http.StatusMovedPermanently) + return + } + if strings.Contains(r.URL.Path, "blobs") { + http.Redirect(w, r, blobServer.URL, http.StatusFound) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + if err != nil { + t.Errorf("Unexpected error during url.Parse: %v", err) + } + registry, err := name.NewRegistry(u.Host, name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + client := http.Client{Transport: &bearerTransport{ + inner: &http.Transport{}, + bearer: authn.AuthConfig{RegistryToken: expectedToken}, + registry: registry, + scheme: "http", + }} + + _, err = client.Get(fmt.Sprintf("http://%s/v2/auth", u.Host)) + if err != nil { + t.Errorf("Unexpected error during Get: %v", err) + } + + _, err = client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Errorf("Unexpected error during Get: %v", err) + } +} + +func TestBearerTransportTokenRefresh(t *testing.T) { + initialToken := "foo" + refreshedToken := "bar" + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hdr := r.Header.Get("Authorization") + if hdr == "Bearer "+refreshedToken { + w.WriteHeader(http.StatusOK) + return + } + if strings.HasPrefix(hdr, "Basic ") { + w.Write([]byte(fmt.Sprintf(`{"token": %q}`, refreshedToken))) + } + + w.Header().Set("WWW-Authenticate", "scope=foo") + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + registry, err := name.NewRegistry(u.Host, name.WeakValidation) + if err != nil { + t.Fatalf("Unexpected error during NewRegistry: %v", err) + } + + // Pass Username/Password + transport := &bearerTransport{ + inner: http.DefaultTransport, + bearer: authn.AuthConfig{RegistryToken: initialToken}, + basic: &authn.Basic{Username: "foo", Password: "bar"}, + registry: registry, + realm: server.URL, + scheme: "http", + } + client := http.Client{Transport: transport} + + res, err := client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Errorf("Unexpected error during client.Get: %v", err) + return + } + if res.StatusCode != http.StatusOK { + t.Errorf("client.Get final StatusCode got %v, want: %v", res.StatusCode, http.StatusOK) + } + if got, want := transport.bearer.RegistryToken, refreshedToken; got != want { + t.Errorf("Expected Bearer token to be refreshed, got %v, want %v", got, want) + } + + // Pass RegistryToken directly + transport.bearer = authn.AuthConfig{RegistryToken: initialToken} + transport.basic = &authn.Bearer{Token: refreshedToken} + client = http.Client{Transport: transport} + + res, err = client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Errorf("Unexpected error during client.Get: %v", err) + return + } + if res.StatusCode != http.StatusOK { + t.Errorf("client.Get final StatusCode got %v, want: %v", res.StatusCode, http.StatusOK) + } + if got, want := transport.bearer.RegistryToken, refreshedToken; got != want { + t.Errorf("Expected Bearer token to be refreshed, got %v, want %v", got, want) + } +} + +func TestBearerTransportOauthRefresh(t *testing.T) { + initialToken := "foo" + accessToken := "bar" + refreshToken := "baz" + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + if err := r.ParseForm(); err != nil { + t.Fatal(err) + } + if it := r.FormValue("refresh_token"); it != initialToken { + t.Errorf("want %s got %s", initialToken, it) + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf(`{"access_token": %q, "refresh_token": %q}`, accessToken, refreshToken))) + return + } + + hdr := r.Header.Get("Authorization") + if hdr == "Bearer "+accessToken { + w.WriteHeader(http.StatusOK) + return + } + + w.Header().Set("WWW-Authenticate", "scope=foo") + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + registry, err := name.NewRegistry(u.Host, name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + transport := &bearerTransport{ + inner: http.DefaultTransport, + basic: authn.FromConfig(authn.AuthConfig{IdentityToken: initialToken}), + registry: registry, + realm: server.URL, + scheme: "http", + scopes: []string{"myscope"}, + service: u.Host, + } + client := http.Client{Transport: transport} + + res, err := client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Fatalf("Unexpected error during client.Get: %v", err) + } + if res.StatusCode != http.StatusOK { + t.Errorf("client.Get final StatusCode got %v, want: %v", res.StatusCode, http.StatusOK) + } + if want, got := transport.bearer.RegistryToken, accessToken; want != got { + t.Errorf("Expected Bearer token to be refreshed, got %v, want %v", got, want) + } + basicAuthConfig, err := transport.basic.Authorization() + if err != nil { + t.Fatal(err) + } + if got, want := basicAuthConfig.IdentityToken, refreshToken; got != want { + t.Errorf("Expected Basic IdentityToken to be refreshed, got %v, want %v", got, want) + } +} + +func TestBearerTransportOauth404Fallback(t *testing.T) { + basicAuth := "basic_auth" + identityToken := "identity_token" + accessToken := "access_token" + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + w.WriteHeader(http.StatusNotFound) + } + + hdr := r.Header.Get("Authorization") + if hdr == "Basic "+basicAuth { + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf(`{"access_token": %q}`, accessToken))) + } + if hdr == "Bearer "+accessToken { + w.WriteHeader(http.StatusOK) + return + } + + w.Header().Set("WWW-Authenticate", "scope=foo") + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + registry, err := name.NewRegistry(u.Host, name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + transport := &bearerTransport{ + inner: http.DefaultTransport, + basic: authn.FromConfig(authn.AuthConfig{ + IdentityToken: identityToken, + Auth: basicAuth, + }), + registry: registry, + realm: server.URL, + scheme: "http", + scopes: []string{"myscope"}, + service: u.Host, + } + client := http.Client{Transport: transport} + + res, err := client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Fatalf("Unexpected error during client.Get: %v", err) + } + if res.StatusCode != http.StatusOK { + t.Errorf("client.Get final StatusCode got %v, want: %v", res.StatusCode, http.StatusOK) + } + if got, want := transport.bearer.RegistryToken, accessToken; got != want { + t.Errorf("Expected Bearer token to be refreshed, got %v, want %v", got, want) + } +} + +type recorder struct { + reqs []*http.Request + resp *http.Response + err error +} + +func newRecorder(resp *http.Response, err error) *recorder { + return &recorder{ + reqs: []*http.Request{}, + resp: resp, + err: err, + } +} + +func (r *recorder) RoundTrip(in *http.Request) (*http.Response, error) { + r.reqs = append(r.reqs, in) + return r.resp, r.err +} + +func TestSchemeOverride(t *testing.T) { + // Record the requests we get in the inner transport. + cannedResponse := http.Response{ + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + } + rec := newRecorder(&cannedResponse, nil) + registry, err := name.NewRegistry("example.com") + if err != nil { + t.Fatalf("Unexpected error during NewRegistry: %v", err) + } + st := &schemeTransport{ + inner: rec, + registry: registry, + scheme: "http", + } + + // We should see the scheme be overridden to "http" for the registry, but the + // scheme for the token server should be unchanged. + tests := []struct { + url string + wantScheme string + }{{ + url: "https://example.com", + wantScheme: "http", + }, { + url: "https://token.example.com", + wantScheme: "https", + }} + + for i, tt := range tests { + req, err := http.NewRequest("GET", tt.url, nil) + if err != nil { + t.Fatalf("Unexpected error during NewRequest: %v", err) + } + + if _, err := st.RoundTrip(req); err != nil { + t.Fatalf("Unexpected error during RoundTrip: %v", err) + } + + if got, want := rec.reqs[i].URL.Scheme, tt.wantScheme; got != want { + t.Errorf("Wrong scheme: wanted %v, got %v", want, got) + } + } +} + +func TestCanonicalAddressResolution(t *testing.T) { + registry, err := name.NewRegistry("does-not-matter", name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + tests := []struct { + registry name.Registry + scheme string + address string + want string + }{{ + registry: registry, + scheme: "http", + address: "registry.example.com", + want: "registry.example.com:80", + }, { + registry: registry, + scheme: "http", + address: "registry.example.com:12345", + want: "registry.example.com:12345", + }, { + registry: registry, + scheme: "https", + address: "registry.example.com", + want: "registry.example.com:443", + }, { + registry: registry, + scheme: "https", + address: "registry.example.com:12345", + want: "registry.example.com:12345", + }, { + registry: registry, + scheme: "http", + address: "registry.example.com:", + want: "registry.example.com:80", + }, { + registry: registry, + scheme: "https", + address: "registry.example.com:", + want: "registry.example.com:443", + }, { + registry: registry, + scheme: "http", + address: "2001:db8::1", + want: "[2001:db8::1]:80", + }, { + registry: registry, + scheme: "https", + address: "2001:db8::1", + want: "[2001:db8::1]:443", + }, { + registry: registry, + scheme: "http", + address: "[2001:db8::1]:12345", + want: "[2001:db8::1]:12345", + }, { + registry: registry, + scheme: "https", + address: "[2001:db8::1]:12345", + want: "[2001:db8::1]:12345", + }, { + registry: registry, + scheme: "http", + address: "[2001:db8::1]:", + want: "[2001:db8::1]:80", + }, { + registry: registry, + scheme: "https", + address: "[2001:db8::1]:", + want: "[2001:db8::1]:443", + }, { + registry: registry, + scheme: "https", + address: "something:is::wrong]:", + want: "something:is::wrong]:", + }} + + for _, tt := range tests { + got := canonicalAddress(tt.address, tt.scheme) + if got != tt.want { + t.Errorf("Wrong canonical host: wanted %v got %v", tt.want, got) + } + } +} + +func TestInsufficientScope(t *testing.T) { + wrong := "the-wrong-scope" + right := "the-right-scope" + realm := "" + expectedService := "my-service.io" + passed := false + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + + scopes := query["scope"] + switch { + case len(scopes) == 0: + if !passed { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Bearer realm=%q,scope=%q", realm, right)) + w.WriteHeader(http.StatusUnauthorized) + } + case len(scopes) == 1: + w.Write([]byte(`{"token": "arbitrary-token"}`)) + default: + passed = true + w.Write([]byte(`{"token": "arbitrary-token-2"}`)) + } + })) + defer server.Close() + + basic := &authn.Basic{Username: "foo", Password: "bar"} + u, err := url.Parse(server.URL) + if err != nil { + t.Error("Unexpected error during url.Parse: ", err) + } + realm = u.Host + + registry, err := name.NewRegistry(expectedService, name.WeakValidation) + if err != nil { + t.Error("Unexpected error during NewRegistry: ", err) + } + + bt := &bearerTransport{ + inner: http.DefaultTransport, + basic: basic, + registry: registry, + realm: server.URL, + scopes: []string{wrong}, + service: expectedService, + scheme: "http", + } + + client := http.Client{Transport: bt} + + res, err := client.Get(fmt.Sprintf("http://%s/v2/foo/bar/blobs/blah", u.Host)) + if err != nil { + t.Error("Unexpected error during client.Get: ", err) + return + } + if res.StatusCode != http.StatusOK { + t.Errorf("client.Get final StatusCode got %v, want: %v", res.StatusCode, http.StatusOK) + } + + if !passed { + t.Error("didn't refresh insufficient scope") + } +} diff --git a/pkg/v1/remote/transport/doc.go b/pkg/v1/remote/transport/doc.go new file mode 100644 index 0000000..ff7025b --- /dev/null +++ b/pkg/v1/remote/transport/doc.go @@ -0,0 +1,18 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package transport provides facilities for setting up an authenticated +// http.RoundTripper given an Authenticator and base RoundTripper. See +// transport.New for more information. +package transport diff --git a/pkg/v1/remote/transport/error.go b/pkg/v1/remote/transport/error.go new file mode 100644 index 0000000..c0e4337 --- /dev/null +++ b/pkg/v1/remote/transport/error.go @@ -0,0 +1,173 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/google/go-containerregistry/internal/redact" +) + +// Error implements error to support the following error specification: +// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors +type Error struct { + Errors []Diagnostic `json:"errors,omitempty"` + // The http status code returned. + StatusCode int + // The request that failed. + Request *http.Request + // The raw body if we couldn't understand it. + rawBody string +} + +// Check that Error implements error +var _ error = (*Error)(nil) + +// Error implements error +func (e *Error) Error() string { + prefix := "" + if e.Request != nil { + prefix = fmt.Sprintf("%s %s: ", e.Request.Method, redact.URL(e.Request.URL)) + } + return prefix + e.responseErr() +} + +func (e *Error) responseErr() string { + switch len(e.Errors) { + case 0: + if len(e.rawBody) == 0 { + if e.Request != nil && e.Request.Method == http.MethodHead { + return fmt.Sprintf("unexpected status code %d %s (HEAD responses have no body, use GET for details)", e.StatusCode, http.StatusText(e.StatusCode)) + } + return fmt.Sprintf("unexpected status code %d %s", e.StatusCode, http.StatusText(e.StatusCode)) + } + return fmt.Sprintf("unexpected status code %d %s: %s", e.StatusCode, http.StatusText(e.StatusCode), e.rawBody) + case 1: + return e.Errors[0].String() + default: + var errors []string + for _, d := range e.Errors { + errors = append(errors, d.String()) + } + return fmt.Sprintf("multiple errors returned: %s", + strings.Join(errors, "; ")) + } +} + +// Temporary returns whether the request that preceded the error is temporary. +func (e *Error) Temporary() bool { + if len(e.Errors) == 0 { + _, ok := temporaryStatusCodes[e.StatusCode] + return ok + } + for _, d := range e.Errors { + if _, ok := temporaryErrorCodes[d.Code]; !ok { + return false + } + } + return true +} + +// Diagnostic represents a single error returned by a Docker registry interaction. +type Diagnostic struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail any `json:"detail,omitempty"` +} + +// String stringifies the Diagnostic in the form: $Code: $Message[; $Detail] +func (d Diagnostic) String() string { + msg := fmt.Sprintf("%s: %s", d.Code, d.Message) + if d.Detail != nil { + msg = fmt.Sprintf("%s; %v", msg, d.Detail) + } + return msg +} + +// ErrorCode is an enumeration of supported error codes. +type ErrorCode string + +// The set of error conditions a registry may return: +// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors-2 +const ( + BlobUnknownErrorCode ErrorCode = "BLOB_UNKNOWN" + BlobUploadInvalidErrorCode ErrorCode = "BLOB_UPLOAD_INVALID" + BlobUploadUnknownErrorCode ErrorCode = "BLOB_UPLOAD_UNKNOWN" + DigestInvalidErrorCode ErrorCode = "DIGEST_INVALID" + ManifestBlobUnknownErrorCode ErrorCode = "MANIFEST_BLOB_UNKNOWN" + ManifestInvalidErrorCode ErrorCode = "MANIFEST_INVALID" + ManifestUnknownErrorCode ErrorCode = "MANIFEST_UNKNOWN" + ManifestUnverifiedErrorCode ErrorCode = "MANIFEST_UNVERIFIED" + NameInvalidErrorCode ErrorCode = "NAME_INVALID" + NameUnknownErrorCode ErrorCode = "NAME_UNKNOWN" + SizeInvalidErrorCode ErrorCode = "SIZE_INVALID" + TagInvalidErrorCode ErrorCode = "TAG_INVALID" + UnauthorizedErrorCode ErrorCode = "UNAUTHORIZED" + DeniedErrorCode ErrorCode = "DENIED" + UnsupportedErrorCode ErrorCode = "UNSUPPORTED" + TooManyRequestsErrorCode ErrorCode = "TOOMANYREQUESTS" + UnknownErrorCode ErrorCode = "UNKNOWN" + + // This isn't defined by either docker or OCI spec, but is defined by docker/distribution: + // https://github.com/distribution/distribution/blob/6a977a5a754baa213041443f841705888107362a/registry/api/errcode/register.go#L60 + UnavailableErrorCode ErrorCode = "UNAVAILABLE" +) + +// TODO: Include other error types. +var temporaryErrorCodes = map[ErrorCode]struct{}{ + BlobUploadInvalidErrorCode: {}, + TooManyRequestsErrorCode: {}, + UnknownErrorCode: {}, + UnavailableErrorCode: {}, +} + +var temporaryStatusCodes = map[int]struct{}{ + http.StatusRequestTimeout: {}, + http.StatusInternalServerError: {}, + http.StatusBadGateway: {}, + http.StatusServiceUnavailable: {}, + http.StatusGatewayTimeout: {}, +} + +// CheckError returns a structured error if the response status is not in codes. +func CheckError(resp *http.Response, codes ...int) error { + for _, code := range codes { + if resp.StatusCode == code { + // This is one of the supported status codes. + return nil + } + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors + structuredError := &Error{} + + // This can fail if e.g. the response body is not valid JSON. That's fine, + // we'll construct an appropriate error string from the body and status code. + _ = json.Unmarshal(b, structuredError) + + structuredError.rawBody = string(b) + structuredError.StatusCode = resp.StatusCode + structuredError.Request = resp.Request + + return structuredError +} diff --git a/pkg/v1/remote/transport/error_test.go b/pkg/v1/remote/transport/error_test.go new file mode 100644 index 0000000..e42ce3a --- /dev/null +++ b/pkg/v1/remote/transport/error_test.go @@ -0,0 +1,236 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "bytes" + "errors" + "io" + "net/http" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestTemporary(t *testing.T) { + tests := []struct { + error *Error + retry bool + }{{ + error: &Error{}, + retry: false, + }, { + error: &Error{ + Errors: []Diagnostic{{ + Code: BlobUploadInvalidErrorCode, + }}, + }, + retry: true, + }, { + error: &Error{ + Errors: []Diagnostic{{ + Code: BlobUploadInvalidErrorCode, + }, { + Code: DeniedErrorCode, + }}, + }, + retry: false, + }, { + error: &Error{ + Errors: []Diagnostic{{ + Code: TooManyRequestsErrorCode, + }}, + }, + retry: true, + }, { + error: &Error{ + Errors: []Diagnostic{{ + Code: UnavailableErrorCode, + }}, + }, + retry: true, + }, { + error: &Error{ + StatusCode: http.StatusInternalServerError, + }, + retry: true, + }} + + for _, test := range tests { + retry := test.error.Temporary() + + if test.retry != retry { + t.Errorf("Temporary(%s) = %t, wanted %t", test.error, retry, test.retry) + } + } +} + +func TestCheckErrorNil(t *testing.T) { + tests := []int{ + http.StatusOK, + http.StatusAccepted, + http.StatusCreated, + http.StatusMovedPermanently, + http.StatusInternalServerError, + } + + for _, code := range tests { + resp := &http.Response{StatusCode: code} + + if err := CheckError(resp, code); err != nil { + t.Errorf("CheckError(%d) = %v", code, err) + } + } +} + +func TestCheckErrorNotError(t *testing.T) { + tests := []struct { + code int + body string + msg string + request *http.Request + }{{ + code: http.StatusBadRequest, + body: "", + msg: "unexpected status code 400 Bad Request", + }, { + code: http.StatusUnauthorized, + // Valid JSON, but not a structured error -- we should still print the body. + body: `{"details":"incorrect username or password"}`, + msg: `unexpected status code 401 Unauthorized: {"details":"incorrect username or password"}`, + }, { + code: http.StatusUnauthorized, + body: "Not JSON", + msg: "GET https://example.com/somepath?access_token=REDACTED&scope=foo&service=bar: unexpected status code 401 Unauthorized: Not JSON", + request: &http.Request{ + Method: http.MethodGet, + URL: &url.URL{ + Scheme: "https", + Host: "example.com", + Path: "somepath", + RawQuery: url.Values{ + "scope": []string{"foo"}, + "service": []string{"bar"}, + "access_token": []string{"hunter2"}, + }.Encode(), + }, + }, + }, { + code: http.StatusUnauthorized, + body: "", + msg: "HEAD https://example.com/somepath: unexpected status code 401 Unauthorized (HEAD responses have no body, use GET for details)", + request: &http.Request{ + Method: http.MethodHead, + URL: &url.URL{ + Scheme: "https", + Host: "example.com", + Path: "somepath", + }, + }, + }} + + for _, test := range tests { + resp := &http.Response{ + StatusCode: test.code, + Body: io.NopCloser(bytes.NewBufferString(test.body)), + Request: test.request, + } + + err := CheckError(resp, http.StatusOK) + if err == nil { + t.Fatalf("CheckError(%d, %s) = nil, wanted error", test.code, test.body) + } + var terr *Error + if !errors.As(err, &terr) { + t.Fatalf("CheckError(%d, %s) = %v, wanted error type", test.code, test.body, err) + } + + if terr.StatusCode != test.code { + t.Errorf("Incorrect status code, got %d, want %d", terr.StatusCode, test.code) + } + + if terr.Error() != test.msg { + t.Errorf("Incorrect message, got %q, want %q", terr.Error(), test.msg) + } + } +} + +func TestCheckErrorWithError(t *testing.T) { + tests := []struct { + name string + code int + errorBody string + msg string + }{{ + name: "Invalid name error", + code: http.StatusBadRequest, + errorBody: `{"errors":[{"code":"NAME_INVALID","message":"a message for you"}],"StatusCode":400}`, + msg: "NAME_INVALID: a message for you", + }, { + name: "Only status code is provided", + code: http.StatusBadRequest, + errorBody: `{"StatusCode":400}`, + msg: "unexpected status code 400 Bad Request: {\"StatusCode\":400}", + }, { + name: "Multiple diagnostics", + code: http.StatusBadRequest, + errorBody: `{"errors":[{"code":"NAME_INVALID","message":"a message for you"}, {"code":"SIZE_INVALID","message":"another message for you", "detail": "with some details"}],"StatusCode":400,"Request":null}`, + msg: "multiple errors returned: NAME_INVALID: a message for you; SIZE_INVALID: another message for you; with some details", + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resp := &http.Response{ + StatusCode: test.code, + Body: io.NopCloser(bytes.NewBuffer([]byte(test.errorBody))), + } + + var terr *Error + if err := CheckError(resp, http.StatusOK); err == nil { + t.Errorf("CheckError(%d, %s) = nil, wanted error", test.code, test.errorBody) + } else if !errors.As(err, &terr) { + t.Errorf("CheckError(%d, %s) = %T, wanted *transport.Error", test.code, test.errorBody, err) + } else if diff := cmp.Diff(test.msg, err.Error()); diff != "" { + t.Errorf("CheckError(%d, %s).Error(); (-want +got) %s", test.code, test.errorBody, diff) + } + }) + } +} + +func TestBodyError(t *testing.T) { + expectedErr := errors.New("whoops") + resp := &http.Response{ + StatusCode: http.StatusOK, + Body: &errReadCloser{expectedErr}, + } + if err := CheckError(resp, http.StatusNotFound); err == nil { + t.Errorf("CheckError() = nil, wanted error %v", expectedErr) + } else if !errors.Is(err, expectedErr) { + t.Errorf("CheckError() = %v, wanted %v", err, expectedErr) + } +} + +type errReadCloser struct { + err error +} + +func (e *errReadCloser) Read(p []byte) (int, error) { + return 0, e.err +} + +func (e *errReadCloser) Close() error { + return e.err +} diff --git a/pkg/v1/remote/transport/logger.go b/pkg/v1/remote/transport/logger.go new file mode 100644 index 0000000..c341f84 --- /dev/null +++ b/pkg/v1/remote/transport/logger.go @@ -0,0 +1,91 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "fmt" + "net/http" + "net/http/httputil" + "time" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/pkg/logs" +) + +type logTransport struct { + inner http.RoundTripper +} + +// NewLogger returns a transport that logs requests and responses to +// github.com/google/go-containerregistry/pkg/logs.Debug. +func NewLogger(inner http.RoundTripper) http.RoundTripper { + return &logTransport{inner} +} + +func (t *logTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + // Inspired by: github.com/motemen/go-loghttp + + // We redact token responses and binary blobs in response/request. + omitBody, reason := redact.FromContext(in.Context()) + if omitBody { + logs.Debug.Printf("--> %s %s [body redacted: %s]", in.Method, in.URL, reason) + } else { + logs.Debug.Printf("--> %s %s", in.Method, in.URL) + } + + // Save these headers so we can redact Authorization. + savedHeaders := in.Header.Clone() + if in.Header != nil && in.Header.Get("authorization") != "" { + in.Header.Set("authorization", "") + } + + b, err := httputil.DumpRequestOut(in, !omitBody) + if err == nil { + logs.Debug.Println(string(b)) + } else { + logs.Debug.Printf("Failed to dump request %s %s: %v", in.Method, in.URL, err) + } + + // Restore the non-redacted headers. + in.Header = savedHeaders + + start := time.Now() + out, err = t.inner.RoundTrip(in) + duration := time.Since(start) + if err != nil { + logs.Debug.Printf("<-- %v %s %s (%s)", err, in.Method, in.URL, duration) + } + if out != nil { + msg := fmt.Sprintf("<-- %d", out.StatusCode) + if out.Request != nil { + msg = fmt.Sprintf("%s %s", msg, out.Request.URL) + } + msg = fmt.Sprintf("%s (%s)", msg, duration) + + if omitBody { + msg = fmt.Sprintf("%s [body redacted: %s]", msg, reason) + } + + logs.Debug.Print(msg) + + b, err := httputil.DumpResponse(out, !omitBody) + if err == nil { + logs.Debug.Println(string(b)) + } else { + logs.Debug.Printf("Failed to dump response %s %s: %v", in.Method, in.URL, err) + } + } + return +} diff --git a/pkg/v1/remote/transport/logger_test.go b/pkg/v1/remote/transport/logger_test.go new file mode 100644 index 0000000..d5b57d3 --- /dev/null +++ b/pkg/v1/remote/transport/logger_test.go @@ -0,0 +1,93 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/pkg/logs" +) + +func TestLogger(t *testing.T) { + canary := "logs.Debug canary" + secret := "super secret do not log" + auth := "my token pls do not log" + reason := "should not log the secret" + + ctx := redact.NewContext(context.Background(), reason) + + req, err := http.NewRequestWithContext(ctx, "GET", "http://example.com", nil) + if err != nil { + t.Fatalf("Unexpected error during NewRequest: %v", err) + } + req.Header.Set("authorization", auth) + + var b bytes.Buffer + logs.Debug.SetOutput(&b) + cannedResponse := http.Response{ + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Header: http.Header{ + "Foo": []string{canary}, + }, + Body: io.NopCloser(strings.NewReader(secret)), + Request: req, + } + tr := NewLogger(newRecorder(&cannedResponse, nil)) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("Unexpected error during RoundTrip: %v", err) + } + + logged := b.String() + if !strings.Contains(logged, canary) { + t.Errorf("Expected logs to contain %s, got %s", canary, logged) + } + if !strings.Contains(logged, reason) { + t.Errorf("Expected logs to contain %s, got %s", canary, logged) + } + if strings.Contains(logged, secret) { + t.Errorf("Expected logs NOT to contain %s, got %s", secret, logged) + } + if strings.Contains(logged, auth) { + t.Errorf("Expected logs NOT to contain %s, got %s", auth, logged) + } +} + +func TestLoggerError(t *testing.T) { + canary := "logs.Debug canary ERROR" + req, err := http.NewRequest("GET", "http://example.com", nil) + if err != nil { + t.Fatalf("Unexpected error during NewRequest: %v", err) + } + + var b bytes.Buffer + logs.Debug.SetOutput(&b) + tr := NewLogger(newRecorder(nil, errors.New(canary))) + if _, err := tr.RoundTrip(req); err == nil { + t.Fatalf("Expected error during RoundTrip, got nil") + } + + logged := b.String() + if !strings.Contains(logged, canary) { + t.Errorf("Expected logs to contain %s, got %s", canary, logged) + } +} diff --git a/pkg/v1/remote/transport/ping.go b/pkg/v1/remote/transport/ping.go new file mode 100644 index 0000000..d852ef8 --- /dev/null +++ b/pkg/v1/remote/transport/ping.go @@ -0,0 +1,227 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + authchallenge "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" +) + +type challenge string + +const ( + anonymous challenge = "anonymous" + basic challenge = "basic" + bearer challenge = "bearer" +) + +// 300ms is the default fallback period for go's DNS dialer but we could make this configurable. +var fallbackDelay = 300 * time.Millisecond + +type pingResp struct { + challenge challenge + + // Following the challenge there are often key/value pairs + // e.g. Bearer service="gcr.io",realm="https://auth.gcr.io/v36/tokenz" + parameters map[string]string + + // The registry's scheme to use. Communicates whether we fell back to http. + scheme string +} + +func (c challenge) Canonical() challenge { + return challenge(strings.ToLower(string(c))) +} + +func ping(ctx context.Context, reg name.Registry, t http.RoundTripper) (*pingResp, error) { + // This first attempts to use "https" for every request, falling back to http + // if the registry matches our localhost heuristic or if it is intentionally + // set to insecure via name.NewInsecureRegistry. + schemes := []string{"https"} + if reg.Scheme() == "http" { + schemes = append(schemes, "http") + } + if len(schemes) == 1 { + return pingSingle(ctx, reg, t, schemes[0]) + } + return pingParallel(ctx, reg, t, schemes) +} + +func pingSingle(ctx context.Context, reg name.Registry, t http.RoundTripper, scheme string) (*pingResp, error) { + client := http.Client{Transport: t} + url := fmt.Sprintf("%s://%s/v2/", scheme, reg.Name()) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer func() { + // By draining the body, make sure to reuse the connection made by + // the ping for the following access to the registry + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + // If we get a 200, then no authentication is needed. + return &pingResp{ + challenge: anonymous, + scheme: scheme, + }, nil + case http.StatusUnauthorized: + if challenges := authchallenge.ResponseChallenges(resp); len(challenges) != 0 { + // If we hit more than one, let's try to find one that we know how to handle. + wac := pickFromMultipleChallenges(challenges) + return &pingResp{ + challenge: challenge(wac.Scheme).Canonical(), + parameters: wac.Parameters, + scheme: scheme, + }, nil + } + // Otherwise, just return the challenge without parameters. + return &pingResp{ + challenge: challenge(resp.Header.Get("WWW-Authenticate")).Canonical(), + scheme: scheme, + }, nil + default: + return nil, CheckError(resp, http.StatusOK, http.StatusUnauthorized) + } +} + +// Based on the golang happy eyeballs dialParallel impl in net/dial.go. +func pingParallel(ctx context.Context, reg name.Registry, t http.RoundTripper, schemes []string) (*pingResp, error) { + returned := make(chan struct{}) + defer close(returned) + + type pingResult struct { + *pingResp + error + primary bool + done bool + } + + results := make(chan pingResult) + + startRacer := func(ctx context.Context, scheme string) { + pr, err := pingSingle(ctx, reg, t, scheme) + select { + case results <- pingResult{pingResp: pr, error: err, primary: scheme == "https", done: true}: + case <-returned: + if pr != nil { + logs.Debug.Printf("%s lost race", scheme) + } + } + } + + var primary, fallback pingResult + + primaryCtx, primaryCancel := context.WithCancel(ctx) + defer primaryCancel() + go startRacer(primaryCtx, schemes[0]) + + fallbackTimer := time.NewTimer(fallbackDelay) + defer fallbackTimer.Stop() + + for { + select { + case <-fallbackTimer.C: + fallbackCtx, fallbackCancel := context.WithCancel(ctx) + defer fallbackCancel() + go startRacer(fallbackCtx, schemes[1]) + + case res := <-results: + if res.error == nil { + return res.pingResp, nil + } + if res.primary { + primary = res + } else { + fallback = res + } + if primary.done && fallback.done { + return nil, multierrs([]error{primary.error, fallback.error}) + } + if res.primary && fallbackTimer.Stop() { + // Primary failed and we haven't started the fallback, + // reset time to start fallback immediately. + fallbackTimer.Reset(0) + } + } + } +} + +func pickFromMultipleChallenges(challenges []authchallenge.Challenge) authchallenge.Challenge { + // It might happen there are multiple www-authenticate headers, e.g. `Negotiate` and `Basic`. + // Picking simply the first one could result eventually in `unrecognized challenge` error, + // that's why we're looping through the challenges in search for one that can be handled. + allowedSchemes := []string{"basic", "bearer"} + + for _, wac := range challenges { + currentScheme := strings.ToLower(wac.Scheme) + for _, allowed := range allowedSchemes { + if allowed == currentScheme { + return wac + } + } + } + + return challenges[0] +} + +type multierrs []error + +func (m multierrs) Error() string { + var b strings.Builder + hasWritten := false + for _, err := range m { + if hasWritten { + b.WriteString("; ") + } + hasWritten = true + b.WriteString(err.Error()) + } + return b.String() +} + +func (m multierrs) As(target any) bool { + for _, err := range m { + if errors.As(err, target) { + return true + } + } + return false +} + +func (m multierrs) Is(target error) bool { + for _, err := range m { + if errors.Is(err, target) { + return true + } + } + return false +} diff --git a/pkg/v1/remote/transport/ping_test.go b/pkg/v1/remote/transport/ping_test.go new file mode 100644 index 0000000..c2ad119 --- /dev/null +++ b/pkg/v1/remote/transport/ping_test.go @@ -0,0 +1,260 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" +) + +var ( + testRegistry, _ = name.NewRegistry("localhost:8080", name.StrictValidation) +) + +func TestPingNoChallenge(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err != nil { + t.Errorf("ping() = %v", err) + } + if pr.challenge != anonymous { + t.Errorf("ping(); got %v, want %v", pr.challenge, anonymous) + } + if pr.scheme != "http" { + t.Errorf("ping(); got %v, want %v", pr.scheme, "http") + } +} + +func TestPingBasicChallengeNoParams(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `BASIC`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err != nil { + t.Errorf("ping() = %v", err) + } + if pr.challenge != basic { + t.Errorf("ping(); got %v, want %v", pr.challenge, basic) + } + if got, want := len(pr.parameters), 0; got != want { + t.Errorf("ping(); got %v, want %v", got, want) + } +} + +func TestPingBearerChallengeWithParams(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Bearer realm="http://auth.example.com/token"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err != nil { + t.Errorf("ping() = %v", err) + } + if pr.challenge != bearer { + t.Errorf("ping(); got %v, want %v", pr.challenge, bearer) + } + if got, want := len(pr.parameters), 1; got != want { + t.Errorf("ping(); got %v, want %v", got, want) + } +} + +func TestPingMultipleChallenges(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("WWW-Authenticate", "Negotiate") + w.Header().Add("WWW-Authenticate", `Basic realm="http://auth.example.com/token"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err != nil { + t.Errorf("ping() = %v", err) + } + if pr.challenge != basic { + t.Errorf("ping(); got %v, want %v", pr.challenge, basic) + } + if got, want := len(pr.parameters), 1; got != want { + t.Errorf("ping(); got %v, want %v", got, want) + } +} + +func TestPingMultipleNotSupportedChallenges(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("WWW-Authenticate", "Negotiate") + w.Header().Add("WWW-Authenticate", "Digest") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err != nil { + t.Errorf("ping() = %v", err) + } + if pr.challenge != "negotiate" { + t.Errorf("ping(); got %v, want %v", pr.challenge, "negotiate") + } +} + +func TestUnsupportedStatus(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Bearer realm="http://auth.example.com/token`) + http.Error(w, "Forbidden", http.StatusForbidden) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + pr, err := ping(context.Background(), testRegistry, tprt) + if err == nil { + t.Errorf("ping() = %v", pr) + } +} + +func TestPingHttpFallback(t *testing.T) { + tests := []struct { + reg name.Registry + wantCount int64 + err string + contains []string + }{{ + reg: mustRegistry("gcr.io"), + wantCount: 1, + err: `Get "https://gcr.io/v2/": http: server gave HTTP response to HTTPS client`, + }, { + reg: mustRegistry("ko.local"), + wantCount: 2, + }, { + reg: mustInsecureRegistry("us.gcr.io"), + wantCount: 0, + contains: []string{"https://us.gcr.io/v2/", "http://us.gcr.io/v2/"}, + }} + + gotCount := int64(0) + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&gotCount, 1) + if r.URL.Scheme != "http" { + // Sleep a little bit so we can exercise the + // happy eyeballs race. + time.Sleep(5 * time.Millisecond) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + fallbackDelay = 2 * time.Millisecond + + for _, test := range tests { + // This is the last one, fatal error it. + if strings.Contains(test.reg.String(), "us.gcr.io") { + server.Close() + } + + _, err := ping(context.Background(), test.reg, tprt) + if got, want := gotCount, test.wantCount; got != want { + t.Errorf("%s: got %d requests, wanted %d", test.reg.String(), got, want) + } + gotCount = 0 + + if err == nil { + if test.err != "" { + t.Error("expected err, got nil") + } + continue + } + if len(test.contains) != 0 { + for _, c := range test.contains { + if !strings.Contains(err.Error(), c) { + t.Errorf("expected err to contain %q but did not: %q", c, err) + } + } + } else if got, want := err.Error(), test.err; got != want { + t.Errorf("got %q want %q", got, want) + } + } +} + +func mustRegistry(r string) name.Registry { + reg, err := name.NewRegistry(r) + if err != nil { + panic(err) + } + return reg +} + +func mustInsecureRegistry(r string) name.Registry { + reg, err := name.NewRegistry(r, name.Insecure) + if err != nil { + panic(err) + } + return reg +} diff --git a/pkg/v1/remote/transport/retry.go b/pkg/v1/remote/transport/retry.go new file mode 100644 index 0000000..e5621e3 --- /dev/null +++ b/pkg/v1/remote/transport/retry.go @@ -0,0 +1,111 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "time" + + "github.com/google/go-containerregistry/internal/retry" +) + +// Sleep for 0.1 then 0.3 seconds. This should cover networking blips. +var defaultBackoff = retry.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, +} + +var _ http.RoundTripper = (*retryTransport)(nil) + +// retryTransport wraps a RoundTripper and retries temporary network errors. +type retryTransport struct { + inner http.RoundTripper + backoff retry.Backoff + predicate retry.Predicate + codes []int +} + +// Option is a functional option for retryTransport. +type Option func(*options) + +type options struct { + backoff retry.Backoff + predicate retry.Predicate + codes []int +} + +// Backoff is an alias of retry.Backoff to expose this configuration option to consumers of this lib +type Backoff = retry.Backoff + +// WithRetryBackoff sets the backoff for retry operations. +func WithRetryBackoff(backoff Backoff) Option { + return func(o *options) { + o.backoff = backoff + } +} + +// WithRetryPredicate sets the predicate for retry operations. +func WithRetryPredicate(predicate func(error) bool) Option { + return func(o *options) { + o.predicate = predicate + } +} + +// WithRetryStatusCodes sets which http response codes will be retried. +func WithRetryStatusCodes(codes ...int) Option { + return func(o *options) { + o.codes = codes + } +} + +// NewRetry returns a transport that retries errors. +func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper { + o := &options{ + backoff: defaultBackoff, + predicate: retry.IsTemporary, + } + + for _, opt := range opts { + opt(o) + } + + return &retryTransport{ + inner: inner, + backoff: o.backoff, + predicate: o.predicate, + codes: o.codes, + } +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + roundtrip := func() error { + out, err = t.inner.RoundTrip(in) + if !retry.Ever(in.Context()) { + return nil + } + if out != nil { + for _, code := range t.codes { + if out.StatusCode == code { + return CheckError(out) + } + } + } + return err + } + retry.Retry(roundtrip, t.predicate, t.backoff) + return +} diff --git a/pkg/v1/remote/transport/retry_test.go b/pkg/v1/remote/transport/retry_test.go new file mode 100644 index 0000000..ded0ce0 --- /dev/null +++ b/pkg/v1/remote/transport/retry_test.go @@ -0,0 +1,177 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/internal/retry" +) + +type mockTransport struct { + errs []error + resps []*http.Response + count int +} + +func (t *mockTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + defer func() { t.count++ }() + if t.count < len(t.resps) { + out = t.resps[t.count] + } + if t.count < len(t.errs) { + err = t.errs[t.count] + } + return +} + +type perm struct{} + +func (e perm) Error() string { + return "permanent error" +} + +type temp struct{} + +func (e temp) Error() string { + return "temporary error" +} + +func (e temp) Temporary() bool { + return true +} + +func resp(code int) *http.Response { + return &http.Response{ + StatusCode: code, + Body: io.NopCloser(strings.NewReader("hi")), + } +} + +func TestRetryTransport(t *testing.T) { + for _, test := range []struct { + errs []error + resps []*http.Response + ctx context.Context + count int + }{{ + // Don't retry retry.Never. + errs: []error{temp{}}, + ctx: retry.Never(context.Background()), + count: 1, + }, { + // Don't retry permanent. + errs: []error{perm{}}, + count: 1, + }, { + // Do retry temp. + errs: []error{temp{}, perm{}}, + count: 2, + }, { + // Stop at some max. + errs: []error{temp{}, temp{}, temp{}, temp{}, temp{}}, + count: 3, + }, { + // Retry http errors. + errs: []error{nil, nil, temp{}, temp{}, temp{}}, + resps: []*http.Response{ + resp(http.StatusRequestTimeout), + resp(http.StatusInternalServerError), + nil, + }, + count: 3, + }} { + mt := mockTransport{ + errs: test.errs, + resps: test.resps, + } + + tr := NewRetry(&mt, + WithRetryBackoff(retry.Backoff{Steps: 3}), + WithRetryPredicate(retry.IsTemporary), + WithRetryStatusCodes(http.StatusRequestTimeout, http.StatusInternalServerError), + ) + + ctx := context.Background() + if test.ctx != nil { + ctx = test.ctx + } + req, err := http.NewRequestWithContext(ctx, "GET", "example.com", nil) + if err != nil { + t.Fatal(err) + } + tr.RoundTrip(req) + if mt.count != test.count { + t.Errorf("wrong count, wanted %d, got %d", test.count, mt.count) + } + } +} + +func TestRetryDefaults(t *testing.T) { + tr := NewRetry(http.DefaultTransport) + rt, ok := tr.(*retryTransport) + if !ok { + t.Fatal("could not cast to retryTransport") + } + + if rt.backoff != defaultBackoff { + t.Fatalf("default backoff wrong: %v", rt.backoff) + } + + if rt.predicate == nil { + t.Fatal("default predicate not set") + } +} + +func TestTimeoutContext(t *testing.T) { + tr := NewRetry(http.DefaultTransport) + + slowServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // hanging request + time.Sleep(time.Second * 1) + })) + defer func() { go func() { slowServer.Close() }() }() + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*20)) + defer cancel() + req, err := http.NewRequest("GET", slowServer.URL, nil) + if err != nil { + t.Fatal(err) + } + req = req.WithContext(ctx) + + result := make(chan error) + + go func() { + _, err := tr.RoundTrip(req) + result <- err + }() + + select { + case err := <-result: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("got: %v, want: %v", err, context.DeadlineExceeded) + } + case <-time.After(time.Millisecond * 100): + t.Fatalf("deadline was not recognized by transport") + } +} diff --git a/pkg/v1/remote/transport/schemer.go b/pkg/v1/remote/transport/schemer.go new file mode 100644 index 0000000..d70b6a8 --- /dev/null +++ b/pkg/v1/remote/transport/schemer.go @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + + "github.com/google/go-containerregistry/pkg/name" +) + +type schemeTransport struct { + // Scheme we should use, determined by ping response. + scheme string + + // Registry we're talking to. + registry name.Registry + + // Wrapped by schemeTransport. + inner http.RoundTripper +} + +// RoundTrip implements http.RoundTripper +func (st *schemeTransport) RoundTrip(in *http.Request) (*http.Response, error) { + // When we ping() the registry, we determine whether to use http or https + // based on which scheme was successful. That is only valid for the + // registry server and not e.g. a separate token server or blob storage, + // so we should only override the scheme if the host is the registry. + if matchesHost(st.registry, in, st.scheme) { + in.URL.Scheme = st.scheme + } + return st.inner.RoundTrip(in) +} diff --git a/pkg/v1/remote/transport/scope.go b/pkg/v1/remote/transport/scope.go new file mode 100644 index 0000000..c3b56f7 --- /dev/null +++ b/pkg/v1/remote/transport/scope.go @@ -0,0 +1,24 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +// Scopes suitable to qualify each Repository +const ( + PullScope string = "pull" + PushScope string = "push,pull" + // For now DELETE is PUSH, which is the read/write ACL. + DeleteScope string = PushScope + CatalogScope string = "catalog" +) diff --git a/pkg/v1/remote/transport/transport.go b/pkg/v1/remote/transport/transport.go new file mode 100644 index 0000000..01fe1fa --- /dev/null +++ b/pkg/v1/remote/transport/transport.go @@ -0,0 +1,116 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "fmt" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" +) + +// New returns a new RoundTripper based on the provided RoundTripper that has been +// setup to authenticate with the remote registry "reg", in the capacity +// laid out by the specified scopes. +// +// Deprecated: Use NewWithContext. +func New(reg name.Registry, auth authn.Authenticator, t http.RoundTripper, scopes []string) (http.RoundTripper, error) { + return NewWithContext(context.Background(), reg, auth, t, scopes) +} + +// NewWithContext returns a new RoundTripper based on the provided RoundTripper that has been +// set up to authenticate with the remote registry "reg", in the capacity +// laid out by the specified scopes. +// In case the RoundTripper is already of the type Wrapper it assumes +// authentication was already done prior to this call, so it just returns +// the provided RoundTripper without further action +func NewWithContext(ctx context.Context, reg name.Registry, auth authn.Authenticator, t http.RoundTripper, scopes []string) (http.RoundTripper, error) { + // When the transport provided is of the type Wrapper this function assumes that the caller already + // executed the necessary login and check. + switch t.(type) { + case *Wrapper: + return t, nil + } + // The handshake: + // 1. Use "t" to ping() the registry for the authentication challenge. + // + // 2a. If we get back a 200, then simply use "t". + // + // 2b. If we get back a 401 with a Basic challenge, then use a transport + // that just attachs auth each roundtrip. + // + // 2c. If we get back a 401 with a Bearer challenge, then use a transport + // that attaches a bearer token to each request, and refreshes is on 401s. + // Perform an initial refresh to seed the bearer token. + + // First we ping the registry to determine the parameters of the authentication handshake + // (if one is even necessary). + pr, err := ping(ctx, reg, t) + if err != nil { + return nil, err + } + + // Wrap t with a useragent transport unless we already have one. + if _, ok := t.(*userAgentTransport); !ok { + t = NewUserAgent(t, "") + } + + // Wrap t in a transport that selects the appropriate scheme based on the ping response. + t = &schemeTransport{ + scheme: pr.scheme, + registry: reg, + inner: t, + } + + switch pr.challenge.Canonical() { + case anonymous, basic: + return &Wrapper{&basicTransport{inner: t, auth: auth, target: reg.RegistryStr()}}, nil + case bearer: + // We require the realm, which tells us where to send our Basic auth to turn it into Bearer auth. + realm, ok := pr.parameters["realm"] + if !ok { + return nil, fmt.Errorf("malformed www-authenticate, missing realm: %v", pr.parameters) + } + service := pr.parameters["service"] + bt := &bearerTransport{ + inner: t, + basic: auth, + realm: realm, + registry: reg, + service: service, + scopes: scopes, + scheme: pr.scheme, + } + if err := bt.refresh(ctx); err != nil { + return nil, err + } + return &Wrapper{bt}, nil + default: + return nil, fmt.Errorf("unrecognized challenge: %s", pr.challenge) + } +} + +// Wrapper results in *not* wrapping supplied transport with additional logic such as retries, useragent and debug logging +// Consumers are opt-ing into providing their own transport without any additional wrapping. +type Wrapper struct { + inner http.RoundTripper +} + +// RoundTrip delegates to the inner RoundTripper +func (w *Wrapper) RoundTrip(in *http.Request) (*http.Response, error) { + return w.inner.RoundTrip(in) +} diff --git a/pkg/v1/remote/transport/transport_test.go b/pkg/v1/remote/transport/transport_test.go new file mode 100644 index 0000000..10389b7 --- /dev/null +++ b/pkg/v1/remote/transport/transport_test.go @@ -0,0 +1,282 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" +) + +var ( + testReference, _ = name.NewTag("localhost:8080/user/image:latest", name.StrictValidation) +) + +func TestTransportNoActionIfTransportIsAlreadyWrapper(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Bearer realm="http://foo.io"`) + http.Error(w, "Should not contact the server", http.StatusBadRequest) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + wTprt := &Wrapper{inner: tprt} + + if _, err := NewWithContext(context.Background(), testReference.Context().Registry, nil, wTprt, []string{testReference.Scope(PullScope)}); err != nil { + t.Errorf("NewWithContext unexpected error %s", err) + } +} + +func TestTransportSelectionAnonymous(t *testing.T) { + // Record the requests we get in the inner transport. + cannedResponse := http.Response{ + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("")), + } + recorder := newRecorder(&cannedResponse, nil) + + basic := &authn.Basic{Username: "foo", Password: "bar"} + reg := testReference.Context().Registry + + tp, err := NewWithContext(context.Background(), reg, basic, recorder, []string{testReference.Scope(PullScope)}) + if err != nil { + t.Errorf("NewWithContext() = %v", err) + } + + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/v2/anything", reg), nil) + if err != nil { + t.Fatalf("Unexpected error during NewRequest: %v", err) + } + if _, err := tp.RoundTrip(req); err != nil { + t.Fatalf("Unexpected error during RoundTrip: %v", err) + } + + if got, want := len(recorder.reqs), 2; got != want { + t.Fatalf("expected %d requests, got %d", want, got) + } + recorded := recorder.reqs[1] + if got, want := recorded.URL.Scheme, "https"; got != want { + t.Errorf("wrong scheme, want %s got %s", want, got) + } +} + +func TestTransportSelectionBasic(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Basic`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + + tp, err := NewWithContext(context.Background(), testReference.Context().Registry, basic, tprt, []string{testReference.Scope(PullScope)}) + if err != nil { + t.Errorf("NewWithContext() = %v", err) + } + if tpw, ok := tp.(*Wrapper); !ok { + t.Errorf("NewWithContext(); got %T, want *Wrapper", tp) + } else if _, ok := tpw.inner.(*basicTransport); !ok { + t.Errorf("NewWithContext(); got %T, want *basicTransport", tp) + } +} + +type badAuth struct{} + +func (a *badAuth) Authorization() (*authn.AuthConfig, error) { + return nil, errors.New("sorry dave, I'm afraid I can't let you do that") +} + +func TestTransportBadAuth(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Bearer realm="http://foo.io"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + if _, err := NewWithContext(context.Background(), testReference.Context().Registry, &badAuth{}, tprt, []string{testReference.Scope(PullScope)}); err == nil { + t.Errorf("NewWithContext() expected err, got nil") + } +} + +func TestTransportSelectionBearer(t *testing.T) { + request := 0 + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + request++ + switch request { + case 1: + // This is an https request that fails, causing us to fall back to http. + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + case 2: + w.Header().Set("WWW-Authenticate", `Bearer realm="http://foo.io"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + case 3: + hdr := r.Header.Get("Authorization") + if !strings.HasPrefix(hdr, "Basic ") { + t.Errorf("Header.Get(Authorization); got %v, want Basic prefix", hdr) + } + if got, want := r.FormValue("scope"), testReference.Scope(PullScope); got != want { + t.Errorf("FormValue(scope); got %v, want %v", got, want) + } + // Check that the service isn't set (we didn't specify it above) + // https://github.com/google/go-containerregistry/issues/1359 + if got, want := r.FormValue("service"), ""; got != want { + t.Errorf("FormValue(service); got %q, want %q", got, want) + } + w.Write([]byte(`{"token": "dfskdjhfkhsjdhfkjhsdf"}`)) + } + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + tp, err := NewWithContext(context.Background(), testReference.Context().Registry, basic, tprt, []string{testReference.Scope(PullScope)}) + if err != nil { + t.Errorf("NewWithContext() = %v", err) + } + if tpw, ok := tp.(*Wrapper); !ok { + t.Errorf("NewWithContext(); got %T, want *Wrapper", tp) + } else if _, ok := tpw.inner.(*bearerTransport); !ok { + t.Errorf("NewWithContext(); got %T, want *bearerTransport", tp) + } +} + +func TestTransportSelectionBearerMissingRealm(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Bearer service="gcr.io"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + tp, err := NewWithContext(context.Background(), testReference.Context().Registry, basic, tprt, []string{testReference.Scope(PullScope)}) + if err == nil || !strings.Contains(err.Error(), "missing realm") { + t.Errorf("NewWithContext() = %v, %v", tp, err) + } +} + +func TestTransportSelectionBearerAuthError(t *testing.T) { + request := 0 + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + request++ + switch request { + case 1: + w.Header().Set("WWW-Authenticate", `Bearer realm="http://foo.io"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + case 2: + http.Error(w, "Oops", http.StatusInternalServerError) + } + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + tp, err := NewWithContext(context.Background(), testReference.Context().Registry, basic, tprt, []string{testReference.Scope(PullScope)}) + if err == nil { + t.Errorf("NewWithContext() = %v", tp) + } +} + +func TestTransportSelectionUnrecognizedChallenge(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Unrecognized`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + tprt := &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + tp, err := NewWithContext(context.Background(), testReference.Context().Registry, basic, tprt, []string{testReference.Scope(PullScope)}) + if err == nil || !strings.Contains(err.Error(), "challenge") { + t.Errorf("NewWithContext() = %v, %v", tp, err) + } +} + +func TestTransportAlwaysTriesHttps(t *testing.T) { + // Use a NewTLSServer so that this speaks TLS even though it's localhost. + // This ensures that we try https even for local registries. + count := 0 + server := httptest.NewTLSServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count++ + w.Write([]byte(`{"token": "dfskdjhfkhsjdhfkjhsdf"}`)) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + if err != nil { + t.Errorf("Unexpected error during url.Parse: %v", err) + } + registry, err := name.NewRegistry(u.Host, name.WeakValidation) + if err != nil { + t.Errorf("Unexpected error during NewRegistry: %v", err) + } + + basic := &authn.Basic{Username: "foo", Password: "bar"} + tp, err := NewWithContext(context.Background(), registry, basic, server.Client().Transport, []string{testReference.Scope(PullScope)}) + if err != nil { + t.Fatalf("NewWithContext() = %v, %v", tp, err) + } + if count == 0 { + t.Errorf("failed to call TLS localhost server") + } +} diff --git a/pkg/v1/remote/transport/useragent.go b/pkg/v1/remote/transport/useragent.go new file mode 100644 index 0000000..74a9e71 --- /dev/null +++ b/pkg/v1/remote/transport/useragent.go @@ -0,0 +1,94 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "fmt" + "net/http" + "runtime/debug" +) + +var ( + // Version can be set via: + // -ldflags="-X 'github.com/google/go-containerregistry/pkg/v1/remote/transport.Version=$TAG'" + Version string + + ggcrVersion = defaultUserAgent +) + +const ( + defaultUserAgent = "go-containerregistry" + moduleName = "github.com/google/go-containerregistry" +) + +type userAgentTransport struct { + inner http.RoundTripper + ua string +} + +func init() { + if v := version(); v != "" { + ggcrVersion = fmt.Sprintf("%s/%s", defaultUserAgent, v) + } +} + +func version() string { + if Version != "" { + // Version was set via ldflags, just return it. + return Version + } + + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + + // Happens for crane and gcrane. + if info.Main.Path == moduleName { + return info.Main.Version + } + + // Anything else. + for _, dep := range info.Deps { + if dep.Path == moduleName { + return dep.Version + } + } + + return "" +} + +// NewUserAgent returns an http.Roundtripper that sets the user agent to +// The provided string plus additional go-containerregistry information, +// e.g. if provided "crane/v0.1.4" and this modules was built at v0.1.4: +// +// User-Agent: crane/v0.1.4 go-containerregistry/v0.1.4 +func NewUserAgent(inner http.RoundTripper, ua string) http.RoundTripper { + if ua == "" { + ua = ggcrVersion + } else { + ua = fmt.Sprintf("%s %s", ua, ggcrVersion) + } + return &userAgentTransport{ + inner: inner, + ua: ua, + } +} + +// RoundTrip implements http.RoundTripper +func (ut *userAgentTransport) RoundTrip(in *http.Request) (*http.Response, error) { + in.Header.Set("User-Agent", ut.ua) + return ut.inner.RoundTrip(in) +} diff --git a/pkg/v1/remote/write.go b/pkg/v1/remote/write.go new file mode 100644 index 0000000..5dbaa7c --- /dev/null +++ b/pkg/v1/remote/write.go @@ -0,0 +1,1003 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + + "github.com/google/go-containerregistry/internal/redact" + "github.com/google/go-containerregistry/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +// Taggable is an interface that enables a manifest PUT (e.g. for tagging). +type Taggable interface { + RawManifest() ([]byte, error) +} + +// Write pushes the provided img to the specified image reference. +func Write(ref name.Reference, img v1.Image, options ...Option) (rerr error) { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + + var p *progress + if o.updates != nil { + p = &progress{updates: o.updates} + p.lastUpdate = &v1.Update{} + p.lastUpdate.Total, err = countImage(img, o.allowNondistributableArtifacts) + if err != nil { + return err + } + defer close(o.updates) + defer func() { _ = p.err(rerr) }() + } + return writeImage(o.context, ref, img, o, p) +} + +func writeImage(ctx context.Context, ref name.Reference, img v1.Image, o *options, progress *progress) error { + ls, err := img.Layers() + if err != nil { + return err + } + scopes := scopesForUploadingImage(ref.Context(), ls) + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + w := writer{ + repo: ref.Context(), + client: &http.Client{Transport: tr}, + progress: progress, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + } + + // Upload individual blobs and collect any errors. + blobChan := make(chan v1.Layer, 2*o.jobs) + g, gctx := errgroup.WithContext(ctx) + for i := 0; i < o.jobs; i++ { + // Start N workers consuming blobs to upload. + g.Go(func() error { + for b := range blobChan { + if err := w.uploadOne(gctx, b); err != nil { + return err + } + } + return nil + }) + } + + // Upload individual layers in goroutines and collect any errors. + // If we can dedupe by the layer digest, try to do so. If we can't determine + // the digest for whatever reason, we can't dedupe and might re-upload. + g.Go(func() error { + defer close(blobChan) + uploaded := map[v1.Hash]bool{} + for _, l := range ls { + l := l + + // Handle foreign layers. + mt, err := l.MediaType() + if err != nil { + return err + } + if !mt.IsDistributable() && !o.allowNondistributableArtifacts { + continue + } + + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { + // If we can determine the layer's digest ahead of + // time, use it to dedupe uploads. + if uploaded[h] { + continue // Already uploading. + } + uploaded[h] = true + } + select { + case blobChan <- l: + case <-gctx.Done(): + return gctx.Err() + } + } + return nil + }) + + if l, err := partial.ConfigLayer(img); err != nil { + // We can't read the ConfigLayer, possibly because of streaming layers, + // since the layer DiffIDs haven't been calculated yet. Attempt to wait + // for the other layers to be uploaded, then try the config again. + if err := g.Wait(); err != nil { + return err + } + + // Now that all the layers are uploaded, try to upload the config file blob. + l, err := partial.ConfigLayer(img) + if err != nil { + return err + } + if err := w.uploadOne(ctx, l); err != nil { + return err + } + } else { + // We *can* read the ConfigLayer, so upload it concurrently with the layers. + g.Go(func() error { + return w.uploadOne(gctx, l) + }) + + // Wait for the layers + config. + if err := g.Wait(); err != nil { + return err + } + } + + // With all of the constituent elements uploaded, upload the manifest + // to commit the image. + return w.commitManifest(ctx, img, ref) +} + +// writer writes the elements of an image to a remote image reference. +type writer struct { + repo name.Repository + client *http.Client + + progress *progress + backoff Backoff + predicate retry.Predicate +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (w *writer) url(path string) url.URL { + return url.URL{ + Scheme: w.repo.Registry.Scheme(), + Host: w.repo.RegistryStr(), + Path: path, + } +} + +// nextLocation extracts the fully-qualified URL to which we should send the next request in an upload sequence. +func (w *writer) nextLocation(resp *http.Response) (string, error) { + loc := resp.Header.Get("Location") + if len(loc) == 0 { + return "", errors.New("missing Location header") + } + u, err := url.Parse(loc) + if err != nil { + return "", err + } + + // If the location header returned is just a url path, then fully qualify it. + // We cannot simply call w.url, since there might be an embedded query string. + return resp.Request.URL.ResolveReference(u).String(), nil +} + +// checkExistingBlob checks if a blob exists already in the repository by making a +// HEAD request to the blob store API. GCR performs an existence check on the +// initiation if "mount" is specified, even if no "from" sources are specified. +// However, this is not broadly applicable to all registries, e.g. ECR. +func (w *writer) checkExistingBlob(ctx context.Context, h v1.Hash) (bool, error) { + u := w.url(fmt.Sprintf("/v2/%s/blobs/%s", w.repo.RepositoryStr(), h.String())) + + req, err := http.NewRequest(http.MethodHead, u.String(), nil) + if err != nil { + return false, err + } + + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil { + return false, err + } + + return resp.StatusCode == http.StatusOK, nil +} + +// checkExistingManifest checks if a manifest exists already in the repository +// by making a HEAD request to the manifest API. +func (w *writer) checkExistingManifest(ctx context.Context, h v1.Hash, mt types.MediaType) (bool, error) { + u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.repo.RepositoryStr(), h.String())) + + req, err := http.NewRequest(http.MethodHead, u.String(), nil) + if err != nil { + return false, err + } + req.Header.Set("Accept", string(mt)) + + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil { + return false, err + } + + return resp.StatusCode == http.StatusOK, nil +} + +// initiateUpload initiates the blob upload, which starts with a POST that can +// optionally include the hash of the layer and a list of repositories from +// which that layer might be read. On failure, an error is returned. +// On success, the layer was either mounted (nothing more to do) or a blob +// upload was initiated and the body of that blob should be sent to the returned +// location. +func (w *writer) initiateUpload(ctx context.Context, from, mount, origin string) (location string, mounted bool, err error) { + u := w.url(fmt.Sprintf("/v2/%s/blobs/uploads/", w.repo.RepositoryStr())) + uv := url.Values{} + if mount != "" && from != "" { + // Quay will fail if we specify a "mount" without a "from". + uv.Set("mount", mount) + uv.Set("from", from) + if origin != "" { + uv.Set("origin", origin) + } + } + u.RawQuery = uv.Encode() + + // Make the request to initiate the blob upload. + req, err := http.NewRequest(http.MethodPost, u.String(), nil) + if err != nil { + return "", false, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return "", false, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusCreated, http.StatusAccepted); err != nil { + if origin != "" && origin != w.repo.RegistryStr() { + // https://github.com/google/go-containerregistry/issues/1404 + logs.Warn.Printf("retrying without mount: %v", err) + return w.initiateUpload(ctx, "", "", "") + } + return "", false, err + } + + // Check the response code to determine the result. + switch resp.StatusCode { + case http.StatusCreated: + // We're done, we were able to fast-path. + return "", true, nil + case http.StatusAccepted: + // Proceed to PATCH, upload has begun. + loc, err := w.nextLocation(resp) + return loc, false, err + default: + panic("Unreachable: initiateUpload") + } +} + +// streamBlob streams the contents of the blob to the specified location. +// On failure, this will return an error. On success, this will return the location +// header indicating how to commit the streamed blob. +func (w *writer) streamBlob(ctx context.Context, layer v1.Layer, streamLocation string) (commitLocation string, rerr error) { + reset := func() {} + defer func() { + if rerr != nil { + reset() + } + }() + blob, err := layer.Compressed() + if err != nil { + return "", err + } + + getBody := layer.Compressed + if w.progress != nil { + var count int64 + blob = &progressReader{rc: blob, progress: w.progress, count: &count} + getBody = func() (io.ReadCloser, error) { + blob, err := layer.Compressed() + if err != nil { + return nil, err + } + return &progressReader{rc: blob, progress: w.progress, count: &count}, nil + } + reset = func() { + w.progress.complete(-count) + } + } + + req, err := http.NewRequest(http.MethodPatch, streamLocation, blob) + if err != nil { + return "", err + } + if _, ok := layer.(*stream.Layer); !ok { + // We can't retry streaming layers. + req.GetBody = getBody + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusNoContent, http.StatusAccepted, http.StatusCreated); err != nil { + return "", err + } + + // The blob has been uploaded, return the location header indicating + // how to commit this layer. + return w.nextLocation(resp) +} + +// commitBlob commits this blob by sending a PUT to the location returned from +// streaming the blob. +func (w *writer) commitBlob(ctx context.Context, location, digest string) error { + u, err := url.Parse(location) + if err != nil { + return err + } + v := u.Query() + v.Set("digest", digest) + u.RawQuery = v.Encode() + + req, err := http.NewRequest(http.MethodPut, u.String(), nil) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + return transport.CheckError(resp, http.StatusCreated) +} + +// incrProgress increments and sends a progress update, if WithProgress is used. +func (w *writer) incrProgress(written int64) { + if w.progress == nil { + return + } + w.progress.complete(written) +} + +// uploadOne performs a complete upload of a single layer. +func (w *writer) uploadOne(ctx context.Context, l v1.Layer) error { + tryUpload := func() error { + ctx := retry.Never(ctx) + var from, mount, origin string + if h, err := l.Digest(); err == nil { + // If we know the digest, this isn't a streaming layer. Do an existence + // check so we can skip uploading the layer if possible. + existing, err := w.checkExistingBlob(ctx, h) + if err != nil { + return err + } + if existing { + size, err := l.Size() + if err != nil { + return err + } + w.incrProgress(size) + logs.Progress.Printf("existing blob: %v", h) + return nil + } + + mount = h.String() + } + if ml, ok := l.(*MountableLayer); ok { + from = ml.Reference.Context().RepositoryStr() + origin = ml.Reference.Context().RegistryStr() + } + + location, mounted, err := w.initiateUpload(ctx, from, mount, origin) + if err != nil { + return err + } else if mounted { + size, err := l.Size() + if err != nil { + return err + } + w.incrProgress(size) + h, err := l.Digest() + if err != nil { + return err + } + logs.Progress.Printf("mounted blob: %s", h.String()) + return nil + } + + // Only log layers with +json or +yaml. We can let through other stuff if it becomes popular. + // TODO(opencontainers/image-spec#791): Would be great to have an actual parser. + mt, err := l.MediaType() + if err != nil { + return err + } + smt := string(mt) + if !(strings.HasSuffix(smt, "+json") || strings.HasSuffix(smt, "+yaml")) { + ctx = redact.NewContext(ctx, "omitting binary blobs from logs") + } + + location, err = w.streamBlob(ctx, l, location) + if err != nil { + return err + } + + h, err := l.Digest() + if err != nil { + return err + } + digest := h.String() + + if err := w.commitBlob(ctx, location, digest); err != nil { + return err + } + logs.Progress.Printf("pushed blob: %s", digest) + return nil + } + + return retry.Retry(tryUpload, w.predicate, w.backoff) +} + +type withLayer interface { + Layer(v1.Hash) (v1.Layer, error) +} + +func (w *writer) writeIndex(ctx context.Context, ref name.Reference, ii v1.ImageIndex, options ...Option) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + + // TODO(#803): Pipe through remote.WithJobs and upload these in parallel. + for _, desc := range index.Manifests { + ref := ref.Context().Digest(desc.Digest.String()) + exists, err := w.checkExistingManifest(ctx, desc.Digest, desc.MediaType) + if err != nil { + return err + } + if exists { + logs.Progress.Print("existing manifest: ", desc.Digest) + continue + } + + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := w.writeIndex(ctx, ref, ii, options...); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := writeImage(ctx, ref, img, o, w.progress); err != nil { + return err + } + default: + // Workaround for #819. + if wl, ok := ii.(withLayer); ok { + layer, err := wl.Layer(desc.Digest) + if err != nil { + return err + } + if err := w.uploadOne(ctx, layer); err != nil { + return err + } + } + } + } + + // With all of the constituent elements uploaded, upload the manifest + // to commit the image. + return w.commitManifest(ctx, ii, ref) +} + +type withMediaType interface { + MediaType() (types.MediaType, error) +} + +// This is really silly, but go interfaces don't let me satisfy remote.Taggable +// with remote.Descriptor because of name collisions between method names and +// struct fields. +// +// Use reflection to either pull the v1.Descriptor out of remote.Descriptor or +// create a descriptor based on the RawManifest and (optionally) MediaType. +func unpackTaggable(t Taggable) ([]byte, *v1.Descriptor, error) { + if d, ok := t.(*Descriptor); ok { + return d.Manifest, &d.Descriptor, nil + } + b, err := t.RawManifest() + if err != nil { + return nil, nil, err + } + + // A reasonable default if Taggable doesn't implement MediaType. + mt := types.DockerManifestSchema2 + + if wmt, ok := t.(withMediaType); ok { + m, err := wmt.MediaType() + if err != nil { + return nil, nil, err + } + mt = m + } + + h, sz, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, nil, err + } + + return b, &v1.Descriptor{ + MediaType: mt, + Size: sz, + Digest: h, + }, nil +} + +// commitSubjectReferrers is responsible for updating the fallback tag manifest to track descriptors referring to a subject for registries that don't yet support the Referrers API. +// TODO: use conditional requests to avoid race conditions +func (w *writer) commitSubjectReferrers(ctx context.Context, sub name.Digest, add v1.Descriptor) error { + // Check if the registry supports Referrers API. + // TODO: This should be done once per registry, not once per subject. + u := w.url(fmt.Sprintf("/v2/%s/referrers/%s", w.repo.RepositoryStr(), sub.DigestStr())) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return err + } + req.Header.Set("Accept", string(types.OCIImageIndex)) + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound, http.StatusBadRequest); err != nil { + return err + } + if resp.StatusCode == http.StatusOK { + // The registry supports Referrers API. The registry is responsible for updating the referrers list. + return nil + } + + // The registry doesn't support Referrers API, we need to update the manifest tagged with the fallback tag. + // Make the request to GET the current manifest. + t := fallbackTag(sub) + u = w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.repo.RepositoryStr(), t.Identifier())) + req, err = http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return err + } + req.Header.Set("Accept", string(types.OCIImageIndex)) + resp, err = w.client.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + var im v1.IndexManifest + if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil { + return err + } else if resp.StatusCode == http.StatusNotFound { + // Not found just means there are no attachments. Start with an empty index. + im = v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + Manifests: []v1.Descriptor{add}, + } + } else { + if err := json.NewDecoder(resp.Body).Decode(&im); err != nil { + return err + } + if im.SchemaVersion != 2 { + return fmt.Errorf("fallback tag manifest is not a schema version 2: %d", im.SchemaVersion) + } + if im.MediaType != types.OCIImageIndex { + return fmt.Errorf("fallback tag manifest is not an OCI image index: %s", im.MediaType) + } + for _, desc := range im.Manifests { + if desc.Digest == add.Digest { + // The digest is already attached, nothing to do. + logs.Progress.Printf("fallback tag %s already had referrer", t.Identifier()) + return nil + } + } + // Append the new descriptor to the index. + im.Manifests = append(im.Manifests, add) + } + + // Sort the manifests for reproducibility. + sort.Slice(im.Manifests, func(i, j int) bool { + return im.Manifests[i].Digest.String() < im.Manifests[j].Digest.String() + }) + logs.Progress.Printf("updating fallback tag %s with new referrer", t.Identifier()) + if err := w.commitManifest(ctx, fallbackTaggable{im}, t); err != nil { + return err + } + return nil +} + +type fallbackTaggable struct { + im v1.IndexManifest +} + +func (f fallbackTaggable) RawManifest() ([]byte, error) { return json.Marshal(f.im) } +func (f fallbackTaggable) MediaType() (types.MediaType, error) { return types.OCIImageIndex, nil } + +// commitManifest does a PUT of the image's manifest. +func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Reference) error { + // If the manifest refers to a subject, we need to check whether we need to update the fallback tag manifest. + raw, err := t.RawManifest() + if err != nil { + return err + } + var mf struct { + MediaType types.MediaType `json:"mediaType"` + Subject *v1.Descriptor `json:"subject,omitempty"` + Config struct { + MediaType types.MediaType `json:"mediaType"` + } `json:"config"` + } + if err := json.Unmarshal(raw, &mf); err != nil { + return err + } + + tryUpload := func() error { + ctx := retry.Never(ctx) + raw, desc, err := unpackTaggable(t) + if err != nil { + return err + } + + u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.repo.RepositoryStr(), ref.Identifier())) + + // Make the request to PUT the serialized manifest + req, err := http.NewRequest(http.MethodPut, u.String(), bytes.NewBuffer(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", string(desc.MediaType)) + + resp, err := w.client.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted); err != nil { + return err + } + + // If the manifest referred to a subject, we may need to update the fallback tag manifest. + // TODO: If this fails, we'll retry the whole upload. We should retry just this part. + if mf.Subject != nil { + h, size, err := v1.SHA256(bytes.NewReader(raw)) + if err != nil { + return err + } + desc := v1.Descriptor{ + ArtifactType: string(mf.Config.MediaType), + MediaType: mf.MediaType, + Digest: h, + Size: size, + } + if err := w.commitSubjectReferrers(ctx, + ref.Context().Digest(mf.Subject.Digest.String()), + desc); err != nil { + return err + } + } + + // The image was successfully pushed! + logs.Progress.Printf("%v: digest: %v size: %d", ref, desc.Digest, desc.Size) + w.incrProgress(int64(len(raw))) + return nil + } + + return retry.Retry(tryUpload, w.predicate, w.backoff) +} + +func scopesForUploadingImage(repo name.Repository, layers []v1.Layer) []string { + // use a map as set to remove duplicates scope strings + scopeSet := map[string]struct{}{} + + for _, l := range layers { + if ml, ok := l.(*MountableLayer); ok { + // we will add push scope for ref.Context() after the loop. + // for now we ask pull scope for references of the same registry + if ml.Reference.Context().String() != repo.String() && ml.Reference.Context().Registry.String() == repo.Registry.String() { + scopeSet[ml.Reference.Scope(transport.PullScope)] = struct{}{} + } + } + } + + scopes := make([]string, 0) + // Push scope should be the first element because a few registries just look at the first scope to determine access. + scopes = append(scopes, repo.Scope(transport.PushScope)) + + for scope := range scopeSet { + scopes = append(scopes, scope) + } + + return scopes +} + +// WriteIndex pushes the provided ImageIndex to the specified image reference. +// WriteIndex will attempt to push all of the referenced manifests before +// attempting to push the ImageIndex, to retain referential integrity. +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) (rerr error) { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + + scopes := []string{ref.Scope(transport.PushScope)} + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + w := writer{ + repo: ref.Context(), + client: &http.Client{Transport: tr}, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + } + + if o.updates != nil { + w.progress = &progress{updates: o.updates} + w.progress.lastUpdate = &v1.Update{} + + defer close(o.updates) + defer func() { w.progress.err(rerr) }() + + w.progress.lastUpdate.Total, err = countIndex(ii, o.allowNondistributableArtifacts) + if err != nil { + return err + } + } + + return w.writeIndex(o.context, ref, ii, options...) +} + +// countImage counts the total size of all layers + config blob + manifest for +// an image. It de-dupes duplicate layers. +func countImage(img v1.Image, allowNondistributableArtifacts bool) (int64, error) { + var total int64 + ls, err := img.Layers() + if err != nil { + return 0, err + } + seen := map[v1.Hash]bool{} + for _, l := range ls { + // Handle foreign layers. + mt, err := l.MediaType() + if err != nil { + return 0, err + } + if !mt.IsDistributable() && !allowNondistributableArtifacts { + continue + } + + // TODO: support streaming layers which update the total count as they write. + if _, ok := l.(*stream.Layer); ok { + return 0, errors.New("cannot use stream.Layer and WithProgress") + } + + // Dedupe layers. + d, err := l.Digest() + if err != nil { + return 0, err + } + if seen[d] { + continue + } + seen[d] = true + + size, err := l.Size() + if err != nil { + return 0, err + } + total += size + } + b, err := img.RawConfigFile() + if err != nil { + return 0, err + } + total += int64(len(b)) + size, err := img.Size() + if err != nil { + return 0, err + } + total += size + return total, nil +} + +// countIndex counts the total size of all images + sub-indexes for an index. +// It does not attempt to de-dupe duplicate images, etc. +func countIndex(idx v1.ImageIndex, allowNondistributableArtifacts bool) (int64, error) { + var total int64 + mf, err := idx.IndexManifest() + if err != nil { + return 0, err + } + + for _, desc := range mf.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + sidx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return 0, err + } + size, err := countIndex(sidx, allowNondistributableArtifacts) + if err != nil { + return 0, err + } + total += size + case types.OCIManifestSchema1, types.DockerManifestSchema2: + simg, err := idx.Image(desc.Digest) + if err != nil { + return 0, err + } + size, err := countImage(simg, allowNondistributableArtifacts) + if err != nil { + return 0, err + } + total += size + default: + // Workaround for #819. + if wl, ok := idx.(withLayer); ok { + layer, err := wl.Layer(desc.Digest) + if err != nil { + return 0, err + } + size, err := layer.Size() + if err != nil { + return 0, err + } + total += size + } + } + } + + size, err := idx.Size() + if err != nil { + return 0, err + } + total += size + return total, nil +} + +// WriteLayer uploads the provided Layer to the specified repo. +func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) (rerr error) { + o, err := makeOptions(repo, options...) + if err != nil { + return err + } + scopes := scopesForUploadingImage(repo, []v1.Layer{layer}) + tr, err := transport.NewWithContext(o.context, repo.Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + w := writer{ + repo: repo, + client: &http.Client{Transport: tr}, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + } + + if o.updates != nil { + w.progress = &progress{updates: o.updates} + w.progress.lastUpdate = &v1.Update{} + + defer close(o.updates) + defer func() { w.progress.err(rerr) }() + + // TODO: support streaming layers which update the total count as they write. + if _, ok := layer.(*stream.Layer); ok { + return errors.New("cannot use stream.Layer and WithProgress") + } + size, err := layer.Size() + if err != nil { + return err + } + w.progress.total(size) + } + return w.uploadOne(o.context, layer) +} + +// Tag adds a tag to the given Taggable via PUT /v2/.../manifests/ +// +// Notable implementations of Taggable are v1.Image, v1.ImageIndex, and +// remote.Descriptor. +// +// If t implements MediaType, we will use that for the Content-Type, otherwise +// we will default to types.DockerManifestSchema2. +// +// Tag does not attempt to write anything other than the manifest, so callers +// should ensure that all blobs or manifests that are referenced by t exist +// in the target registry. +func Tag(tag name.Tag, t Taggable, options ...Option) error { + return Put(tag, t, options...) +} + +// Put adds a manifest from the given Taggable via PUT /v1/.../manifest/ +// +// Notable implementations of Taggable are v1.Image, v1.ImageIndex, and +// remote.Descriptor. +// +// If t implements MediaType, we will use that for the Content-Type, otherwise +// we will default to types.DockerManifestSchema2. +// +// Put does not attempt to write anything other than the manifest, so callers +// should ensure that all blobs or manifests that are referenced by t exist +// in the target registry. +func Put(ref name.Reference, t Taggable, options ...Option) error { + o, err := makeOptions(ref.Context(), options...) + if err != nil { + return err + } + scopes := []string{ref.Scope(transport.PushScope)} + + // TODO: This *always* does a token exchange. For some registries, + // that's pretty slow. Some ideas; + // * Tag could take a list of tags. + // * Allow callers to pass in a transport.Transport, typecheck + // it to allow them to reuse the transport across multiple calls. + // * WithTag option to do multiple manifest PUTs in commitManifest. + tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes) + if err != nil { + return err + } + w := writer{ + repo: ref.Context(), + client: &http.Client{Transport: tr}, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + } + + return w.commitManifest(o.context, t, ref) +} diff --git a/pkg/v1/remote/write_test.go b/pkg/v1/remote/write_test.go new file mode 100644 index 0000000..7235c96 --- /dev/null +++ b/pkg/v1/remote/write_test.go @@ -0,0 +1,1643 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "context" + "crypto" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func mustNewTag(t *testing.T, s string) name.Tag { + tag, err := name.NewTag(s, name.WeakValidation) + if err != nil { + t.Fatalf("NewTag(%v) = %v", s, err) + } + return tag +} + +func TestUrl(t *testing.T) { + tests := []struct { + tag string + path string + url string + }{{ + tag: "gcr.io/foo/bar:latest", + path: "/v2/foo/bar/manifests/latest", + url: "https://gcr.io/v2/foo/bar/manifests/latest", + }, { + tag: "localhost:8080/foo/bar:baz", + path: "/v2/foo/bar/blobs/upload", + url: "http://localhost:8080/v2/foo/bar/blobs/upload", + }} + + for _, test := range tests { + w := &writer{ + repo: mustNewTag(t, test.tag).Context(), + } + if got, want := w.url(test.path), test.url; got.String() != want { + t.Errorf("url(%v) = %v, want %v", test.path, got.String(), want) + } + } +} + +func TestNextLocation(t *testing.T) { + tests := []struct { + location string + url string + }{{ + location: "https://gcr.io/v2/foo/bar/blobs/uploads/1234567?baz=blah", + url: "https://gcr.io/v2/foo/bar/blobs/uploads/1234567?baz=blah", + }, { + location: "/v2/foo/bar/blobs/uploads/1234567?baz=blah", + url: "https://gcr.io/v2/foo/bar/blobs/uploads/1234567?baz=blah", + }} + + ref := mustNewTag(t, "gcr.io/foo/bar:latest") + w := &writer{ + repo: ref.Context(), + } + + for _, test := range tests { + resp := &http.Response{ + Header: map[string][]string{ + "Location": {test.location}, + }, + Request: &http.Request{ + URL: &url.URL{ + Scheme: ref.Registry.Scheme(), + Host: ref.RegistryStr(), + }, + }, + } + + got, err := w.nextLocation(resp) + if err != nil { + t.Errorf("nextLocation(%v) = %v", resp, err) + } + want := test.url + if got != want { + t.Errorf("nextLocation(%v) = %v, want %v", resp, got, want) + } + } +} + +type closer interface { + Close() +} + +func setupImage(t *testing.T) v1.Image { + rnd, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + return rnd +} + +func setupIndex(t *testing.T, children int64) v1.ImageIndex { + rnd, err := random.Index(1024, 1, children) + if err != nil { + t.Fatalf("random.Index() = %v", err) + } + return rnd +} + +func mustConfigName(t *testing.T, img v1.Image) v1.Hash { + h, err := img.ConfigName() + if err != nil { + t.Fatalf("ConfigName() = %v", err) + } + return h +} + +func setupWriter(repo string, handler http.HandlerFunc) (*writer, closer, error) { + server := httptest.NewServer(handler) + return setupWriterWithServer(server, repo) +} + +func setupWriterWithServer(server *httptest.Server, repo string) (*writer, closer, error) { + u, err := url.Parse(server.URL) + if err != nil { + server.Close() + return nil, nil, err + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, repo), name.WeakValidation) + if err != nil { + server.Close() + return nil, nil, err + } + + return &writer{ + repo: tag.Context(), + client: http.DefaultClient, + predicate: defaultRetryPredicate, + backoff: defaultRetryBackoff, + }, server, nil +} + +func TestCheckExistingBlob(t *testing.T) { + tests := []struct { + name string + status int + existing bool + wantErr bool + }{{ + name: "success", + status: http.StatusOK, + existing: true, + }, { + name: "not found", + status: http.StatusNotFound, + existing: false, + }, { + name: "error", + status: http.StatusInternalServerError, + existing: false, + wantErr: true, + }} + + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "foo/bar" + expectedPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, h.String()) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodHead) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + http.Error(w, http.StatusText(test.status), test.status) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + existing, err := w.checkExistingBlob(context.Background(), h) + if test.existing != existing { + t.Errorf("checkExistingBlob() = %v, want %v", existing, test.existing) + } + if err != nil && !test.wantErr { + t.Errorf("checkExistingBlob() = %v", err) + } else if err == nil && test.wantErr { + t.Error("checkExistingBlob() wanted err, got nil") + } + }) + } +} + +func TestInitiateUploadNoMountsExists(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "foo/bar" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{"baz/bar"}, + }.Encode() + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Mounted", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + _, mounted, err := w.initiateUpload(context.Background(), "baz/bar", h.String(), "") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if !mounted { + t.Error("initiateUpload() = !mounted, want mounted") + } +} + +func TestInitiateUploadNoMountsInitiated(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "baz/blah" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{"baz/bar"}, + }.Encode() + expectedLocation := "https://somewhere.io/upload?foo=bar" + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + w.Header().Set("Location", expectedLocation) + http.Error(w, "Initiated", http.StatusAccepted) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + location, mounted, err := w.initiateUpload(context.Background(), "baz/bar", h.String(), "") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if mounted { + t.Error("initiateUpload() = mounted, want !mounted") + } + if location != expectedLocation { + t.Errorf("initiateUpload(); got %v, want %v", location, expectedLocation) + } +} + +func TestInitiateUploadNoMountsBadStatus(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "ugh/another" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{"baz/bar"}, + }.Encode() + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Unknown", http.StatusNoContent) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + location, mounted, err := w.initiateUpload(context.Background(), "baz/bar", h.String(), "") + if err == nil { + t.Errorf("intiateUpload() = %v, %v; wanted error", location, mounted) + } +} + +func TestInitiateUploadMountsWithMountFromDifferentRegistry(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "yet/again" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{"baz/bar"}, + }.Encode() + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Mounted", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + _, mounted, err := w.initiateUpload(context.Background(), "baz/bar", h.String(), "") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if !mounted { + t.Error("initiateUpload() = !mounted, want mounted") + } +} + +func TestInitiateUploadMountsWithMountFromTheSameRegistry(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedMountRepo := "a/different/repo" + expectedRepo := "yet/again" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{expectedMountRepo}, + }.Encode() + + serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Mounted", http.StatusCreated) + }) + server := httptest.NewServer(serverHandler) + + w, closer, err := setupWriterWithServer(server, expectedRepo) + if err != nil { + t.Fatalf("setupWriterWithServer() = %v", err) + } + defer closer.Close() + + _, mounted, err := w.initiateUpload(context.Background(), expectedMountRepo, h.String(), "") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if !mounted { + t.Error("initiateUpload() = !mounted, want mounted") + } +} + +func TestInitiateUploadMountsWithOrigin(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedMountRepo := "a/different/repo" + expectedRepo := "yet/again" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedOrigin := "fakeOrigin" + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{expectedMountRepo}, + "origin": []string{expectedOrigin}, + }.Encode() + + serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Mounted", http.StatusCreated) + }) + server := httptest.NewServer(serverHandler) + + w, closer, err := setupWriterWithServer(server, expectedRepo) + if err != nil { + t.Fatalf("setupWriterWithServer() = %v", err) + } + defer closer.Close() + + _, mounted, err := w.initiateUpload(context.Background(), expectedMountRepo, h.String(), "fakeOrigin") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if !mounted { + t.Error("initiateUpload() = !mounted, want mounted") + } +} + +func TestInitiateUploadMountsWithOriginFallback(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedMountRepo := "a/different/repo" + expectedRepo := "yet/again" + expectedPath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + expectedOrigin := "fakeOrigin" + expectedQuery := url.Values{ + "mount": []string{h.String()}, + "from": []string{expectedMountRepo}, + "origin": []string{expectedOrigin}, + }.Encode() + + queries := []string{expectedQuery, ""} + queryCount := 0 + + serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != queries[queryCount] { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + if queryCount == 0 { + http.Error(w, "nope", http.StatusUnauthorized) + } else { + http.Error(w, "Mounted", http.StatusCreated) + } + queryCount++ + }) + server := httptest.NewServer(serverHandler) + + w, closer, err := setupWriterWithServer(server, expectedRepo) + if err != nil { + t.Fatalf("setupWriterWithServer() = %v", err) + } + defer closer.Close() + + _, mounted, err := w.initiateUpload(context.Background(), expectedMountRepo, h.String(), "fakeOrigin") + if err != nil { + t.Errorf("intiateUpload() = %v", err) + } + if !mounted { + t.Error("initiateUpload() = !mounted, want mounted") + } +} + +func TestDedupeLayers(t *testing.T) { + newBlob := func() io.ReadCloser { return io.NopCloser(bytes.NewReader(bytes.Repeat([]byte{'a'}, 10000))) } + + img, err := random.Image(1024, 3) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + + // Append three identical tarball.Layers, which should be deduped + // because contents can be hashed before uploading. + for i := 0; i < 3; i++ { + tl, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return newBlob(), nil }) + if err != nil { + t.Fatalf("LayerFromOpener(#%d): %v", i, err) + } + img, err = mutate.AppendLayers(img, tl) + if err != nil { + t.Fatalf("mutate.AppendLayer(#%d): %v", i, err) + } + } + + // Append three identical stream.Layers, whose uploads will *not* be + // deduped since Write can't tell they're identical ahead of time. + for i := 0; i < 3; i++ { + sl := stream.NewLayer(newBlob()) + img, err = mutate.AppendLayers(img, sl) + if err != nil { + t.Fatalf("mutate.AppendLayer(#%d): %v", i, err) + } + } + + expectedRepo := "write/time" + headPathPrefix := fmt.Sprintf("/v2/%s/blobs/", expectedRepo) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + uploadPath := "/upload" + commitPath := "/commit" + var numUploads int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.HasPrefix(r.URL.Path, headPathPrefix) && r.URL.Path != initiatePath { + http.Error(w, "NotFound", http.StatusNotFound) + return + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + w.Header().Set("Location", uploadPath) + http.Error(w, "Accepted", http.StatusAccepted) + case uploadPath: + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + atomic.AddInt32(&numUploads, 1) + w.Header().Set("Location", commitPath) + http.Error(w, "Created", http.StatusCreated) + case commitPath: + http.Error(w, "Created", http.StatusCreated) + case manifestPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := Write(tag, img); err != nil { + t.Errorf("Write: %v", err) + } + + // 3 random layers, 1 tarball layer (deduped), 3 stream layers (not deduped), 1 image config blob + wantUploads := int32(3 + 1 + 3 + 1) + if numUploads != wantUploads { + t.Fatalf("Write uploaded %d blobs, want %d", numUploads, wantUploads) + } +} + +func TestStreamBlob(t *testing.T) { + img := setupImage(t) + expectedPath := "/vWhatever/I/decide" + expectedCommitLocation := "https://commit.io/v12/blob" + + w, closer, err := setupWriter("what/ever", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + got, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("ReadAll(Body) = %v", err) + } + want, err := img.RawConfigFile() + if err != nil { + t.Errorf("RawConfigFile() = %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("bytes.Equal(); got %v, want %v", got, want) + } + w.Header().Set("Location", expectedCommitLocation) + http.Error(w, "Created", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + streamLocation := w.url(expectedPath) + + l, err := partial.ConfigLayer(img) + if err != nil { + t.Fatalf("ConfigLayer: %v", err) + } + + commitLocation, err := w.streamBlob(context.Background(), l, streamLocation.String()) + if err != nil { + t.Errorf("streamBlob() = %v", err) + } + if commitLocation != expectedCommitLocation { + t.Errorf("streamBlob(); got %v, want %v", commitLocation, expectedCommitLocation) + } +} + +func TestStreamLayer(t *testing.T) { + var n, wantSize int64 = 10000, 49 + newBlob := func() io.ReadCloser { return io.NopCloser(bytes.NewReader(bytes.Repeat([]byte{'a'}, int(n)))) } + wantDigest := "sha256:3d7c465be28d9e1ed810c42aeb0e747b44441424f566722ba635dc93c947f30e" + + expectedPath := "/vWhatever/I/decide" + expectedCommitLocation := "https://commit.io/v12/blob" + w, closer, err := setupWriter("what/ever", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + + h := crypto.SHA256.New() + s, err := io.Copy(h, r.Body) + if err != nil { + t.Errorf("Reading body: %v", err) + } + if s != wantSize { + t.Errorf("Received %d bytes, want %d", s, wantSize) + } + gotDigest := "sha256:" + hex.EncodeToString(h.Sum(nil)) + if gotDigest != wantDigest { + t.Errorf("Received bytes with digest %q, want %q", gotDigest, wantDigest) + } + + w.Header().Set("Location", expectedCommitLocation) + http.Error(w, "Created", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + streamLocation := w.url(expectedPath) + sl := stream.NewLayer(newBlob()) + + commitLocation, err := w.streamBlob(context.Background(), sl, streamLocation.String()) + if err != nil { + t.Errorf("streamBlob: %v", err) + } + if commitLocation != expectedCommitLocation { + t.Errorf("streamBlob(); got %v, want %v", commitLocation, expectedCommitLocation) + } +} + +func TestCommitBlob(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedPath := "/no/commitment/issues" + expectedQuery := url.Values{ + "digest": []string{h.String()}, + }.Encode() + + w, closer, err := setupWriter("what/ever", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if r.URL.RawQuery != expectedQuery { + t.Errorf("RawQuery; got %v, want %v", r.URL.RawQuery, expectedQuery) + } + http.Error(w, "Created", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + commitLocation := w.url(expectedPath) + + if err := w.commitBlob(context.Background(), commitLocation.String(), h.String()); err != nil { + t.Errorf("commitBlob() = %v", err) + } +} + +func TestUploadOne(t *testing.T) { + img := setupImage(t) + h := mustConfigName(t, img) + expectedRepo := "baz/blah" + headPath := fmt.Sprintf("/v2/%s/blobs/%s", expectedRepo, h.String()) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + streamPath := "/path/to/upload" + commitPath := "/path/to/commit" + ctx := context.Background() + + uploaded := false + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case headPath: + if r.Method != http.MethodHead { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodHead) + } + if uploaded { + return + } + http.Error(w, "NotFound", http.StatusNotFound) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + w.Header().Set("Location", streamPath) + http.Error(w, "Initiated", http.StatusAccepted) + case streamPath: + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + got, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("ReadAll(Body) = %v", err) + } + want, err := img.RawConfigFile() + if err != nil { + t.Errorf("RawConfigFile() = %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("bytes.Equal(); got %v, want %v", got, want) + } + w.Header().Set("Location", commitPath) + http.Error(w, "Initiated", http.StatusAccepted) + case commitPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + uploaded = true + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + l, err := partial.ConfigLayer(img) + if err != nil { + t.Fatalf("ConfigLayer: %v", err) + } + ml := &MountableLayer{ + Layer: l, + Reference: w.repo.Digest(h.String()), + } + if err := w.uploadOne(ctx, ml); err != nil { + t.Errorf("uploadOne() = %v", err) + } + // Hit the existing blob path. + if err := w.uploadOne(ctx, l); err != nil { + t.Errorf("uploadOne() = %v", err) + } +} + +func TestUploadOneStreamedLayer(t *testing.T) { + expectedRepo := "baz/blah" + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + streamPath := "/path/to/upload" + commitPath := "/path/to/commit" + ctx := context.Background() + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + w.Header().Set("Location", streamPath) + http.Error(w, "Initiated", http.StatusAccepted) + case streamPath: + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + // TODO(jasonhall): What should we check here? + w.Header().Set("Location", commitPath) + http.Error(w, "Initiated", http.StatusAccepted) + case commitPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + var n, wantSize int64 = 10000, 49 + newBlob := func() io.ReadCloser { return io.NopCloser(bytes.NewReader(bytes.Repeat([]byte{'a'}, int(n)))) } + wantDigest := "sha256:3d7c465be28d9e1ed810c42aeb0e747b44441424f566722ba635dc93c947f30e" + wantDiffID := "sha256:27dd1f61b867b6a0f6e9d8a41c43231de52107e53ae424de8f847b821db4b711" + l := stream.NewLayer(newBlob()) + if err := w.uploadOne(ctx, l); err != nil { + t.Fatalf("uploadOne: %v", err) + } + + if dig, err := l.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if dig.String() != wantDigest { + t.Errorf("Digest got %q, want %q", dig, wantDigest) + } + if diffID, err := l.DiffID(); err != nil { + t.Errorf("DiffID: %v", err) + } else if diffID.String() != wantDiffID { + t.Errorf("DiffID got %q, want %q", diffID, wantDiffID) + } + if size, err := l.Size(); err != nil { + t.Errorf("Size: %v", err) + } else if size != wantSize { + t.Errorf("Size got %d, want %d", size, wantSize) + } +} + +func TestCommitImage(t *testing.T) { + img := setupImage(t) + ctx := context.Background() + + expectedRepo := "foo/bar" + expectedPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + got, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("ReadAll(Body) = %v", err) + } + want, err := img.RawManifest() + if err != nil { + t.Errorf("RawManifest() = %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("bytes.Equal(); got %v, want %v", got, want) + } + mt, err := img.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } + if got, want := r.Header.Get("Content-Type"), string(mt); got != want { + t.Errorf("Header; got %v, want %v", got, want) + } + http.Error(w, "Created", http.StatusCreated) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + if err := w.commitManifest(ctx, img, w.repo.Tag("latest")); err != nil { + t.Error("commitManifest() = ", err) + } +} + +func TestWrite(t *testing.T) { + img := setupImage(t) + expectedRepo := "write/time" + headPathPrefix := fmt.Sprintf("/v2/%s/blobs/", expectedRepo) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.HasPrefix(r.URL.Path, headPathPrefix) && r.URL.Path != initiatePath { + http.Error(w, "NotFound", http.StatusNotFound) + return + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + http.Error(w, "Mounted", http.StatusCreated) + case manifestPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := Write(tag, img); err != nil { + t.Errorf("Write() = %v", err) + } +} + +func TestWriteWithErrors(t *testing.T) { + img := setupImage(t) + expectedRepo := "write/time" + headPathPrefix := fmt.Sprintf("/v2/%s/blobs/", expectedRepo) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + + errorBody := `{"errors":[{"code":"NAME_INVALID","message":"some explanation of how things were messed up."}],"StatusCode":400}` + expectedErrMsg, err := regexp.Compile(`POST .+ NAME_INVALID: some explanation of how things were messed up.`) + if err != nil { + t.Error(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.HasPrefix(r.URL.Path, headPathPrefix) && r.URL.Path != initiatePath { + http.Error(w, "NotFound", http.StatusNotFound) + return + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(errorBody)) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + c := make(chan v1.Update, 100) + + var terr *transport.Error + if err := Write(tag, img, WithProgress(c)); err == nil { + t.Error("Write() = nil; wanted error") + } else if !errors.As(err, &terr) { + t.Errorf("Write() = %T; wanted *transport.Error", err) + } else if !expectedErrMsg.Match([]byte(terr.Error())) { + diff := cmp.Diff(expectedErrMsg, terr.Error()) + t.Errorf("Write(); (-want +got) = %s", diff) + } + + var last v1.Update + for update := range c { + last = update + } + if last.Error == nil { + t.Error("Progress chan didn't report error") + } +} + +func TestDockerhubScopes(t *testing.T) { + src, err := name.ParseReference("busybox") + if err != nil { + t.Fatal(err) + } + rl, err := random.Layer(1024, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + ml := &MountableLayer{ + Layer: rl, + Reference: src, + } + want := src.Scope(transport.PullScope) + + for _, s := range []string{ + "jonjohnson/busybox", + "docker.io/jonjohnson/busybox", + "index.docker.io/jonjohnson/busybox", + } { + dst, err := name.ParseReference(s) + if err != nil { + t.Fatal(err) + } + + scopes := scopesForUploadingImage(dst.Context(), []v1.Layer{ml}) + + if len(scopes) != 2 { + t.Errorf("Should have two scopes (src and dst), got %d", len(scopes)) + } else if diff := cmp.Diff(want, scopes[1]); diff != "" { + t.Errorf("TestDockerhubScopes %q: (-want +got) = %v", s, diff) + } + } +} + +func TestScopesForUploadingImage(t *testing.T) { + referenceToUpload, err := name.NewTag("example.com/sample/sample:latest", name.WeakValidation) + if err != nil { + t.Fatalf("name.NewTag() = %v", err) + } + + sameReference, err := name.NewTag("example.com/sample/sample:previous", name.WeakValidation) + if err != nil { + t.Fatalf("name.NewTag() = %v", err) + } + + anotherRepo1, err := name.NewTag("example.com/sample/another_repo1:latest", name.WeakValidation) + if err != nil { + t.Fatalf("name.NewTag() = %v", err) + } + + anotherRepo2, err := name.NewTag("example.com/sample/another_repo2:latest", name.WeakValidation) + if err != nil { + t.Fatalf("name.NewTag() = %v", err) + } + + repoOnOtherRegistry, err := name.NewTag("other-domain.com/sample/any_repo:latest", name.WeakValidation) + if err != nil { + t.Fatalf("name.NewTag() = %v", err) + } + + img := setupImage(t) + layers, err := img.Layers() + if err != nil { + t.Fatalf("img.Layers() = %v", err) + } + dummyLayer := layers[0] + + testCases := []struct { + name string + reference name.Reference + layers []v1.Layer + expected []string + }{ + { + name: "empty layers", + reference: referenceToUpload, + layers: []v1.Layer{}, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + }, + }, + { + name: "mountable layers with same reference", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: sameReference, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + }, + }, + { + name: "mountable layers with single reference with no-duplicate", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + anotherRepo1.Scope(transport.PullScope), + }, + }, + { + name: "mountable layers with single reference with duplicate", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + anotherRepo1.Scope(transport.PullScope), + }, + }, + { + name: "mountable layers with multiple references with no-duplicates", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo2, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + anotherRepo1.Scope(transport.PullScope), + anotherRepo2.Scope(transport.PullScope), + }, + }, + { + name: "mountable layers with multiple references with duplicates", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo2, + }, + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo1, + }, + &MountableLayer{ + Layer: dummyLayer, + Reference: anotherRepo2, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + anotherRepo1.Scope(transport.PullScope), + anotherRepo2.Scope(transport.PullScope), + }, + }, + { + name: "cross repository mountable layer", + reference: referenceToUpload, + layers: []v1.Layer{ + &MountableLayer{ + Layer: dummyLayer, + Reference: repoOnOtherRegistry, + }, + }, + expected: []string{ + referenceToUpload.Scope(transport.PushScope), + }, + }, + } + + for _, tc := range testCases { + actual := scopesForUploadingImage(tc.reference.Context(), tc.layers) + + if want, got := tc.expected[0], actual[0]; want != got { + t.Errorf("TestScopesForUploadingImage() %s: Wrong first scope; want %v, got %v", tc.name, want, got) + } + + less := func(a, b string) bool { + return strings.Compare(a, b) <= -1 + } + if diff := cmp.Diff(tc.expected[1:], actual[1:], cmpopts.SortSlices(less)); diff != "" { + t.Errorf("TestScopesForUploadingImage() %s: Wrong scopes (-want +got) = %v", tc.name, diff) + } + } +} + +func TestCheckExistingManifest(t *testing.T) { + tests := []struct { + name string + status int + existing bool + wantErr bool + }{{ + name: "success", + status: http.StatusOK, + existing: true, + }, { + name: "not found", + status: http.StatusNotFound, + existing: false, + }, { + name: "error", + status: http.StatusInternalServerError, + existing: false, + wantErr: true, + }} + + img := setupImage(t) + h := mustDigest(t, img) + mt := mustMediaType(t, img) + expectedRepo := "foo/bar" + expectedPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, h.String()) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w, closer, err := setupWriter(expectedRepo, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodHead) + } + if r.URL.Path != expectedPath { + t.Errorf("URL; got %v, want %v", r.URL.Path, expectedPath) + } + if got, want := r.Header.Get("Accept"), string(mt); got != want { + t.Errorf("r.Header['Accept']; got %v, want %v", got, want) + } + http.Error(w, http.StatusText(test.status), test.status) + })) + if err != nil { + t.Fatalf("setupWriter() = %v", err) + } + defer closer.Close() + + existing, err := w.checkExistingManifest(context.Background(), h, mt) + if test.existing != existing { + t.Errorf("checkExistingManifest() = %v, want %v", existing, test.existing) + } + if err != nil && !test.wantErr { + t.Errorf("checkExistingManifest() = %v", err) + } else if err == nil && test.wantErr { + t.Error("checkExistingManifest() wanted err, got nil") + } + }) + } +} + +func TestWriteIndex(t *testing.T) { + idx := setupIndex(t, 2) + expectedRepo := "write/time" + headPathPrefix := fmt.Sprintf("/v2/%s/blobs/", expectedRepo) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + childDigest := mustIndexManifest(t, idx).Manifests[0].Digest + childPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, childDigest) + existinChildDigest := mustIndexManifest(t, idx).Manifests[1].Digest + existingChildPath := fmt.Sprintf("/v2/%s/manifests/%s", expectedRepo, existinChildDigest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.HasPrefix(r.URL.Path, headPathPrefix) && r.URL.Path != initiatePath { + http.Error(w, "NotFound", http.StatusNotFound) + return + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + http.Error(w, "Mounted", http.StatusCreated) + case manifestPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + case existingChildPath: + if r.Method == http.MethodHead { + http.Error(w, http.StatusText(http.StatusOK), http.StatusOK) + return + } + t.Errorf("Unexpected method; got %v, want %v", r.Method, http.MethodHead) + case childPath: + if r.Method == http.MethodHead { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := WriteIndex(tag, idx); err != nil { + t.Errorf("WriteIndex() = %v", err) + } +} + +// If we actually attempt to read the contents, this will fail the test. +type fakeForeignLayer struct { + t *testing.T +} + +func (l *fakeForeignLayer) MediaType() (types.MediaType, error) { + return types.DockerForeignLayer, nil +} + +func (l *fakeForeignLayer) Size() (int64, error) { + return 0, nil +} + +func (l *fakeForeignLayer) Digest() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("a", 64)}, nil +} + +func (l *fakeForeignLayer) DiffID() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("a", 64)}, nil +} + +func (l *fakeForeignLayer) Compressed() (io.ReadCloser, error) { + l.t.Helper() + l.t.Errorf("foreign layer not skipped: Compressed") + return nil, nil +} + +func (l *fakeForeignLayer) Uncompressed() (io.ReadCloser, error) { + l.t.Helper() + l.t.Errorf("foreign layer not skipped: Uncompressed") + return nil, nil +} + +func TestSkipForeignLayersByDefault(t *testing.T) { + // Set up an image with a foreign layer. + base := setupImage(t) + img, err := mutate.AppendLayers(base, &fakeForeignLayer{t: t}) + if err != nil { + t.Fatal(err) + } + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + dst := fmt.Sprintf("%s/test/foreign/upload", u.Host) + ref, err := name.ParseReference(dst) + if err != nil { + t.Fatal(err) + } + + if err := Write(ref, img); err != nil { + t.Errorf("failed to Write: %v", err) + } +} + +func TestWriteForeignLayerIfOptionSet(t *testing.T) { + // Set up an image with a foreign layer. + base := setupImage(t) + foreignLayer, err := random.Layer(1024, types.DockerForeignLayer) + if err != nil { + t.Fatal("random.Layer:", err) + } + img, err := mutate.AppendLayers(base, foreignLayer) + if err != nil { + t.Fatal(err) + } + + expectedRepo := "write/time" + headPathPrefix := fmt.Sprintf("/v2/%s/blobs/", expectedRepo) + initiatePath := fmt.Sprintf("/v2/%s/blobs/uploads/", expectedRepo) + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", expectedRepo) + uploadPath := "/upload" + commitPath := "/commit" + var numUploads int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.HasPrefix(r.URL.Path, headPathPrefix) && r.URL.Path != initiatePath { + http.Error(w, "NotFound", http.StatusNotFound) + return + } + switch r.URL.Path { + case "/v2/": + w.WriteHeader(http.StatusOK) + case initiatePath: + if r.Method != http.MethodPost { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPost) + } + w.Header().Set("Location", uploadPath) + http.Error(w, "Accepted", http.StatusAccepted) + case uploadPath: + if r.Method != http.MethodPatch { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPatch) + } + atomic.AddInt32(&numUploads, 1) + w.Header().Set("Location", commitPath) + http.Error(w, "Created", http.StatusCreated) + case commitPath: + http.Error(w, "Created", http.StatusCreated) + case manifestPath: + if r.Method != http.MethodPut { + t.Errorf("Method; got %v, want %v", r.Method, http.MethodPut) + } + http.Error(w, "Created", http.StatusCreated) + default: + t.Fatalf("Unexpected path: %v", r.URL.Path) + } + })) + defer server.Close() + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("url.Parse(%v) = %v", server.URL, err) + } + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", u.Host, expectedRepo), name.WeakValidation) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + if err := Write(tag, img, WithNondistributable); err != nil { + t.Errorf("Write: %v", err) + } + + // 1 random layer, 1 foreign layer, 1 image config blob + wantUploads := int32(1 + 1 + 1) + if numUploads != wantUploads { + t.Fatalf("Write uploaded %d blobs, want %d", numUploads, wantUploads) + } +} + +func TestTag(t *testing.T) { + idx := setupIndex(t, 3) + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + src := fmt.Sprintf("%s/test/tag:src", u.Host) + srcRef, err := name.NewTag(src) + if err != nil { + t.Fatal(err) + } + + if err := WriteIndex(srcRef, idx); err != nil { + t.Fatal(err) + } + + dst := fmt.Sprintf("%s/test/tag:dst", u.Host) + dstRef, err := name.NewTag(dst) + if err != nil { + t.Fatal(err) + } + + if err := Tag(dstRef, idx); err != nil { + t.Fatal(err) + } + + got, err := Index(dstRef) + if err != nil { + t.Fatal(err) + } + + if err := validate.Index(got); err != nil { + t.Errorf("Validate() = %v", err) + } +} + +func TestTagDescriptor(t *testing.T) { + idx := setupIndex(t, 3) + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + src := fmt.Sprintf("%s/test/tag:src", u.Host) + srcRef, err := name.NewTag(src) + if err != nil { + t.Fatal(err) + } + + if err := WriteIndex(srcRef, idx); err != nil { + t.Fatal(err) + } + + desc, err := Get(srcRef) + if err != nil { + t.Fatal(err) + } + + dst := fmt.Sprintf("%s/test/tag:dst", u.Host) + dstRef, err := name.NewTag(dst) + if err != nil { + t.Fatal(err) + } + + if err := Tag(dstRef, desc); err != nil { + t.Fatal(err) + } +} + +func TestNestedIndex(t *testing.T) { + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + src := fmt.Sprintf("%s/test/tag:src", u.Host) + srcRef, err := name.NewTag(src) + if err != nil { + t.Fatal(err) + } + + child, err := random.Index(1024, 1, 1) + if err != nil { + t.Fatal(err) + } + parent := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{ + Add: child, + Descriptor: v1.Descriptor{ + URLs: []string{"example.com/url"}, + }, + }) + + l, err := random.Layer(100, types.DockerLayer) + if err != nil { + t.Fatal(err) + } + + parent = mutate.AppendManifests(parent, mutate.IndexAddendum{ + Add: l, + }) + + if err := WriteIndex(srcRef, parent); err != nil { + t.Fatal(err) + } + pulled, err := Index(srcRef) + if err != nil { + t.Fatal(err) + } + + if err := validate.Index(pulled); err != nil { + t.Fatalf("validate.Index: %v", err) + } + + digest, err := child.Digest() + if err != nil { + t.Fatal(err) + } + + pulledChild, err := pulled.ImageIndex(digest) + if err != nil { + t.Fatal(err) + } + + desc, err := partial.Descriptor(pulledChild) + if err != nil { + t.Fatal(err) + } + + if len(desc.URLs) != 1 { + t.Fatalf("expected url for pulledChild") + } + + if want, got := "example.com/url", desc.URLs[0]; want != got { + t.Errorf("pulledChild.urls[0] = %s != %s", got, want) + } +} + +func BenchmarkWrite(b *testing.B) { + // unfortunately the registry _and_ the img have caching behaviour, so we need a new registry + // and image every iteration of benchmarking. + for i := 0; i < b.N; i++ { + // set up the registry + s := httptest.NewServer(registry.New()) + defer s.Close() + + // load the image + img, err := random.Image(50*1024*1024, 10) + if err != nil { + b.Fatalf("random.Image(...): %v", err) + } + + b.ResetTimer() + + tagStr := strings.TrimPrefix(s.URL+"/test/image:tag", "http://") + tag, err := name.NewTag(tagStr) + if err != nil { + b.Fatalf("parsing tag (%s): %v", tagStr, err) + } + + err = Write(tag, img) + if err != nil { + b.Fatalf("pushing tag one: %v", err) + } + } +} diff --git a/pkg/v1/static/layer.go b/pkg/v1/static/layer.go new file mode 100644 index 0000000..a4bbe69 --- /dev/null +++ b/pkg/v1/static/layer.go @@ -0,0 +1,68 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package static + +import ( + "bytes" + "io" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// NewLayer returns a layer containing the given bytes, with the given mediaType. +// +// Contents will not be compressed. +func NewLayer(b []byte, mt types.MediaType) v1.Layer { + return &staticLayer{b: b, mt: mt} +} + +type staticLayer struct { + b []byte + mt types.MediaType + + once sync.Once + h v1.Hash +} + +func (l *staticLayer) Digest() (v1.Hash, error) { + var err error + // Only calculate digest the first time we're asked. + l.once.Do(func() { + l.h, _, err = v1.SHA256(bytes.NewReader(l.b)) + }) + return l.h, err +} + +func (l *staticLayer) DiffID() (v1.Hash, error) { + return l.Digest() +} + +func (l *staticLayer) Compressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(l.b)), nil +} + +func (l *staticLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(l.b)), nil +} + +func (l *staticLayer) Size() (int64, error) { + return int64(len(l.b)), nil +} + +func (l *staticLayer) MediaType() (types.MediaType, error) { + return l.mt, nil +} diff --git a/pkg/v1/static/static_test.go b/pkg/v1/static/static_test.go new file mode 100644 index 0000000..1dcc86a --- /dev/null +++ b/pkg/v1/static/static_test.go @@ -0,0 +1,83 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package static + +import ( + "io" + "strings" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestNewLayer(t *testing.T) { + b := []byte(strings.Repeat(".", 10)) + l := NewLayer(b, types.OCILayer) + + // This does basically nothing. + if err := validate.Layer(l, validate.Fast); err != nil { + t.Fatal(err) + } + + // Digest and DiffID match, and match expectations. + h, err := l.Digest() + if err != nil { + t.Fatal(err) + } + h2, err := l.DiffID() + if err != nil { + t.Fatal(err) + } + if h != h2 { + t.Errorf("Digest != DiffID; digest is %v, diffid is %v", h, h2) + } + wantDigest, err := v1.NewHash("sha256:537f3fb69ba01fc388a3a5c920c485b2873d5f327305c3dd2004d6a04451659b") + if err != nil { + t.Fatal(err) + } + if h != wantDigest { + t.Errorf("Digest mismatch; got %v, want %v", h, wantDigest) + } + + sz, err := l.Size() + if err != nil { + t.Fatal(err) + } + if sz != 10 { + t.Errorf("Size mismatch; got %d, want %d", sz, 10) + } + + mt, err := l.MediaType() + if err != nil { + t.Fatal(err) + } + if mt != types.OCILayer { + t.Errorf("MediaType mismatch; got %v, want %v", mt, types.OCILayer) + } + + r, err := l.Uncompressed() + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(got) != string(b) { + t.Errorf("Contents mismatch: got %q, want %q", string(got), string(b)) + } +} diff --git a/pkg/v1/stream/README.md b/pkg/v1/stream/README.md new file mode 100644 index 0000000..da0dda4 --- /dev/null +++ b/pkg/v1/stream/README.md @@ -0,0 +1,68 @@ +# `stream` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream) + +The `stream` package contains an implementation of +[`v1.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1#Layer) +that supports _streaming_ access, i.e. the layer contents are read once and not +buffered. + +## Usage + +```go +package main + +import ( + "os" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/stream" +) + +// upload the contents of stdin as a layer to a local registry +func main() { + repo, err := name.NewRepository("localhost:5000/stream") + if err != nil { + panic(err) + } + + layer := stream.NewLayer(os.Stdin) + + if err := remote.WriteLayer(repo, layer); err != nil { + panic(err) + } +} +``` + +## Structure + +This implements the layer portion of an [image +upload](/pkg/v1/remote#anatomy-of-an-image-upload). We launch a goroutine that +is responsible for hashing the uncompressed contents to compute the `DiffID`, +gzipping them to produce the `Compressed` contents, and hashing/counting the +bytes to produce the `Digest`/`Size`. This goroutine writes to an +`io.PipeWriter`, which blocks until `Compressed` reads the gzipped contents from +the corresponding `io.PipeReader`. + +

+ +

+ +## Caveats + +This assumes that you have an uncompressed layer (i.e. a tarball) and would like +to compress it. Calling `Uncompressed` is always an error. Likewise, other +methods are invalid until the contents of `Compressed` have been completely +consumed and `Close`d. + +Using a `stream.Layer` will likely not work without careful consideration. For +example, in the `mutate` package, we defer computing the manifest and config +file until they are actually called. This allows you to `mutate.Append` a +streaming layer to an image without accidentally consuming it. Similarly, in +`remote.Write`, if calling `Digest` on a layer fails, we attempt to upload the +layer anyway, understanding that we may be dealing with a `stream.Layer` whose +contents need to be uploaded before we can upload the config file. + +Given the [structure](#structure) of how this is implemented, forgetting to +`Close` a `stream.Layer` will leak a goroutine. diff --git a/pkg/v1/stream/layer.go b/pkg/v1/stream/layer.go new file mode 100644 index 0000000..d6f2df8 --- /dev/null +++ b/pkg/v1/stream/layer.go @@ -0,0 +1,273 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package stream implements a single-pass streaming v1.Layer. +package stream + +import ( + "bufio" + "compress/gzip" + "crypto" + "encoding/hex" + "errors" + "hash" + "io" + "os" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var ( + // ErrNotComputed is returned when the requested value is not yet + // computed because the stream has not been consumed yet. + ErrNotComputed = errors.New("value not computed until stream is consumed") + + // ErrConsumed is returned by Compressed when the underlying stream has + // already been consumed and closed. + ErrConsumed = errors.New("stream was already consumed") +) + +// Layer is a streaming implementation of v1.Layer. +type Layer struct { + blob io.ReadCloser + consumed bool + compression int + + mu sync.Mutex + digest, diffID *v1.Hash + size int64 + mediaType types.MediaType +} + +var _ v1.Layer = (*Layer)(nil) + +// LayerOption applies options to layer +type LayerOption func(*Layer) + +// WithCompressionLevel sets the gzip compression. See `gzip.NewWriterLevel` for possible values. +func WithCompressionLevel(level int) LayerOption { + return func(l *Layer) { + l.compression = level + } +} + +// WithMediaType is a functional option for overriding the layer's media type. +func WithMediaType(mt types.MediaType) LayerOption { + return func(l *Layer) { + l.mediaType = mt + } +} + +// NewLayer creates a Layer from an io.ReadCloser. +func NewLayer(rc io.ReadCloser, opts ...LayerOption) *Layer { + layer := &Layer{ + blob: rc, + compression: gzip.BestSpeed, + // We use DockerLayer for now as uncompressed layers + // are unimplemented + mediaType: types.DockerLayer, + } + + for _, opt := range opts { + opt(layer) + } + + return layer +} + +// Digest implements v1.Layer. +func (l *Layer) Digest() (v1.Hash, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.digest == nil { + return v1.Hash{}, ErrNotComputed + } + return *l.digest, nil +} + +// DiffID implements v1.Layer. +func (l *Layer) DiffID() (v1.Hash, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.diffID == nil { + return v1.Hash{}, ErrNotComputed + } + return *l.diffID, nil +} + +// Size implements v1.Layer. +func (l *Layer) Size() (int64, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.size == 0 { + return 0, ErrNotComputed + } + return l.size, nil +} + +// MediaType implements v1.Layer +func (l *Layer) MediaType() (types.MediaType, error) { + return l.mediaType, nil +} + +// Uncompressed implements v1.Layer. +func (l *Layer) Uncompressed() (io.ReadCloser, error) { + return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") +} + +// Compressed implements v1.Layer. +func (l *Layer) Compressed() (io.ReadCloser, error) { + if l.consumed { + return nil, ErrConsumed + } + return newCompressedReader(l) +} + +// finalize sets the layer to consumed and computes all hash and size values. +func (l *Layer) finalize(uncompressed, compressed hash.Hash, size int64) error { + l.mu.Lock() + defer l.mu.Unlock() + + diffID, err := v1.NewHash("sha256:" + hex.EncodeToString(uncompressed.Sum(nil))) + if err != nil { + return err + } + l.diffID = &diffID + + digest, err := v1.NewHash("sha256:" + hex.EncodeToString(compressed.Sum(nil))) + if err != nil { + return err + } + l.digest = &digest + + l.size = size + l.consumed = true + return nil +} + +type compressedReader struct { + pr io.Reader + closer func() error +} + +func newCompressedReader(l *Layer) (*compressedReader, error) { + // Collect digests of compressed and uncompressed stream and size of + // compressed stream. + h := crypto.SHA256.New() + zh := crypto.SHA256.New() + count := &countWriter{} + + // gzip.Writer writes to the output stream via pipe, a hasher to + // capture compressed digest, and a countWriter to capture compressed + // size. + pr, pw := io.Pipe() + + // Write compressed bytes to be read by the pipe.Reader, hashed by zh, and counted by count. + mw := io.MultiWriter(pw, zh, count) + + // Buffer the output of the gzip writer so we don't have to wait on pr to keep writing. + // 64K ought to be small enough for anybody. + bw := bufio.NewWriterSize(mw, 2<<16) + zw, err := gzip.NewWriterLevel(bw, l.compression) + if err != nil { + return nil, err + } + + doneDigesting := make(chan struct{}) + + cr := &compressedReader{ + pr: pr, + closer: func() error { + // Immediately close pw without error. There are three ways to get + // here. + // + // 1. There was a copy error due from the underlying reader, in which + // case the error will not be overwritten. + // 2. Copying from the underlying reader completed successfully. + // 3. Close has been called before the underlying reader has been + // fully consumed. In this case pw must be closed in order to + // keep the flush of bw from blocking indefinitely. + // + // NOTE: pw.Close never returns an error. The signature is only to + // implement io.Closer. + _ = pw.Close() + + // Close the inner ReadCloser. + // + // NOTE: net/http will call close on success, so if we've already + // closed the inner rc, it's not an error. + if err := l.blob.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + return err + } + + // Finalize layer with its digest and size values. + <-doneDigesting + return l.finalize(h, zh, count.n) + }, + } + go func() { + // Copy blob into the gzip writer, which also hashes and counts the + // size of the compressed output, and hasher of the raw contents. + _, copyErr := io.Copy(io.MultiWriter(h, zw), l.blob) + + // Close the gzip writer once copying is done. If this is done in the + // Close method of compressedReader instead, then it can cause a panic + // when the compressedReader is closed before the blob is fully + // consumed and io.Copy in this goroutine is still blocking. + closeErr := zw.Close() + + // Check errors from writing and closing streams. + if copyErr != nil { + close(doneDigesting) + pw.CloseWithError(copyErr) + return + } + if closeErr != nil { + close(doneDigesting) + pw.CloseWithError(closeErr) + return + } + + // Flush the buffer once all writes are complete to the gzip writer. + if err := bw.Flush(); err != nil { + close(doneDigesting) + pw.CloseWithError(err) + return + } + + // Notify closer that digests are done being written. + close(doneDigesting) + + // Close the compressed reader to calculate digest/diffID/size. This + // will cause pr to return EOF which will cause readers of the + // Compressed stream to finish reading. + pw.CloseWithError(cr.Close()) + }() + + return cr, nil +} + +func (cr *compressedReader) Read(b []byte) (int, error) { return cr.pr.Read(b) } + +func (cr *compressedReader) Close() error { return cr.closer() } + +// countWriter counts bytes written to it. +type countWriter struct{ n int64 } + +func (c *countWriter) Write(p []byte) (int, error) { + c.n += int64(len(p)) + return len(p), nil +} diff --git a/pkg/v1/stream/layer_test.go b/pkg/v1/stream/layer_test.go new file mode 100644 index 0000000..e65452b --- /dev/null +++ b/pkg/v1/stream/layer_test.go @@ -0,0 +1,298 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stream + +import ( + "archive/tar" + "bytes" + "crypto/rand" + "errors" + "fmt" + "io" + "strings" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestStreamVsBuffer(t *testing.T) { + var n, wantSize int64 = 10000, 49 + newBlob := func() io.ReadCloser { return io.NopCloser(bytes.NewReader(bytes.Repeat([]byte{'a'}, int(n)))) } + wantDigest := "sha256:3d7c465be28d9e1ed810c42aeb0e747b44441424f566722ba635dc93c947f30e" + wantDiffID := "sha256:27dd1f61b867b6a0f6e9d8a41c43231de52107e53ae424de8f847b821db4b711" + + // Check that streaming some content results in the expected digest/diffID/size. + l := NewLayer(newBlob()) + if c, err := l.Compressed(); err != nil { + t.Errorf("Compressed: %v", err) + } else { + if _, err := io.Copy(io.Discard, c); err != nil { + t.Errorf("error reading Compressed: %v", err) + } + if err := c.Close(); err != nil { + t.Errorf("Close: %v", err) + } + } + if d, err := l.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if d.String() != wantDigest { + t.Errorf("stream Digest got %q, want %q", d.String(), wantDigest) + } + if d, err := l.DiffID(); err != nil { + t.Errorf("DiffID: %v", err) + } else if d.String() != wantDiffID { + t.Errorf("stream DiffID got %q, want %q", d.String(), wantDiffID) + } + if s, err := l.Size(); err != nil { + t.Errorf("Size: %v", err) + } else if s != wantSize { + t.Errorf("stream Size got %d, want %d", s, wantSize) + } + + // Test that buffering the same contents and using + // tarball.LayerFromOpener results in the same digest/diffID/size. + tl, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return newBlob(), nil }) + if err != nil { + t.Fatalf("LayerFromOpener: %v", err) + } + if d, err := tl.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if d.String() != wantDigest { + t.Errorf("tarball Digest got %q, want %q", d.String(), wantDigest) + } + if d, err := tl.DiffID(); err != nil { + t.Errorf("DiffID: %v", err) + } else if d.String() != wantDiffID { + t.Errorf("tarball DiffID got %q, want %q", d.String(), wantDiffID) + } + if s, err := tl.Size(); err != nil { + t.Errorf("Size: %v", err) + } else if s != wantSize { + t.Errorf("stream Size got %d, want %d", s, wantSize) + } + + // Test with different compression + l2 := NewLayer(newBlob(), WithCompressionLevel(2)) + l2WantDigest := "sha256:c9afe7b0da6783232e463e12328cb306142548384accf3995806229c9a6a707f" + if c, err := l2.Compressed(); err != nil { + t.Errorf("Compressed: %v", err) + } else { + if _, err := io.Copy(io.Discard, c); err != nil { + t.Errorf("error reading Compressed: %v", err) + } + if err := c.Close(); err != nil { + t.Errorf("Close: %v", err) + } + } + if d, err := l2.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if d.String() != l2WantDigest { + t.Errorf("stream Digest got %q, want %q", d.String(), l2WantDigest) + } +} + +func TestLargeStream(t *testing.T) { + var n, wantSize int64 = 10000000, 10000788 // "Compressing" n random bytes results in this many bytes. + sl := NewLayer(io.NopCloser(io.LimitReader(rand.Reader, n))) + rc, err := sl.Compressed() + if err != nil { + t.Fatalf("Uncompressed: %v", err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Fatalf("Reading layer: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + if dig, err := sl.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if dig.String() == (v1.Hash{}).String() { + t.Errorf("Digest got %q, want anything else", (v1.Hash{}).String()) + } + if diffID, err := sl.DiffID(); err != nil { + t.Errorf("DiffID: %v", err) + } else if diffID.String() == (v1.Hash{}).String() { + t.Errorf("DiffID got %q, want anything else", (v1.Hash{}).String()) + } + if size, err := sl.Size(); err != nil { + t.Errorf("Size: %v", err) + } else if size != wantSize { + t.Errorf("Size got %d, want %d", size, wantSize) + } +} + +func TestStreamableLayerFromTarball(t *testing.T) { + pr, pw := io.Pipe() + tw := tar.NewWriter(pw) + go func() { + // "Stream" a bunch of files into the layer. + pw.CloseWithError(func() error { + for i := 0; i < 1000; i++ { + name := fmt.Sprintf("file-%d.txt", i) + body := fmt.Sprintf("i am file number %d", i) + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Mode: 0600, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + if _, err := tw.Write([]byte(body)); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + return nil + }()) + }() + + l := NewLayer(pr) + rc, err := l.Compressed() + if err != nil { + t.Fatalf("Compressed: %v", err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Fatalf("Copy: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + wantDigest := "sha256:ed80efd7e7e884fb59db568f234332283b341b96155e872d638de42d55a34198" + if got, err := l.Digest(); err != nil { + t.Errorf("Digest: %v", err) + } else if got.String() != wantDigest { + t.Errorf("Digest: got %q, want %q", got.String(), wantDigest) + } +} + +// TestNotComputed tests that Digest/DiffID/Size return ErrNotComputed before +// the stream has been consumed. +func TestNotComputed(t *testing.T) { + l := NewLayer(io.NopCloser(bytes.NewBufferString("hi"))) + + // All methods should return ErrNotComputed until the stream has been + // consumed and closed. + if _, err := l.Size(); !errors.Is(err, ErrNotComputed) { + t.Errorf("Size: got %v, want %v", err, ErrNotComputed) + } + if _, err := l.Digest(); err == nil { + t.Errorf("Digest: got %v, want %v", err, ErrNotComputed) + } + if _, err := l.DiffID(); err == nil { + t.Errorf("DiffID: got %v, want %v", err, ErrNotComputed) + } +} + +// TestConsumed tests that Compressed returns ErrConsumed when the stream has +// already been consumed. +func TestConsumed(t *testing.T) { + l := NewLayer(io.NopCloser(strings.NewReader("hello"))) + rc, err := l.Compressed() + if err != nil { + t.Errorf("Compressed: %v", err) + } + if _, err := io.Copy(io.Discard, rc); err != nil { + t.Errorf("Error reading contents: %v", err) + } + if err := rc.Close(); err != nil { + t.Errorf("Close: %v", err) + } + + if _, err := l.Compressed(); !errors.Is(err, ErrConsumed) { + t.Errorf("Compressed() after consuming; got %v, want %v", err, ErrConsumed) + } +} + +func TestCloseTextStreamBeforeConsume(t *testing.T) { + // Create stream layer from tar pipe + l := NewLayer(io.NopCloser(strings.NewReader("hello"))) + rc, err := l.Compressed() + if err != nil { + t.Fatalf("Compressed: %v", err) + } + + // Close stream layer before consuming + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } +} + +func TestCloseTarStreamBeforeConsume(t *testing.T) { + // Write small tar to pipe + pr, pw := io.Pipe() + tw := tar.NewWriter(pw) + go func() { + pw.CloseWithError(func() error { + body := "test file" + if err := tw.WriteHeader(&tar.Header{ + Name: "test.txt", + Mode: 0600, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + if _, err := tw.Write([]byte(body)); err != nil { + return err + } + return tw.Close() + }()) + }() + + // Create stream layer from tar pipe + l := NewLayer(pr) + rc, err := l.Compressed() + if err != nil { + t.Fatalf("Compressed: %v", err) + } + + // Close stream layer before consuming + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } +} + +func TestMediaType(t *testing.T) { + l := NewLayer(io.NopCloser(strings.NewReader("hello"))) + mediaType, err := l.MediaType() + + if err != nil { + t.Fatalf("MediaType(): %v", err) + } + + if got, want := mediaType, types.DockerLayer; got != want { + t.Errorf("MediaType(): want %q, got %q", want, got) + } +} + +func TestMediaTypeOption(t *testing.T) { + l := NewLayer(io.NopCloser(strings.NewReader("hello")), WithMediaType(types.OCILayer)) + mediaType, err := l.MediaType() + + if err != nil { + t.Fatalf("MediaType(): %v", err) + } + + if got, want := mediaType, types.OCILayer; got != want { + t.Errorf("MediaType(): want %q, got %q", want, got) + } +} diff --git a/pkg/v1/tarball/README.md b/pkg/v1/tarball/README.md new file mode 100644 index 0000000..03f339b --- /dev/null +++ b/pkg/v1/tarball/README.md @@ -0,0 +1,280 @@ +# `tarball` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball) + +This package produces tarballs that can consumed via `docker load`. Note +that this is a _different_ format from the [`legacy`](/pkg/legacy/tarball) +tarballs that are produced by `docker save`, but this package is still able to +read the legacy tarballs produced by `docker save`. + +## Usage + +```go +package main + +import ( + "os" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func main() { + // Read a tarball from os.Args[1] that contains ubuntu. + tag, err := name.NewTag("ubuntu") + if err != nil { + panic(err) + } + img, err := tarball.ImageFromPath(os.Args[1], &tag) + if err != nil { + panic(err) + } + + // Write that tarball to os.Args[2] with a different tag. + newTag, err := name.NewTag("ubuntu:newest") + if err != nil { + panic(err) + } + f, err := os.Create(os.Args[2]) + if err != nil { + panic(err) + } + defer f.Close() + + if err := tarball.Write(newTag, img, f); err != nil { + panic(err) + } +} +``` + +## Structure + +

+ +

+ +Let's look at what happens when we write out a tarball: + + +### `ubuntu:latest` + +``` +$ crane pull ubuntu ubuntu.tar && mkdir ubuntu && tar xf ubuntu.tar -C ubuntu && rm ubuntu.tar +$ tree ubuntu/ +ubuntu/ +├── 423ae2b273f4c17ceee9e8482fa8d071d90c7d052ae208e1fe4963fceb3d6954.tar.gz +├── b6b53be908de2c0c78070fff0a9f04835211b3156c4e73785747af365e71a0d7.tar.gz +├── de83a2304fa1f7c4a13708a0d15b9704f5945c2be5cbb2b3ed9b2ccb718d0b3d.tar.gz +├── f9a83bce3af0648efaa60b9bb28225b09136d2d35d0bed25ac764297076dec1b.tar.gz +├── manifest.json +└── sha256:72300a873c2ca11c70d0c8642177ce76ff69ae04d61a5813ef58d40ff66e3e7c + +0 directories, 6 files +``` + +There are a couple interesting files here. + +`manifest.json` is the entrypoint: a list of [`tarball.Descriptor`s](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/tarball#Descriptor) +that describe the images contained in this tarball. + +For each image, this has the `RepoTags` (how it was pulled), a `Config` file +that points to the image's config file, a list of `Layers`, and (optionally) +`LayerSources`. + +``` +$ jq < ubuntu/manifest.json +[ + { + "Config": "sha256:72300a873c2ca11c70d0c8642177ce76ff69ae04d61a5813ef58d40ff66e3e7c", + "RepoTags": [ + "ubuntu" + ], + "Layers": [ + "423ae2b273f4c17ceee9e8482fa8d071d90c7d052ae208e1fe4963fceb3d6954.tar.gz", + "de83a2304fa1f7c4a13708a0d15b9704f5945c2be5cbb2b3ed9b2ccb718d0b3d.tar.gz", + "f9a83bce3af0648efaa60b9bb28225b09136d2d35d0bed25ac764297076dec1b.tar.gz", + "b6b53be908de2c0c78070fff0a9f04835211b3156c4e73785747af365e71a0d7.tar.gz" + ] + } +] +``` + +The config file and layers are exactly what you would expect, and match the +registry representations of the same artifacts. You'll notice that the +`manifest.json` contains similar information as the registry manifest, but isn't +quite the same: + +``` +$ crane manifest ubuntu@sha256:0925d086715714114c1988f7c947db94064fd385e171a63c07730f1fa014e6f9 +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 3408, + "digest": "sha256:72300a873c2ca11c70d0c8642177ce76ff69ae04d61a5813ef58d40ff66e3e7c" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 26692096, + "digest": "sha256:423ae2b273f4c17ceee9e8482fa8d071d90c7d052ae208e1fe4963fceb3d6954" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 35365, + "digest": "sha256:de83a2304fa1f7c4a13708a0d15b9704f5945c2be5cbb2b3ed9b2ccb718d0b3d" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 852, + "digest": "sha256:f9a83bce3af0648efaa60b9bb28225b09136d2d35d0bed25ac764297076dec1b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 163, + "digest": "sha256:b6b53be908de2c0c78070fff0a9f04835211b3156c4e73785747af365e71a0d7" + } + ] +} +``` + +This makes it difficult to maintain image digests when roundtripping images +through the tarball format, so it's not a great format if you care about +provenance. + +The ubuntu example didn't have any `LayerSources` -- let's look at another image +that does. + +### `hello-world:nanoserver` + +``` +$ crane pull hello-world:nanoserver@sha256:63c287625c2b0b72900e562de73c0e381472a83b1b39217aef3856cd398eca0b nanoserver.tar +$ mkdir nanoserver && tar xf nanoserver.tar -C nanoserver && rm nanoserver.tar +$ tree nanoserver/ +nanoserver/ +├── 10d1439be4eb8819987ec2e9c140d44d74d6b42a823d57fe1953bd99948e1bc0.tar.gz +├── a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053.tar.gz +├── be21f08f670160cbae227e3053205b91d6bfa3de750b90c7e00bd2c511ccb63a.tar.gz +├── manifest.json +└── sha256:bc5d255ea81f83c8c38a982a6d29a6f2198427d258aea5f166e49856896b2da6 + +0 directories, 5 files + +$ jq < nanoserver/manifest.json +[ + { + "Config": "sha256:bc5d255ea81f83c8c38a982a6d29a6f2198427d258aea5f166e49856896b2da6", + "RepoTags": [ + "index.docker.io/library/hello-world:i-was-a-digest" + ], + "Layers": [ + "a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053.tar.gz", + "be21f08f670160cbae227e3053205b91d6bfa3de750b90c7e00bd2c511ccb63a.tar.gz", + "10d1439be4eb8819987ec2e9c140d44d74d6b42a823d57fe1953bd99948e1bc0.tar.gz" + ], + "LayerSources": { + "sha256:26fd2d9d4c64a4f965bbc77939a454a31b607470f430b5d69fc21ded301fa55e": { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 101145811, + "digest": "sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053", + "urls": [ + "https://mcr.microsoft.com/v2/windows/nanoserver/blobs/sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053" + ] + } + } + } +] +``` + +A couple things to note about this `manifest.json` versus the other: +* The `RepoTags` field is a bit weird here. `hello-world` is a multi-platform + image, so We had to pull this image by digest, since we're (I'm) on + amd64/linux and wanted to grab a windows image. Since the tarball format + expects a tag under `RepoTags`, and we didn't pull by tag, we replace the + digest with a sentinel `i-was-a-digest` "tag" to appease docker. +* The `LayerSources` has enough information to reconstruct the foreign layers + pointer when pushing/pulling from the registry. For legal reasons, microsoft + doesn't want anyone but them to serve windows base images, so the mediaType + here indicates a "foreign" or "non-distributable" layer with an URL for where + you can download it from microsoft (see the [OCI + image-spec](https://github.com/opencontainers/image-spec/blob/master/layer.md#non-distributable-layers)). + +We can look at what's in the registry to explain both of these things: +``` +$ crane manifest hello-world:nanoserver | jq . +{ + "manifests": [ + { + "digest": "sha256:63c287625c2b0b72900e562de73c0e381472a83b1b39217aef3856cd398eca0b", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.1040" + }, + "size": 1124 + } + ], + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "schemaVersion": 2 +} + + +# Note the media type and "urls" field. +$ crane manifest hello-world:nanoserver@sha256:63c287625c2b0b72900e562de73c0e381472a83b1b39217aef3856cd398eca0b | jq . +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1721, + "digest": "sha256:bc5d255ea81f83c8c38a982a6d29a6f2198427d258aea5f166e49856896b2da6" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 101145811, + "digest": "sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053", + "urls": [ + "https://mcr.microsoft.com/v2/windows/nanoserver/blobs/sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053" + ] + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1669, + "digest": "sha256:be21f08f670160cbae227e3053205b91d6bfa3de750b90c7e00bd2c511ccb63a" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 949, + "digest": "sha256:10d1439be4eb8819987ec2e9c140d44d74d6b42a823d57fe1953bd99948e1bc0" + } + ] +} +``` + +The `LayerSources` map is keyed by the diffid. Note that `sha256:26fd2d9d4c64a4f965bbc77939a454a31b607470f430b5d69fc21ded301fa55e` matches the first layer in the config file: +``` +$ jq '.[0].LayerSources' < nanoserver/manifest.json +{ + "sha256:26fd2d9d4c64a4f965bbc77939a454a31b607470f430b5d69fc21ded301fa55e": { + "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + "size": 101145811, + "digest": "sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053", + "urls": [ + "https://mcr.microsoft.com/v2/windows/nanoserver/blobs/sha256:a35da61c356213336e646756218539950461ff2bf096badf307a23add6e70053" + ] + } +} + +$ jq < nanoserver/sha256\:bc5d255ea81f83c8c38a982a6d29a6f2198427d258aea5f166e49856896b2da6 | jq .rootfs +{ + "type": "layers", + "diff_ids": [ + "sha256:26fd2d9d4c64a4f965bbc77939a454a31b607470f430b5d69fc21ded301fa55e", + "sha256:601cf7d78c62e4b4d32a7bbf96a17606a9cea5bd9d22ffa6f34aa431d056b0e8", + "sha256:a1e1a3bf6529adcce4d91dce2cad86c2604a66b507ccbc4d2239f3da0ec5aab9" + ] +} +``` diff --git a/pkg/v1/tarball/doc.go b/pkg/v1/tarball/doc.go new file mode 100644 index 0000000..4eb79bb --- /dev/null +++ b/pkg/v1/tarball/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarball provides facilities for reading/writing v1.Images from/to +// a tarball on-disk. +package tarball diff --git a/pkg/v1/tarball/image.go b/pkg/v1/tarball/image.go new file mode 100644 index 0000000..1f977e1 --- /dev/null +++ b/pkg/v1/tarball/image.go @@ -0,0 +1,429 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sync" + + comp "github.com/google/go-containerregistry/internal/compression" + "github.com/google/go-containerregistry/pkg/compression" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type image struct { + opener Opener + manifest *Manifest + config []byte + imgDescriptor *Descriptor + + tag *name.Tag +} + +type uncompressedImage struct { + *image +} + +type compressedImage struct { + *image + manifestLock sync.Mutex // Protects manifest + manifest *v1.Manifest +} + +var _ partial.UncompressedImageCore = (*uncompressedImage)(nil) +var _ partial.CompressedImageCore = (*compressedImage)(nil) + +// Opener is a thunk for opening a tar file. +type Opener func() (io.ReadCloser, error) + +func pathOpener(path string) Opener { + return func() (io.ReadCloser, error) { + return os.Open(path) + } +} + +// ImageFromPath returns a v1.Image from a tarball located on path. +func ImageFromPath(path string, tag *name.Tag) (v1.Image, error) { + return Image(pathOpener(path), tag) +} + +// LoadManifest load manifest +func LoadManifest(opener Opener) (Manifest, error) { + m, err := extractFileFromTar(opener, "manifest.json") + if err != nil { + return nil, err + } + defer m.Close() + + var manifest Manifest + + if err := json.NewDecoder(m).Decode(&manifest); err != nil { + return nil, err + } + return manifest, nil +} + +// Image exposes an image from the tarball at the provided path. +func Image(opener Opener, tag *name.Tag) (v1.Image, error) { + img := &image{ + opener: opener, + tag: tag, + } + if err := img.loadTarDescriptorAndConfig(); err != nil { + return nil, err + } + + // Peek at the first layer and see if it's compressed. + if len(img.imgDescriptor.Layers) > 0 { + compressed, err := img.areLayersCompressed() + if err != nil { + return nil, err + } + if compressed { + c := compressedImage{ + image: img, + } + return partial.CompressedToImage(&c) + } + } + + uc := uncompressedImage{ + image: img, + } + return partial.UncompressedToImage(&uc) +} + +func (i *image) MediaType() (types.MediaType, error) { + return types.DockerManifestSchema2, nil +} + +// Descriptor stores the manifest data for a single image inside a `docker save` tarball. +type Descriptor struct { + Config string + RepoTags []string + Layers []string + + // Tracks foreign layer info. Key is DiffID. + LayerSources map[v1.Hash]v1.Descriptor `json:",omitempty"` +} + +// Manifest represents the manifests of all images as the `manifest.json` file in a `docker save` tarball. +type Manifest []Descriptor + +func (m Manifest) findDescriptor(tag *name.Tag) (*Descriptor, error) { + if tag == nil { + if len(m) != 1 { + return nil, errors.New("tarball must contain only a single image to be used with tarball.Image") + } + return &(m)[0], nil + } + for _, img := range m { + for _, tagStr := range img.RepoTags { + repoTag, err := name.NewTag(tagStr) + if err != nil { + return nil, err + } + + // Compare the resolved names, since there are several ways to specify the same tag. + if repoTag.Name() == tag.Name() { + return &img, nil + } + } + } + return nil, fmt.Errorf("tag %s not found in tarball", tag) +} + +func (i *image) areLayersCompressed() (bool, error) { + if len(i.imgDescriptor.Layers) == 0 { + return false, errors.New("0 layers found in image") + } + layer := i.imgDescriptor.Layers[0] + blob, err := extractFileFromTar(i.opener, layer) + if err != nil { + return false, err + } + defer blob.Close() + + cp, _, err := comp.PeekCompression(blob) + if err != nil { + return false, err + } + + return cp != compression.None, nil +} + +func (i *image) loadTarDescriptorAndConfig() error { + m, err := extractFileFromTar(i.opener, "manifest.json") + if err != nil { + return err + } + defer m.Close() + + if err := json.NewDecoder(m).Decode(&i.manifest); err != nil { + return err + } + + if i.manifest == nil { + return errors.New("no valid manifest.json in tarball") + } + + i.imgDescriptor, err = i.manifest.findDescriptor(i.tag) + if err != nil { + return err + } + + cfg, err := extractFileFromTar(i.opener, i.imgDescriptor.Config) + if err != nil { + return err + } + defer cfg.Close() + + i.config, err = io.ReadAll(cfg) + if err != nil { + return err + } + return nil +} + +func (i *image) RawConfigFile() ([]byte, error) { + return i.config, nil +} + +// tarFile represents a single file inside a tar. Closing it closes the tar itself. +type tarFile struct { + io.Reader + io.Closer +} + +func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { + f, err := opener() + if err != nil { + return nil, err + } + close := true + defer func() { + if close { + f.Close() + } + }() + + tf := tar.NewReader(f) + for { + hdr, err := tf.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + if hdr.Name == filePath { + if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink { + currentDir := filepath.Dir(filePath) + return extractFileFromTar(opener, path.Join(currentDir, path.Clean(hdr.Linkname))) + } + close = false + return tarFile{ + Reader: tf, + Closer: f, + }, nil + } + } + return nil, fmt.Errorf("file %s not found in tar", filePath) +} + +// uncompressedLayerFromTarball implements partial.UncompressedLayer +type uncompressedLayerFromTarball struct { + diffID v1.Hash + mediaType types.MediaType + opener Opener + filePath string +} + +// foreignUncompressedLayer implements partial.UncompressedLayer but returns +// a custom descriptor. This allows the foreign layer URLs to be included in +// the generated image manifest for uncompressed layers. +type foreignUncompressedLayer struct { + uncompressedLayerFromTarball + desc v1.Descriptor +} + +func (fl *foreignUncompressedLayer) Descriptor() (*v1.Descriptor, error) { + return &fl.desc, nil +} + +// DiffID implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) DiffID() (v1.Hash, error) { + return ulft.diffID, nil +} + +// Uncompressed implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) { + return extractFileFromTar(ulft.opener, ulft.filePath) +} + +func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) { + return ulft.mediaType, nil +} + +func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { + cfg, err := partial.ConfigFile(i) + if err != nil { + return nil, err + } + for idx, diffID := range cfg.RootFS.DiffIDs { + if diffID == h { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + mt := types.DockerLayer + if bd, ok := i.imgDescriptor.LayerSources[h]; ok { + // Overwrite the mediaType for foreign layers. + return &foreignUncompressedLayer{ + uncompressedLayerFromTarball: uncompressedLayerFromTarball{ + diffID: diffID, + mediaType: bd.MediaType, + opener: i.opener, + filePath: i.imgDescriptor.Layers[idx], + }, + desc: bd, + }, nil + } + return &uncompressedLayerFromTarball{ + diffID: diffID, + mediaType: mt, + opener: i.opener, + filePath: i.imgDescriptor.Layers[idx], + }, nil + } + } + return nil, fmt.Errorf("diff id %q not found", h) +} + +func (c *compressedImage) Manifest() (*v1.Manifest, error) { + c.manifestLock.Lock() + defer c.manifestLock.Unlock() + if c.manifest != nil { + return c.manifest, nil + } + + b, err := c.RawConfigFile() + if err != nil { + return nil, err + } + + cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + c.manifest = &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.DockerManifestSchema2, + Config: v1.Descriptor{ + MediaType: types.DockerConfigJSON, + Size: cfgSize, + Digest: cfgHash, + }, + } + + for i, p := range c.imgDescriptor.Layers { + cfg, err := partial.ConfigFile(c) + if err != nil { + return nil, err + } + diffid := cfg.RootFS.DiffIDs[i] + if d, ok := c.imgDescriptor.LayerSources[diffid]; ok { + // If it's a foreign layer, just append the descriptor so we can avoid + // reading the entire file. + c.manifest.Layers = append(c.manifest.Layers, d) + } else { + l, err := extractFileFromTar(c.opener, p) + if err != nil { + return nil, err + } + defer l.Close() + sha, size, err := v1.SHA256(l) + if err != nil { + return nil, err + } + c.manifest.Layers = append(c.manifest.Layers, v1.Descriptor{ + MediaType: types.DockerLayer, + Size: size, + Digest: sha, + }) + } + } + return c.manifest, nil +} + +func (c *compressedImage) RawManifest() ([]byte, error) { + return partial.RawManifest(c) +} + +// compressedLayerFromTarball implements partial.CompressedLayer +type compressedLayerFromTarball struct { + desc v1.Descriptor + opener Opener + filePath string +} + +// Digest implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Digest() (v1.Hash, error) { + return clft.desc.Digest, nil +} + +// Compressed implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { + return extractFileFromTar(clft.opener, clft.filePath) +} + +// MediaType implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) { + return clft.desc.MediaType, nil +} + +// Size implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Size() (int64, error) { + return clft.desc.Size, nil +} + +func (c *compressedImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + m, err := c.Manifest() + if err != nil { + return nil, err + } + for i, l := range m.Layers { + if l.Digest == h { + fp := c.imgDescriptor.Layers[i] + return &compressedLayerFromTarball{ + desc: l, + opener: c.opener, + filePath: fp, + }, nil + } + } + return nil, fmt.Errorf("blob %v not found", h) +} diff --git a/pkg/v1/tarball/image_test.go b/pkg/v1/tarball/image_test.go new file mode 100644 index 0000000..3a46400 --- /dev/null +++ b/pkg/v1/tarball/image_test.go @@ -0,0 +1,139 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "io" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestManifestAndConfig(t *testing.T) { + img, err := ImageFromPath("testdata/test_image_1.tar", nil) + if err != nil { + t.Fatalf("Error loading image: %v", err) + } + manifest, err := img.Manifest() + if err != nil { + t.Fatalf("Error loading manifest: %v", err) + } + if len(manifest.Layers) != 1 { + t.Fatalf("layers should be 1, got %d", len(manifest.Layers)) + } + + config, err := img.ConfigFile() + if err != nil { + t.Fatalf("Error loading config file: %v", err) + } + if len(config.History) != 1 { + t.Fatalf("history length should be 1, got %d", len(config.History)) + } + + if err := validate.Image(img); err != nil { + t.Errorf("Validate() = %v", err) + } +} + +func TestNullManifest(t *testing.T) { + img, err := ImageFromPath("testdata/null_manifest.tar", nil) + if err == nil { + t.Fatalf("Error expected loading null image: %v", img) + } +} + +func TestNoManifest(t *testing.T) { + img, err := ImageFromPath("testdata/no_manifest.tar", nil) + if err == nil { + t.Fatalf("Error expected loading image: %v", img) + } +} + +func TestBundleSingle(t *testing.T) { + img, err := ImageFromPath("testdata/test_bundle.tar", nil) + if err == nil { + t.Fatalf("Error expected loading image: %v", img) + } +} + +func TestBundleMultiple(t *testing.T) { + for _, imgName := range []string{ + "test_image_1", + "test_image_2", + "test_image_1:latest", + "test_image_2:latest", + "index.docker.io/library/test_image_1:latest", + } { + t.Run(imgName, func(t *testing.T) { + tag, err := name.NewTag(imgName, name.WeakValidation) + if err != nil { + t.Fatalf("Error creating tag: %v", err) + } + img, err := ImageFromPath("testdata/test_bundle.tar", &tag) + if err != nil { + t.Fatalf("Error loading image: %v", err) + } + if _, err := img.Manifest(); err != nil { + t.Fatalf("Unexpected error loading manifest: %v", err) + } + + if err := validate.Image(img); err != nil { + t.Errorf("Validate() = %v", err) + } + }) + } +} + +func TestLayerLink(t *testing.T) { + tag, err := name.NewTag("bazel/v1/tarball:test_image_3", name.WeakValidation) + if err != nil { + t.Fatalf("Error creating tag: %v", err) + } + img, err := ImageFromPath("testdata/test_link.tar", &tag) + if err != nil { + t.Fatalf("Error loading image: %v", img) + } + hash := v1.Hash{ + Algorithm: "sha256", + Hex: "8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17", + } + layer, err := img.LayerByDiffID(hash) + if err != nil { + t.Fatalf("Error getting layer by diff ID: %v, %v", hash, err) + } + rc, err := layer.Uncompressed() + if err != nil { + t.Fatal(err) + } + bs, err := io.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + if len(bs) == 0 { + t.Errorf("layer.Uncompressed() returned a link file") + } +} + +func TestLoadManifest(t *testing.T) { + manifest, err := LoadManifest(pathOpener("testdata/test_load_manifest.tar")) + if err != nil { + t.Fatalf("Error load manifest: %v", err) + } + if len(manifest) == 0 { + t.Fatalf("get nothing") + } +} diff --git a/pkg/v1/tarball/layer.go b/pkg/v1/tarball/layer.go new file mode 100644 index 0000000..a344e92 --- /dev/null +++ b/pkg/v1/tarball/layer.go @@ -0,0 +1,349 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "os" + "sync" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/google/go-containerregistry/internal/and" + comp "github.com/google/go-containerregistry/internal/compression" + gestargz "github.com/google/go-containerregistry/internal/estargz" + ggzip "github.com/google/go-containerregistry/internal/gzip" + "github.com/google/go-containerregistry/internal/zstd" + "github.com/google/go-containerregistry/pkg/compression" + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + compressedopener Opener + uncompressedopener Opener + compression compression.Compression + compressionLevel int + annotations map[string]string + estgzopts []estargz.Option + mediaType types.MediaType +} + +// Descriptor implements partial.withDescriptor. +func (l *layer) Descriptor() (*v1.Descriptor, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + return &v1.Descriptor{ + Size: l.size, + Digest: digest, + Annotations: l.annotations, + MediaType: l.mediaType, + }, nil +} + +// Digest implements v1.Layer +func (l *layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +// DiffID implements v1.Layer +func (l *layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +// Compressed implements v1.Layer +func (l *layer) Compressed() (io.ReadCloser, error) { + return l.compressedopener() +} + +// Uncompressed implements v1.Layer +func (l *layer) Uncompressed() (io.ReadCloser, error) { + return l.uncompressedopener() +} + +// Size implements v1.Layer +func (l *layer) Size() (int64, error) { + return l.size, nil +} + +// MediaType implements v1.Layer +func (l *layer) MediaType() (types.MediaType, error) { + return l.mediaType, nil +} + +// LayerOption applies options to layer +type LayerOption func(*layer) + +// WithCompression is a functional option for overriding the default +// compression algorithm used for compressing uncompressed tarballs. +// Please note that WithCompression(compression.ZStd) should be used +// in conjunction with WithMediaType(types.OCILayerZStd) +func WithCompression(comp compression.Compression) LayerOption { + return func(l *layer) { + switch comp { + case compression.ZStd: + l.compression = compression.ZStd + case compression.GZip: + l.compression = compression.GZip + case compression.None: + logs.Warn.Printf("Compression type 'none' is not supported for tarball layers; using gzip compression.") + l.compression = compression.GZip + default: + logs.Warn.Printf("Unexpected compression type for WithCompression(): %s; using gzip compression instead.", comp) + l.compression = compression.GZip + } + } +} + +// WithCompressionLevel is a functional option for overriding the default +// compression level used for compressing uncompressed tarballs. +func WithCompressionLevel(level int) LayerOption { + return func(l *layer) { + l.compressionLevel = level + } +} + +// WithMediaType is a functional option for overriding the layer's media type. +func WithMediaType(mt types.MediaType) LayerOption { + return func(l *layer) { + l.mediaType = mt + } +} + +// WithCompressedCaching is a functional option that overrides the +// logic for accessing the compressed bytes to memoize the result +// and avoid expensive repeated gzips. +func WithCompressedCaching(l *layer) { + var once sync.Once + var err error + + buf := bytes.NewBuffer(nil) + og := l.compressedopener + + l.compressedopener = func() (io.ReadCloser, error) { + once.Do(func() { + var rc io.ReadCloser + rc, err = og() + if err == nil { + defer rc.Close() + _, err = io.Copy(buf, rc) + } + }) + if err != nil { + return nil, err + } + + return io.NopCloser(bytes.NewBuffer(buf.Bytes())), nil + } +} + +// WithEstargzOptions is a functional option that allow the caller to pass +// through estargz.Options to the underlying compression layer. This is +// only meaningful when estargz is enabled. +func WithEstargzOptions(opts ...estargz.Option) LayerOption { + return func(l *layer) { + l.estgzopts = opts + } +} + +// WithEstargz is a functional option that explicitly enables estargz support. +func WithEstargz(l *layer) { + oguncompressed := l.uncompressedopener + estargz := func() (io.ReadCloser, error) { + crc, err := oguncompressed() + if err != nil { + return nil, err + } + eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compressionLevel)) + rc, h, err := gestargz.ReadCloser(crc, eopts...) + if err != nil { + return nil, err + } + l.annotations[estargz.TOCJSONDigestAnnotation] = h.String() + return &and.ReadCloser{ + Reader: rc, + CloseFunc: func() error { + err := rc.Close() + if err != nil { + return err + } + // As an optimization, leverage the DiffID exposed by the estargz ReadCloser + l.diffID, err = v1.NewHash(rc.DiffID().String()) + return err + }, + }, nil + } + uncompressed := func() (io.ReadCloser, error) { + urc, err := estargz() + if err != nil { + return nil, err + } + return ggzip.UnzipReadCloser(urc) + } + + l.compressedopener = estargz + l.uncompressedopener = uncompressed +} + +// LayerFromFile returns a v1.Layer given a tarball +func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) { + opener := func() (io.ReadCloser, error) { + return os.Open(path) + } + return LayerFromOpener(opener, opts...) +} + +// LayerFromOpener returns a v1.Layer given an Opener function. +// The Opener may return either an uncompressed tarball (common), +// or a compressed tarball (uncommon). +// +// When using this in conjunction with something like remote.Write +// the uncompressed path may end up gzipping things multiple times: +// 1. Compute the layer SHA256 +// 2. Upload the compressed layer. +// +// Since gzip can be expensive, we support an option to memoize the +// compression that can be passed here: tarball.WithCompressedCaching +func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { + comp, err := comp.GetCompression(opener) + if err != nil { + return nil, err + } + + layer := &layer{ + compression: compression.GZip, + compressionLevel: gzip.BestSpeed, + annotations: make(map[string]string, 1), + mediaType: types.DockerLayer, + } + + if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" { + opts = append([]LayerOption{WithEstargz}, opts...) + } + + switch comp { + case compression.GZip: + layer.compressedopener = opener + layer.uncompressedopener = func() (io.ReadCloser, error) { + urc, err := opener() + if err != nil { + return nil, err + } + return ggzip.UnzipReadCloser(urc) + } + case compression.ZStd: + layer.compressedopener = opener + layer.uncompressedopener = func() (io.ReadCloser, error) { + urc, err := opener() + if err != nil { + return nil, err + } + return zstd.UnzipReadCloser(urc) + } + default: + layer.uncompressedopener = opener + layer.compressedopener = func() (io.ReadCloser, error) { + crc, err := opener() + if err != nil { + return nil, err + } + + if layer.compression == compression.ZStd { + return zstd.ReadCloserLevel(crc, layer.compressionLevel), nil + } + + return ggzip.ReadCloserLevel(crc, layer.compressionLevel), nil + } + } + + for _, opt := range opts { + opt(layer) + } + + // Warn if media type does not match compression + var mediaTypeMismatch = false + switch layer.compression { + case compression.GZip: + mediaTypeMismatch = + layer.mediaType != types.OCILayer && + layer.mediaType != types.OCIRestrictedLayer && + layer.mediaType != types.DockerLayer + + case compression.ZStd: + mediaTypeMismatch = layer.mediaType != types.OCILayerZStd + } + + if mediaTypeMismatch { + logs.Warn.Printf("Unexpected mediaType (%s) for selected compression in %s in LayerFromOpener().", layer.mediaType, layer.compression) + } + + if layer.digest, layer.size, err = computeDigest(layer.compressedopener); err != nil { + return nil, err + } + + empty := v1.Hash{} + if layer.diffID == empty { + if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil { + return nil, err + } + } + + return layer, nil +} + +// LayerFromReader returns a v1.Layer given a io.Reader. +// +// The reader's contents are read and buffered to a temp file in the process. +// +// Deprecated: Use LayerFromOpener or stream.NewLayer instead, if possible. +func LayerFromReader(reader io.Reader, opts ...LayerOption) (v1.Layer, error) { + tmp, err := os.CreateTemp("", "") + if err != nil { + return nil, fmt.Errorf("creating temp file to buffer reader: %w", err) + } + if _, err := io.Copy(tmp, reader); err != nil { + return nil, fmt.Errorf("writing temp file to buffer reader: %w", err) + } + return LayerFromFile(tmp.Name(), opts...) +} + +func computeDigest(opener Opener) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + + return v1.SHA256(rc) +} + +func computeDiffID(opener Opener) (v1.Hash, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, err + } + defer rc.Close() + + digest, _, err := v1.SHA256(rc) + return digest, err +} diff --git a/pkg/v1/tarball/layer_test.go b/pkg/v1/tarball/layer_test.go new file mode 100644 index 0000000..5d93360 --- /dev/null +++ b/pkg/v1/tarball/layer_test.go @@ -0,0 +1,381 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "bytes" + "compress/gzip" + "io" + "os" + "testing" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/compression" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestLayerFromFile(t *testing.T) { + setupFixtures(t) + defer teardownFixtures(t) + + tarLayer, err := LayerFromFile("testdata/content.tar") + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + tarGzLayer, err := LayerFromFile("gzip_content.tgz") + if err != nil { + t.Fatalf("Unable to create layer from compressed tar file: %v", err) + } + + tarZstdLayer, err := LayerFromFile("zstd_content.tar.zst") + if err != nil { + t.Fatalf("Unable to create layer from compressed tar file: %v", err) + } + + if err := compare.Layers(tarLayer, tarGzLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } + + if err := compare.Layers(tarLayer, tarZstdLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } + + if err := validate.Layer(tarLayer); err != nil { + t.Errorf("validate.Layer(tarLayer): %v", err) + } + + if err := validate.Layer(tarGzLayer); err != nil { + t.Errorf("validate.Layer(tarGzLayer): %v", err) + } + + if err := validate.Layer(tarZstdLayer); err != nil { + t.Errorf("validate.Layer(tarZstdLayer): %v", err) + } + + getTestDigest := func(testName string, opts ...LayerOption) v1.Hash { + layer, err := LayerFromFile("testdata/content.tar", opts...) + if err != nil { + t.Fatalf("Unable to create layer with '%s' compression from tar file: %v", testName, err) + } + + digest, err := layer.Digest() + if err != nil { + t.Fatalf("Unable to generate digest with '%s' compression: %v", testName, err) + } + + return digest + } + + defaultDigest := getTestDigest("Gzip Default", WithCompressionLevel(gzip.DefaultCompression)) + speedDigest := getTestDigest("Gzip BestSpeed", WithCompressionLevel(gzip.BestSpeed)) + zstdDigest := getTestDigest("Zstd Default", WithCompression(compression.ZStd)) + zstdDigest1 := getTestDigest("Zstd BestSpeed", WithCompression(compression.ZStd), WithCompressionLevel(1)) + + if defaultDigest.String() == speedDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if defaultDigest.String() == zstdDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if defaultDigest.String() == zstdDigest1.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } +} + +func TestLayerFromFileEstargz(t *testing.T) { + setupFixtures(t) + defer teardownFixtures(t) + + tarLayer, err := LayerFromFile("testdata/content.tar", WithEstargz) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := validate.Layer(tarLayer); err != nil { + t.Errorf("validate.Layer(tarLayer): %v", err) + } + + tarLayerDefaultCompression, err := LayerFromFile("testdata/content.tar", WithEstargz, WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + t.Fatalf("Unable to create layer with 'Default' compression from tar file: %v", err) + } + descriptorDefaultCompression, err := tarLayerDefaultCompression.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorDefaultCompression.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorDefaultCompression.Annotations) + } + + defaultDigest, err := tarLayerDefaultCompression.Digest() + if err != nil { + t.Fatal("Unable to generate digest with 'Default' compression", err) + } + + tarLayerSpeedCompression, err := LayerFromFile("testdata/content.tar", WithEstargz, WithCompressionLevel(gzip.BestSpeed)) + if err != nil { + t.Fatalf("Unable to create layer with 'BestSpeed' compression from tar file: %v", err) + } + descriptorSpeedCompression, err := tarLayerSpeedCompression.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorSpeedCompression.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorSpeedCompression.Annotations) + } + + speedDigest, err := tarLayerSpeedCompression.Digest() + if err != nil { + t.Fatal("Unable to generate digest with 'BestSpeed' compression", err) + } + + if defaultDigest.String() == speedDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation] { + t.Errorf("wanted different toc digests got default: %s, speed: %s", + descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation], + descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation]) + } + + tarLayerPrioritizedFiles, err := LayerFromFile("testdata/content.tar", + WithEstargz, + // We compare with default, so pass for apples-to-apples comparison. + WithCompressionLevel(gzip.DefaultCompression), + // By passing a list of priority files, we expect the layer to be different. + WithEstargzOptions(estargz.WithPrioritizedFiles([]string{ + "./bat", + }))) + if err != nil { + t.Fatalf("Unable to create layer with prioritized files from tar file: %v", err) + } + descriptorPrioritizedFiles, err := tarLayerPrioritizedFiles.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorPrioritizedFiles.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorPrioritizedFiles.Annotations) + } + + prioritizedDigest, err := tarLayerPrioritizedFiles.Digest() + if err != nil { + t.Fatal("Unable to generate digest with prioritized files", err) + } + + if defaultDigest.String() == prioritizedDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation] { + t.Errorf("wanted different toc digests got default: %s, prioritized: %s", + descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation], + descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation]) + } +} + +func TestLayerFromOpenerReader(t *testing.T) { + setupFixtures(t) + defer teardownFixtures(t) + + ucBytes, err := os.ReadFile("testdata/content.tar") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + count := 0 + ucOpener := func() (io.ReadCloser, error) { + count++ + return io.NopCloser(bytes.NewReader(ucBytes)), nil + } + tarLayer, err := LayerFromOpener(ucOpener, WithCompressedCaching) + if err != nil { + t.Fatal("Unable to create layer from tar file:", err) + } + for i := 0; i < 10; i++ { + tarLayer.Compressed() + } + + // Store the count and reset the counter. + cachedCount := count + count = 0 + + tarLayer, err = LayerFromOpener(ucOpener) + if err != nil { + t.Fatal("Unable to create layer from tar file:", err) + } + for i := 0; i < 10; i++ { + tarLayer.Compressed() + } + + // We expect three calls: gzip sniff, diffid computation, cached compression + if cachedCount != 3 { + t.Errorf("cached count = %d, wanted %d", cachedCount, 3) + } + if cachedCount+10 != count { + t.Errorf("count = %d, wanted %d", count, cachedCount+10) + } + + gzBytes, err := os.ReadFile("gzip_content.tgz") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + gzOpener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(gzBytes)), nil + } + tarGzLayer, err := LayerFromOpener(gzOpener) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := compare.Layers(tarLayer, tarGzLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } + + zstdBytes, err := os.ReadFile("zstd_content.tar.zst") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + zstdOpener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(zstdBytes)), nil + } + tarZstdLayer, err := LayerFromOpener(zstdOpener) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := compare.Layers(tarLayer, tarZstdLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } +} + +func TestWithMediaType(t *testing.T) { + setupFixtures(t) + defer teardownFixtures(t) + + l, err := LayerFromFile("testdata/content.tar") + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + got, err := l.MediaType() + if err != nil { + t.Fatalf("MediaType: %v", err) + } + if want := types.DockerLayer; got != want { + t.Errorf("got %v, want %v", got, want) + } + + l, err = LayerFromFile("testdata/content.tar", WithMediaType(types.OCILayer)) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + got, err = l.MediaType() + if err != nil { + t.Fatalf("MediaType: %v", err) + } + if want := types.OCILayer; got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestLayerFromReader(t *testing.T) { + setupFixtures(t) + defer teardownFixtures(t) + + ucBytes, err := os.ReadFile("testdata/content.tar") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + tarLayer, err := LayerFromReader(bytes.NewReader(ucBytes)) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + gzBytes, err := os.ReadFile("gzip_content.tgz") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + tarGzLayer, err := LayerFromReader(bytes.NewReader(gzBytes)) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := compare.Layers(tarLayer, tarGzLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } + + zstdBytes, err := os.ReadFile("zstd_content.tar.zst") + if err != nil { + t.Fatalf("Unable to read tar file: %v", err) + } + tarZstdLayer, err := LayerFromReader(bytes.NewReader(zstdBytes)) + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := compare.Layers(tarLayer, tarZstdLayer); err != nil { + t.Errorf("compare.Layers: %v", err) + } +} + +// Compression settings matter in order for the digest, size, +// compressed assertions to pass +// +// Since our gzip.GzipReadCloser uses gzip.BestSpeed +// we need our fixture to use the same - bazel's pkg_tar doesn't +// seem to let you control compression settings +func setupFixtures(t *testing.T) { + t.Helper() + + setupCompressedTar(t, "gzip_content.tgz") + setupCompressedTar(t, "zstd_content.tar.zst") +} + +func setupCompressedTar(t *testing.T, fileName string) { + t.Helper() + in, err := os.Open("testdata/content.tar") + if err != nil { + t.Errorf("Error setting up fixtures: %v", err) + } + + defer in.Close() + + out, err := os.Create(fileName) + if err != nil { + t.Errorf("Error setting up fixtures: %v", err) + } + + defer out.Close() + + gw, _ := gzip.NewWriterLevel(out, gzip.BestSpeed) + defer gw.Close() + + _, err = io.Copy(gw, in) + if err != nil { + t.Errorf("Error setting up fixtures: %v", err) + } +} + +func teardownFixtures(t *testing.T) { + t.Helper() + if err := os.Remove("gzip_content.tgz"); err != nil { + t.Errorf("Error tearing down fixtures: %v", err) + } + if err := os.Remove("zstd_content.tar.zst"); err != nil { + t.Errorf("Error tearing down fixtures: %v", err) + } +} diff --git a/pkg/v1/tarball/progress_test.go b/pkg/v1/tarball/progress_test.go new file mode 100644 index 0000000..c722749 --- /dev/null +++ b/pkg/v1/tarball/progress_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball_test + +import ( + "errors" + "fmt" + "io" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func ExampleWithProgress() { + // buffered channel to make the example test easier + c := make(chan v1.Update, 200) + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + fmt.Printf("error creating temp file: %v\n", err) + return + } + defer fp.Close() + defer os.Remove(fp.Name()) + + img, err := tarball.ImageFromPath("testdata/test_image_1.tar", nil) + go func() { + _ = tarball.WriteToFile(fp.Name(), nil, img, tarball.WithProgress(c)) + }() + for update := range c { + switch { + case update.Error != nil && errors.Is(update.Error, io.EOF): + fmt.Fprintf(os.Stderr, "receive error message: %v\n", err) + fmt.Printf("%d/%d", update.Complete, update.Total) + // Output: 4096/4096 + return + case update.Error != nil: + fmt.Printf("error writing tarball: %v\n", update.Error) + return + default: + fmt.Fprintf(os.Stderr, "receive update: %#v\n", update) + } + } +} diff --git a/pkg/v1/tarball/testdata/bar b/pkg/v1/tarball/testdata/bar new file mode 100644 index 0000000..5716ca5 --- /dev/null +++ b/pkg/v1/tarball/testdata/bar @@ -0,0 +1 @@ +bar diff --git a/pkg/v1/tarball/testdata/bat/bat b/pkg/v1/tarball/testdata/bat/bat new file mode 100644 index 0000000..1054901 --- /dev/null +++ b/pkg/v1/tarball/testdata/bat/bat @@ -0,0 +1 @@ +bat diff --git a/pkg/v1/tarball/testdata/baz b/pkg/v1/tarball/testdata/baz new file mode 100644 index 0000000..7601807 --- /dev/null +++ b/pkg/v1/tarball/testdata/baz @@ -0,0 +1 @@ +baz diff --git a/pkg/v1/tarball/testdata/content.tar b/pkg/v1/tarball/testdata/content.tar new file mode 100755 index 0000000..55f4d1d Binary files /dev/null and b/pkg/v1/tarball/testdata/content.tar differ diff --git a/pkg/v1/tarball/testdata/foo b/pkg/v1/tarball/testdata/foo new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/pkg/v1/tarball/testdata/foo @@ -0,0 +1 @@ +foo diff --git a/pkg/v1/tarball/testdata/no_manifest.tar b/pkg/v1/tarball/testdata/no_manifest.tar new file mode 100644 index 0000000..319db1d Binary files /dev/null and b/pkg/v1/tarball/testdata/no_manifest.tar differ diff --git a/pkg/v1/tarball/testdata/null_manifest.tar b/pkg/v1/tarball/testdata/null_manifest.tar new file mode 100644 index 0000000..2a65fcd Binary files /dev/null and b/pkg/v1/tarball/testdata/null_manifest.tar differ diff --git a/pkg/v1/tarball/testdata/test_bundle.tar b/pkg/v1/tarball/testdata/test_bundle.tar new file mode 100755 index 0000000..1ad0f79 Binary files /dev/null and b/pkg/v1/tarball/testdata/test_bundle.tar differ diff --git a/pkg/v1/tarball/testdata/test_image_1.tar b/pkg/v1/tarball/testdata/test_image_1.tar new file mode 100755 index 0000000..0fe1a21 Binary files /dev/null and b/pkg/v1/tarball/testdata/test_image_1.tar differ diff --git a/pkg/v1/tarball/testdata/test_image_2.tar b/pkg/v1/tarball/testdata/test_image_2.tar new file mode 100755 index 0000000..bdfe0ef Binary files /dev/null and b/pkg/v1/tarball/testdata/test_image_2.tar differ diff --git a/pkg/v1/tarball/testdata/test_link.tar b/pkg/v1/tarball/testdata/test_link.tar new file mode 100644 index 0000000..e83064f Binary files /dev/null and b/pkg/v1/tarball/testdata/test_link.tar differ diff --git a/pkg/v1/tarball/testdata/test_load_manifest.tar b/pkg/v1/tarball/testdata/test_load_manifest.tar new file mode 100644 index 0000000..0fe1a21 Binary files /dev/null and b/pkg/v1/tarball/testdata/test_load_manifest.tar differ diff --git a/pkg/v1/tarball/write.go b/pkg/v1/tarball/write.go new file mode 100644 index 0000000..e607df1 --- /dev/null +++ b/pkg/v1/tarball/write.go @@ -0,0 +1,457 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +// WriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.Write with a new file. +func WriteToFile(p string, ref name.Reference, img v1.Image, opts ...WriteOption) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return Write(ref, img, w, opts...) +} + +// MultiWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiWrite with a new file. +func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image, opts ...WriteOption) error { + refToImage := make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWriteToFile(p, refToImage, opts...) +} + +// MultiRefWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file. +func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image, opts ...WriteOption) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return MultiRefWrite(refToImage, w, opts...) +} + +// Write is a wrapper to write a single image and tag to a tarball. +func Write(ref name.Reference, img v1.Image, w io.Writer, opts ...WriteOption) error { + return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w, opts...) +} + +// MultiWrite writes the contents of each image to the provided writer, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer, opts ...WriteOption) error { + refToImage := make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWrite(refToImage, w, opts...) +} + +// MultiRefWrite writes the contents of each image to the provided writer, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer, opts ...WriteOption) error { + // process options + o := &writeOptions{ + updates: nil, + } + for _, option := range opts { + if err := option(o); err != nil { + return err + } + } + + imageToTags := dedupRefToImage(refToImage) + size, mBytes, err := getSizeAndManifest(imageToTags) + if err != nil { + return sendUpdateReturn(o, err) + } + + return writeImagesToTar(imageToTags, mBytes, size, w, o) +} + +// sendUpdateReturn return the passed in error message, also sending on update channel, if it exists +func sendUpdateReturn(o *writeOptions, err error) error { + if o != nil && o.updates != nil { + o.updates <- v1.Update{ + Error: err, + } + } + return err +} + +// sendProgressWriterReturn return the passed in error message, also sending on update channel, if it exists, along with downloaded information +func sendProgressWriterReturn(pw *progressWriter, err error) error { + if pw != nil { + return pw.Error(err) + } + return err +} + +// writeImagesToTar writes the images to the tarball +func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w io.Writer, o *writeOptions) (err error) { + if w == nil { + return sendUpdateReturn(o, errors.New("must pass valid writer")) + } + + tw := w + var pw *progressWriter + + // we only calculate the sizes and use a progressWriter if we were provided + // an option with a progress channel + if o != nil && o.updates != nil { + pw = &progressWriter{ + w: w, + updates: o.updates, + size: size, + } + tw = pw + } + + tf := tar.NewWriter(tw) + defer tf.Close() + + seenLayerDigests := make(map[string]struct{}) + + for img := range imageToTags { + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil { + return sendProgressWriterReturn(pw, err) + } + + // Write the layers. + layers, err := img.Layers() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + layerFiles := make([]string, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + // Munge the file name to appease ancient technology. + // + // tar assumes anything with a colon is a remote tape drive: + // https://www.gnu.org/software/tar/manual/html_section/tar_45.html + // Drop the algorithm prefix, e.g. "sha256:" + hex := d.Hex + + // gunzip expects certain file extensions: + // https://www.gnu.org/software/gzip/manual/html_node/Overview.html + layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + + if _, ok := seenLayerDigests[hex]; ok { + continue + } + seenLayerDigests[hex] = struct{}{} + + r, err := l.Compressed() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + blobSize, err := l.Size() + if err != nil { + return sendProgressWriterReturn(pw, err) + } + + if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil { + return sendProgressWriterReturn(pw, err) + } + } + } + if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(m), int64(len(m))); err != nil { + return sendProgressWriterReturn(pw, err) + } + + // be sure to close the tar writer so everything is flushed out before we send our EOF + if err := tf.Close(); err != nil { + return sendProgressWriterReturn(pw, err) + } + // send an EOF to indicate finished on the channel, but nil as our return error + _ = sendProgressWriterReturn(pw, io.EOF) + return nil +} + +// calculateManifest calculates the manifest and optionally the size of the tar file +func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error) { + if len(imageToTags) == 0 { + return nil, errors.New("set of images is empty") + } + + for img, tags := range imageToTags { + cfgName, err := img.ConfigName() + if err != nil { + return nil, err + } + + // Store foreign layer info. + layerSources := make(map[v1.Hash]v1.Descriptor) + + // Write the layers. + layers, err := img.Layers() + if err != nil { + return nil, err + } + layerFiles := make([]string, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + return nil, err + } + // Munge the file name to appease ancient technology. + // + // tar assumes anything with a colon is a remote tape drive: + // https://www.gnu.org/software/tar/manual/html_section/tar_45.html + // Drop the algorithm prefix, e.g. "sha256:" + hex := d.Hex + + // gunzip expects certain file extensions: + // https://www.gnu.org/software/gzip/manual/html_node/Overview.html + layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + + // Add to LayerSources if it's a foreign layer. + desc, err := partial.BlobDescriptor(img, d) + if err != nil { + return nil, err + } + if !desc.MediaType.IsDistributable() { + diffid, err := partial.BlobToDiffID(img, d) + if err != nil { + return nil, err + } + layerSources[diffid] = *desc + } + } + + // Generate the tar descriptor and write it. + m = append(m, Descriptor{ + Config: cfgName.String(), + RepoTags: tags, + Layers: layerFiles, + LayerSources: layerSources, + }) + } + // sort by name of the repotags so it is consistent. Alternatively, we could sort by hash of the + // descriptor, but that would make it hard for humans to process + sort.Slice(m, func(i, j int) bool { + return strings.Join(m[i].RepoTags, ",") < strings.Join(m[j].RepoTags, ",") + }) + + return m, nil +} + +// CalculateSize calculates the expected complete size of the output tar file +func CalculateSize(refToImage map[name.Reference]v1.Image) (size int64, err error) { + imageToTags := dedupRefToImage(refToImage) + size, _, err = getSizeAndManifest(imageToTags) + return size, err +} + +func getSizeAndManifest(imageToTags map[v1.Image][]string) (int64, []byte, error) { + m, err := calculateManifest(imageToTags) + if err != nil { + return 0, nil, fmt.Errorf("unable to calculate manifest: %w", err) + } + mBytes, err := json.Marshal(m) + if err != nil { + return 0, nil, fmt.Errorf("could not marshall manifest to bytes: %w", err) + } + + size, err := calculateTarballSize(imageToTags, mBytes) + if err != nil { + return 0, nil, fmt.Errorf("error calculating tarball size: %w", err) + } + return size, mBytes, nil +} + +// calculateTarballSize calculates the size of the tar file +func calculateTarballSize(imageToTags map[v1.Image][]string, mBytes []byte) (size int64, err error) { + seenLayerDigests := make(map[string]struct{}) + for img, name := range imageToTags { + manifest, err := img.Manifest() + if err != nil { + return size, fmt.Errorf("unable to get manifest for img %s: %w", name, err) + } + size += calculateSingleFileInTarSize(manifest.Config.Size) + for _, l := range manifest.Layers { + hex := l.Digest.Hex + if _, ok := seenLayerDigests[hex]; ok { + continue + } + seenLayerDigests[hex] = struct{}{} + size += calculateSingleFileInTarSize(l.Size) + } + } + // add the manifest + size += calculateSingleFileInTarSize(int64(len(mBytes))) + + // add the two padding blocks that indicate end of a tar file + size += 1024 + return size, nil +} + +func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string { + imageToTags := make(map[v1.Image][]string) + + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; !ok || tags == nil { + imageToTags[img] = []string{} + } + // Docker cannot load tarballs without an explicit tag: + // https://github.com/google/go-containerregistry/issues/890 + // + // We can't use the fully qualified tag.Name() because of rules_docker: + // https://github.com/google/go-containerregistry/issues/527 + // + // If the tag is "latest", but tag.String() doesn't end in ":latest", + // just append it. Kind of gross, but should work for now. + ts := tag.String() + if tag.Identifier() == name.DefaultTag && !strings.HasSuffix(ts, ":"+name.DefaultTag) { + ts = fmt.Sprintf("%s:%s", ts, name.DefaultTag) + } + imageToTags[img] = append(imageToTags[img], ts) + } else if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } + } + + return imageToTags +} + +// writeTarEntry writes a file to the provided writer with a corresponding tar header +func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error { + hdr := &tar.Header{ + Mode: 0644, + Typeflag: tar.TypeReg, + Size: size, + Name: path, + } + if err := tf.WriteHeader(hdr); err != nil { + return err + } + _, err := io.Copy(tf, r) + return err +} + +// ComputeManifest get the manifest.json that will be written to the tarball +// for multiple references +func ComputeManifest(refToImage map[name.Reference]v1.Image) (Manifest, error) { + imageToTags := dedupRefToImage(refToImage) + return calculateManifest(imageToTags) +} + +// WriteOption a function option to pass to Write() +type WriteOption func(*writeOptions) error +type writeOptions struct { + updates chan<- v1.Update +} + +// WithProgress create a WriteOption for passing to Write() that enables +// a channel to receive updates as they are downloaded and written to disk. +func WithProgress(updates chan<- v1.Update) WriteOption { + return func(o *writeOptions) error { + o.updates = updates + return nil + } +} + +// progressWriter is a writer which will send the download progress +type progressWriter struct { + w io.Writer + updates chan<- v1.Update + size, complete int64 +} + +func (pw *progressWriter) Write(p []byte) (int, error) { + n, err := pw.w.Write(p) + if err != nil { + return n, err + } + + pw.complete += int64(n) + + pw.updates <- v1.Update{ + Total: pw.size, + Complete: pw.complete, + } + + return n, err +} + +func (pw *progressWriter) Error(err error) error { + pw.updates <- v1.Update{ + Total: pw.size, + Complete: pw.complete, + Error: err, + } + return err +} + +func (pw *progressWriter) Close() error { + pw.updates <- v1.Update{ + Total: pw.size, + Complete: pw.complete, + Error: io.EOF, + } + return io.EOF +} + +// calculateSingleFileInTarSize calculate the size a file will take up in a tar archive, +// given the input data. Provided by rounding up to nearest whole block (512) +// and adding header 512 +func calculateSingleFileInTarSize(in int64) (out int64) { + // doing this manually, because math.Round() works with float64 + out += in + if remainder := out % 512; remainder != 0 { + out += (512 - remainder) + } + out += 512 + return out +} diff --git a/pkg/v1/tarball/write_test.go b/pkg/v1/tarball/write_test.go new file mode 100644 index 0000000..fdfe499 --- /dev/null +++ b/pkg/v1/tarball/write_test.go @@ -0,0 +1,502 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball_test + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/google/go-containerregistry/internal/compare" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestWrite(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image.") + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag.") + } + if err := tarball.WriteToFile(fp.Name(), tag, randImage); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + + // Make sure the image is valid and can be loaded. + // Load it both by nil and by its name. + for _, it := range []*name.Tag{nil, &tag} { + tarImage, err := tarball.ImageFromPath(fp.Name(), it) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + + if err := compare.Images(randImage, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } + + // Try loading a different tag, it should error. + fakeTag, err := name.NewTag("gcr.io/notthistag:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error generating tag: %v", err) + } + if _, err := tarball.ImageFromPath(fp.Name(), &fakeTag); err == nil { + t.Errorf("Expected error loading tag %v from image", fakeTag) + } +} + +func TestMultiWriteSameImage(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image.") + } + + // Make two tags that point to the random image above. + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1.") + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2.") + } + dig3, err := name.NewDigest("gcr.io/baz/baz@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test dig3.") + } + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage + refToImage[tag2] = randImage + refToImage[dig3] = randImage + + // Write the images with both tags to the tarball + if err := tarball.MultiRefWriteToFile(fp.Name(), refToImage); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + + if err := compare.Images(randImage, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } +} + +func TestMultiWriteDifferentImages(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage1, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 1.") + } + + // Make another random image + randImage2, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 2.") + } + + // Make another random image + randImage3, err := random.Image(256, 8) + if err != nil { + t.Fatalf("Error creating random image 3.") + } + + // Create two tags, one pointing to each image created. + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1.") + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2.") + } + dig3, err := name.NewDigest("gcr.io/baz/baz@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test dig3.") + } + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage1 + refToImage[tag2] = randImage2 + refToImage[dig3] = randImage3 + + // Write both images to the tarball. + if err := tarball.MultiRefWriteToFile(fp.Name(), refToImage); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref, img := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + + if err := compare.Images(img, tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } +} + +func TestWriteForeignLayers(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 1) + if err != nil { + t.Fatalf("Error creating random image.") + } + tag, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag.") + } + randLayer, err := random.Layer(512, types.DockerForeignLayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + img, err := mutate.Append(randImage, mutate.Addendum{ + Layer: randLayer, + URLs: []string{ + "example.com", + }, + }) + if err != nil { + t.Fatal(err) + } + if err := tarball.WriteToFile(fp.Name(), tag, img); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Fatalf("validate.Image(): %v", err) + } + + m, err := tarImage.Manifest() + if err != nil { + t.Fatal(err) + } + + if got, want := m.Layers[1].MediaType, types.DockerForeignLayer; got != want { + t.Errorf("Wrong MediaType: %s != %s", got, want) + } + if got, want := m.Layers[1].URLs[0], "example.com"; got != want { + t.Errorf("Wrong URLs: %s != %s", got, want) + } +} + +func TestWriteSharedLayers(t *testing.T) { + // Make a tempfile for tarball writes. + fp, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("Error creating temp file.") + } + t.Log(fp.Name()) + defer fp.Close() + defer os.Remove(fp.Name()) + + // Make a random image + randImage, err := random.Image(256, 1) + if err != nil { + t.Fatalf("Error creating random image.") + } + tag1, err := name.NewTag("gcr.io/foo/bar:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag1.") + } + tag2, err := name.NewTag("gcr.io/baz/bat:latest", name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2.") + } + randLayer, err := random.Layer(512, types.DockerLayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + mutatedImage, err := mutate.Append(randImage, mutate.Addendum{ + Layer: randLayer, + }) + if err != nil { + t.Fatal(err) + } + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage + refToImage[tag2] = mutatedImage + + // Write the images with both tags to the tarball + if err := tarball.MultiRefWriteToFile(fp.Name(), refToImage); err != nil { + t.Fatalf("Unexpected error writing tarball: %v", err) + } + for ref := range refToImage { + tag, ok := ref.(name.Tag) + if !ok { + continue + } + + tarImage, err := tarball.ImageFromPath(fp.Name(), &tag) + if err != nil { + t.Fatalf("Unexpected error reading tarball: %v", err) + } + + if err := validate.Image(tarImage); err != nil { + t.Errorf("validate.Image: %v", err) + } + + if err := compare.Images(refToImage[tag], tarImage); err != nil { + t.Errorf("compare.Images: %v", err) + } + } + _, err = fp.Seek(0, io.SeekStart) + if err != nil { + t.Fatalf("Seek to start of file: %v", err) + } + layers, err := randImage.Layers() + if err != nil { + t.Fatalf("Get image layers: %v", err) + } + layers = append(layers, randLayer) + wantDigests := make(map[string]struct{}) + for _, layer := range layers { + d, err := layer.Digest() + if err != nil { + t.Fatalf("Get layer digest: %v", err) + } + wantDigests[d.Hex] = struct{}{} + } + + const layerFileSuffix = ".tar.gz" + r := tar.NewReader(fp) + for { + hdr, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("Get tar header: %v", err) + } + if strings.HasSuffix(hdr.Name, layerFileSuffix) { + hex := hdr.Name[:len(hdr.Name)-len(layerFileSuffix)] + if _, ok := wantDigests[hex]; ok { + delete(wantDigests, hex) + } else { + t.Errorf("Found unwanted layer with digest %q", hex) + } + } + } + if len(wantDigests) != 0 { + for hex := range wantDigests { + t.Errorf("Expected to find layer with digest %q but it didn't exist", hex) + } + } +} + +func TestComputeManifest(t *testing.T) { + var randomTag, mutatedTag = "ubuntu", "gcr.io/baz/bat:latest" + + // https://github.com/google/go-containerregistry/issues/890 + randomTagWritten := "ubuntu:latest" + + // Make a random image + randImage, err := random.Image(256, 1) + if err != nil { + t.Fatalf("Error creating random image.") + } + randConfig, err := randImage.ConfigName() + if err != nil { + t.Fatalf("error getting random image config: %v", err) + } + tag1, err := name.NewTag(randomTag) + if err != nil { + t.Fatalf("Error creating test tag1.") + } + tag2, err := name.NewTag(mutatedTag, name.StrictValidation) + if err != nil { + t.Fatalf("Error creating test tag2.") + } + randLayer, err := random.Layer(512, types.DockerLayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + mutatedImage, err := mutate.Append(randImage, mutate.Addendum{ + Layer: randLayer, + }) + if err != nil { + t.Fatal(err) + } + mutatedConfig, err := mutatedImage.ConfigName() + if err != nil { + t.Fatalf("error getting mutated image config: %v", err) + } + randomLayersHashes, err := getLayersHashes(randImage) + if err != nil { + t.Fatalf("error getting random image hashes: %v", err) + } + randomLayersFilenames := getLayersFilenames(randomLayersHashes) + + mutatedLayersHashes, err := getLayersHashes(mutatedImage) + if err != nil { + t.Fatalf("error getting mutated image hashes: %v", err) + } + mutatedLayersFilenames := getLayersFilenames(mutatedLayersHashes) + + refToImage := make(map[name.Reference]v1.Image) + refToImage[tag1] = randImage + refToImage[tag2] = mutatedImage + + // calculate the manifest + m, err := tarball.ComputeManifest(refToImage) + if err != nil { + t.Fatalf("Unexpected error calculating manifest: %v", err) + } + // the order of these two is based on the repo tags + // so mutated "gcr.io/baz/bat:latest" is before random "gcr.io/foo/bar:latest" + expected := []tarball.Descriptor{ + { + Config: mutatedConfig.String(), + RepoTags: []string{mutatedTag}, + Layers: mutatedLayersFilenames, + }, + { + Config: randConfig.String(), + RepoTags: []string{randomTagWritten}, + Layers: randomLayersFilenames, + }, + } + if len(m) != len(expected) { + t.Fatalf("mismatched manifest lengths: actual %d, expected %d", len(m), len(expected)) + } + mBytes, err := json.Marshal(m) + if err != nil { + t.Fatalf("unable to marshall actual manifest to json: %v", err) + } + eBytes, err := json.Marshal(expected) + if err != nil { + t.Fatalf("unable to marshall expected manifest to json: %v", err) + } + if !bytes.Equal(mBytes, eBytes) { + t.Errorf("mismatched manifests.\nActual: %s\nExpected: %s", string(mBytes), string(eBytes)) + } +} + +func TestComputeManifest_FailsOnNoRefs(t *testing.T) { + _, err := tarball.ComputeManifest(nil) + if err == nil || !strings.Contains(err.Error(), "set of images is empty") { + t.Error("expected calculateManifest to fail with nil input") + } + + _, err = tarball.ComputeManifest(map[name.Reference]v1.Image{}) + if err == nil || !strings.Contains(err.Error(), "set of images is empty") { + t.Error("expected calculateManifest to fail with empty input") + } +} + +func getLayersHashes(img v1.Image) ([]string, error) { + hashes := []string{} + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("error getting image layers: %w", err) + } + for i, l := range layers { + hash, err := l.Digest() + if err != nil { + return nil, fmt.Errorf("error getting digest for layer %d: %w", i, err) + } + hashes = append(hashes, hash.Hex) + } + return hashes, nil +} + +func getLayersFilenames(hashes []string) []string { + filenames := []string{} + for _, h := range hashes { + filenames = append(filenames, fmt.Sprintf("%s.tar.gz", h)) + } + return filenames +} diff --git a/pkg/v1/types/types.go b/pkg/v1/types/types.go new file mode 100644 index 0000000..efc6bd6 --- /dev/null +++ b/pkg/v1/types/types.go @@ -0,0 +1,82 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types holds common OCI media types. +package types + +// MediaType is an enumeration of the supported mime types that an element of an image might have. +type MediaType string + +// The collection of known MediaType values. +const ( + OCIContentDescriptor MediaType = "application/vnd.oci.descriptor.v1+json" + OCIImageIndex MediaType = "application/vnd.oci.image.index.v1+json" + OCIManifestSchema1 MediaType = "application/vnd.oci.image.manifest.v1+json" + OCIConfigJSON MediaType = "application/vnd.oci.image.config.v1+json" + OCILayer MediaType = "application/vnd.oci.image.layer.v1.tar+gzip" + OCILayerZStd MediaType = "application/vnd.oci.image.layer.v1.tar+zstd" + OCIRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" + OCIUncompressedLayer MediaType = "application/vnd.oci.image.layer.v1.tar" + OCIUncompressedRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar" + + DockerManifestSchema1 MediaType = "application/vnd.docker.distribution.manifest.v1+json" + DockerManifestSchema1Signed MediaType = "application/vnd.docker.distribution.manifest.v1+prettyjws" + DockerManifestSchema2 MediaType = "application/vnd.docker.distribution.manifest.v2+json" + DockerManifestList MediaType = "application/vnd.docker.distribution.manifest.list.v2+json" + DockerLayer MediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + DockerConfigJSON MediaType = "application/vnd.docker.container.image.v1+json" + DockerPluginConfig MediaType = "application/vnd.docker.plugin.v1+json" + DockerForeignLayer MediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + DockerUncompressedLayer MediaType = "application/vnd.docker.image.rootfs.diff.tar" + + OCIVendorPrefix = "vnd.oci" + DockerVendorPrefix = "vnd.docker" +) + +// IsDistributable returns true if a layer is distributable, see: +// https://github.com/opencontainers/image-spec/blob/master/layer.md#non-distributable-layers +func (m MediaType) IsDistributable() bool { + switch m { + case DockerForeignLayer, OCIRestrictedLayer, OCIUncompressedRestrictedLayer: + return false + } + return true +} + +// IsImage returns true if the mediaType represents an image manifest, as opposed to something else, like an index. +func (m MediaType) IsImage() bool { + switch m { + case OCIManifestSchema1, DockerManifestSchema2: + return true + } + return false +} + +// IsIndex returns true if the mediaType represents an index, as opposed to something else, like an image. +func (m MediaType) IsIndex() bool { + switch m { + case OCIImageIndex, DockerManifestList: + return true + } + return false +} + +// IsConfig returns true if the mediaType represents a config, as opposed to something else, like an image. +func (m MediaType) IsConfig() bool { + switch m { + case OCIConfigJSON, DockerConfigJSON: + return true + } + return false +} diff --git a/pkg/v1/types/types_test.go b/pkg/v1/types/types_test.go new file mode 100644 index 0000000..7a8d356 --- /dev/null +++ b/pkg/v1/types/types_test.go @@ -0,0 +1,112 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "testing" + +func TestIsDistributable(t *testing.T) { + for _, mt := range []MediaType{ + OCIRestrictedLayer, + OCIUncompressedRestrictedLayer, + DockerForeignLayer, + } { + if mt.IsDistributable() { + t.Errorf("%s: should not be distributable", mt) + } + } + + for _, mt := range []MediaType{ + OCIContentDescriptor, + OCIImageIndex, + OCIManifestSchema1, + OCIConfigJSON, + OCILayer, + OCIUncompressedLayer, + DockerManifestSchema1, + DockerManifestSchema1Signed, + DockerManifestSchema2, + DockerManifestList, + DockerLayer, + DockerConfigJSON, + DockerPluginConfig, + DockerUncompressedLayer, + } { + if !mt.IsDistributable() { + t.Errorf("%s: should be distributable", mt) + } + } +} + +func TestIsImage(t *testing.T) { + for _, mt := range []MediaType{ + OCIManifestSchema1, DockerManifestSchema2, + } { + if !mt.IsImage() { + t.Errorf("%s: should be image", mt) + } + } + + for _, mt := range []MediaType{ + OCIContentDescriptor, + OCIImageIndex, + OCIConfigJSON, + OCILayer, + OCIRestrictedLayer, + OCIUncompressedLayer, + OCIUncompressedRestrictedLayer, + + DockerManifestList, + DockerLayer, + DockerConfigJSON, + DockerPluginConfig, + DockerForeignLayer, + DockerUncompressedLayer, + } { + if mt.IsImage() { + t.Errorf("%s: should not be image", mt) + } + } +} + +func TestIsIndex(t *testing.T) { + for _, mt := range []MediaType{ + OCIImageIndex, DockerManifestList, + } { + if !mt.IsIndex() { + t.Errorf("%s: should be index", mt) + } + } + + for _, mt := range []MediaType{ + OCIContentDescriptor, + OCIConfigJSON, + OCILayer, + OCIRestrictedLayer, + OCIUncompressedLayer, + OCIUncompressedRestrictedLayer, + OCIManifestSchema1, + + DockerManifestSchema2, + DockerLayer, + DockerConfigJSON, + DockerPluginConfig, + DockerForeignLayer, + DockerUncompressedLayer, + } { + if mt.IsIndex() { + t.Errorf("%s: should not be index", mt) + } + } +} diff --git a/pkg/v1/validate/doc.go b/pkg/v1/validate/doc.go new file mode 100644 index 0000000..91ca87a --- /dev/null +++ b/pkg/v1/validate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package validate provides methods for validating image correctness. +package validate diff --git a/pkg/v1/validate/image.go b/pkg/v1/validate/image.go new file mode 100644 index 0000000..94fb767 --- /dev/null +++ b/pkg/v1/validate/image.go @@ -0,0 +1,288 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "bytes" + "errors" + "fmt" + "io" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +// Image validates that img does not violate any invariants of the image format. +func Image(img v1.Image, opt ...Option) error { + errs := []string{} + if err := validateLayers(img, opt...); err != nil { + errs = append(errs, fmt.Sprintf("validating layers: %v", err)) + } + + if err := validateConfig(img); err != nil { + errs = append(errs, fmt.Sprintf("validating config: %v", err)) + } + + if err := validateManifest(img); err != nil { + errs = append(errs, fmt.Sprintf("validating manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateConfig(img v1.Image) error { + cn, err := img.ConfigName() + if err != nil { + return err + } + + rc, err := img.RawConfigFile() + if err != nil { + return err + } + + hash, size, err := v1.SHA256(bytes.NewReader(rc)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + pcf, err := v1.ParseConfigFile(bytes.NewReader(rc)) + if err != nil { + return err + } + + errs := []string{} + if cn != hash { + errs = append(errs, fmt.Sprintf("mismatched config digest: ConfigName()=%s, SHA256(RawConfigFile())=%s", cn, hash)) + } + + if want, got := m.Config.Size, size; want != got { + errs = append(errs, fmt.Sprintf("mismatched config size: Manifest.Config.Size()=%d, len(RawConfigFile())=%d", want, got)) + } + + if diff := cmp.Diff(pcf, cf); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched config content: (-ParseConfigFile(RawConfigFile()) +ConfigFile()) %s", diff)) + } + + if cf.RootFS.Type != "layers" { + errs = append(errs, fmt.Sprintf("invalid ConfigFile.RootFS.Type: %q != %q", cf.RootFS.Type, "layers")) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateLayers(img v1.Image, opt ...Option) error { + o := makeOptions(opt...) + + layers, err := img.Layers() + if err != nil { + return err + } + + if o.fast { + return layersExist(layers) + } + + digests := []v1.Hash{} + diffids := []v1.Hash{} + udiffids := []v1.Hash{} + sizes := []int64{} + for i, layer := range layers { + cl, err := computeLayer(layer) + if errors.Is(err, io.ErrUnexpectedEOF) { + // Errored while reading tar content of layer because a header or + // content section was not the correct length. This is most likely + // due to an incomplete download or otherwise interrupted process. + m, err := img.Manifest() + if err != nil { + return fmt.Errorf("undersized layer[%d] content", i) + } + return fmt.Errorf("undersized layer[%d] content: Manifest.Layers[%d].Size=%d", i, i, m.Layers[i].Size) + } + if err != nil { + return err + } + // Compute all of these first before we call Config() and Manifest() to allow + // for lazy access e.g. for stream.Layer. + digests = append(digests, cl.digest) + diffids = append(diffids, cl.diffid) + udiffids = append(udiffids, cl.uncompressedDiffid) + sizes = append(sizes, cl.size) + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + errs := []string{} + for i, layer := range layers { + digest, err := layer.Digest() + if err != nil { + return err + } + diffid, err := layer.DiffID() + if err != nil { + return err + } + size, err := layer.Size() + if err != nil { + return err + } + mediaType, err := layer.MediaType() + if err != nil { + return err + } + + if _, err := img.LayerByDigest(digest); err != nil { + return err + } + + if _, err := img.LayerByDiffID(diffid); err != nil { + return err + } + + if digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Digest()=%s, SHA256(Compressed())=%s", i, digest, digests[i])) + } + + if m.Layers[i].Digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Manifest.Layers[%d].Digest=%s, SHA256(Compressed())=%s", i, i, m.Layers[i].Digest, digests[i])) + } + + if diffid != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", i, diffid, diffids[i])) + } + + if diffid != udiffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Uncompressed())=%s", i, diffid, udiffids[i])) + } + + if cf.RootFS.DiffIDs[i] != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: ConfigFile.RootFS.DiffIDs[%d]=%s, SHA256(Gunzip(Compressed()))=%s", i, i, cf.RootFS.DiffIDs[i], diffids[i])) + } + + if size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Size()=%d, len(Compressed())=%d", i, size, sizes[i])) + } + + if m.Layers[i].Size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Manifest.Layers[%d].Size=%d, len(Compressed())=%d", i, i, m.Layers[i].Size, sizes[i])) + } + + if m.Layers[i].MediaType != mediaType { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] mediaType: Manifest.Layers[%d].MediaType=%s, layer.MediaType()=%s", i, i, m.Layers[i].MediaType, mediaType)) + } + } + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateManifest(img v1.Image) error { + digest, err := img.Digest() + if err != nil { + return err + } + + size, err := img.Size() + if err != nil { + return err + } + + rm, err := img.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + pm, err := v1.ParseManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if size != int64(len(rm)) { + errs = append(errs, fmt.Sprintf("mismatched manifest size: Size()=%d, len(RawManifest())=%d", size, len(rm))) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func layersExist(layers []v1.Layer) error { + errs := []string{} + for _, layer := range layers { + ok, err := partial.Exists(layer) + if err != nil { + errs = append(errs, err.Error()) + } + if !ok { + errs = append(errs, "layer does not exist") + } + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} diff --git a/pkg/v1/validate/index.go b/pkg/v1/validate/index.go new file mode 100644 index 0000000..7514dc4 --- /dev/null +++ b/pkg/v1/validate/index.go @@ -0,0 +1,175 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Index validates that idx does not violate any invariants of the index format. +func Index(idx v1.ImageIndex, opt ...Option) error { + errs := []string{} + + if err := validateChildren(idx, opt...); err != nil { + errs = append(errs, fmt.Sprintf("validating children: %v", err)) + } + + if err := validateIndexManifest(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating index manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +type withLayer interface { + Layer(v1.Hash) (v1.Layer, error) +} + +func validateChildren(idx v1.ImageIndex, opt ...Option) error { + manifest, err := idx.IndexManifest() + if err != nil { + return err + } + + errs := []string{} + for i, desc := range manifest.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + idx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := Index(idx, opt...); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate index Manifests[%d](%s): %v", i, desc.Digest, err)) + } + if err := validateMediaType(idx, desc.MediaType); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate index MediaType[%d](%s): %v", i, desc.Digest, err)) + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := idx.Image(desc.Digest) + if err != nil { + return err + } + if err := Image(img, opt...); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate image Manifests[%d](%s): %v", i, desc.Digest, err)) + } + if err := validateMediaType(img, desc.MediaType); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate image MediaType[%d](%s): %v", i, desc.Digest, err)) + } + default: + // Workaround for #819. + if wl, ok := idx.(withLayer); ok { + layer, err := wl.Layer(desc.Digest) + if err != nil { + return fmt.Errorf("failed to get layer Manifests[%d]: %w", i, err) + } + if err := Layer(layer, opt...); err != nil { + lerr := fmt.Sprintf("failed to validate layer Manifests[%d](%s): %v", i, desc.Digest, err) + if desc.MediaType.IsDistributable() { + errs = append(errs, lerr) + } else { + logs.Warn.Printf("nondistributable layer failure: %v", lerr) + } + } + } else { + logs.Warn.Printf("Unexpected manifest: %s", desc.MediaType) + } + } + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +type withMediaType interface { + MediaType() (types.MediaType, error) +} + +func validateMediaType(i withMediaType, want types.MediaType) error { + got, err := i.MediaType() + if err != nil { + return err + } + if want != got { + return fmt.Errorf("mismatched mediaType: MediaType() = %v != %v", got, want) + } + + return nil +} + +func validateIndexManifest(idx v1.ImageIndex) error { + digest, err := idx.Digest() + if err != nil { + return err + } + + size, err := idx.Size() + if err != nil { + return err + } + + rm, err := idx.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := idx.IndexManifest() + if err != nil { + return err + } + + pm, err := v1.ParseIndexManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseIndexManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if size != int64(len(rm)) { + errs = append(errs, fmt.Sprintf("mismatched manifest size: Size()=%d, len(RawManifest())=%d", size, len(rm))) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} diff --git a/pkg/v1/validate/layer.go b/pkg/v1/validate/layer.go new file mode 100644 index 0000000..fdd8f38 --- /dev/null +++ b/pkg/v1/validate/layer.go @@ -0,0 +1,191 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "archive/tar" + "compress/gzip" + "crypto" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +// Layer validates that the values return by its methods are consistent with the +// contents returned by Compressed and Uncompressed. +func Layer(layer v1.Layer, opt ...Option) error { + o := makeOptions(opt...) + if o.fast { + ok, err := partial.Exists(layer) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("layer does not exist") + } + return nil + } + + cl, err := computeLayer(layer) + if err != nil { + return err + } + + errs := []string{} + + digest, err := layer.Digest() + if err != nil { + return err + } + diffid, err := layer.DiffID() + if err != nil { + return err + } + size, err := layer.Size() + if err != nil { + return err + } + + if digest != cl.digest { + errs = append(errs, fmt.Sprintf("mismatched digest: Digest()=%s, SHA256(Compressed())=%s", digest, cl.digest)) + } + + if diffid != cl.diffid { + errs = append(errs, fmt.Sprintf("mismatched diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", diffid, cl.diffid)) + } + + if diffid != cl.uncompressedDiffid { + errs = append(errs, fmt.Sprintf("mismatched diffid: DiffID()=%s, SHA256(Uncompressed())=%s", diffid, cl.uncompressedDiffid)) + } + + if size != cl.size { + errs = append(errs, fmt.Sprintf("mismatched size: Size()=%d, len(Compressed())=%d", size, cl.size)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +type computedLayer struct { + // Calculated from Compressed stream. + digest v1.Hash + size int64 + diffid v1.Hash + + // Calculated from Uncompressed stream. + uncompressedDiffid v1.Hash + uncompressedSize int64 +} + +func computeLayer(layer v1.Layer) (*computedLayer, error) { + compressed, err := layer.Compressed() + if err != nil { + return nil, err + } + + // Keep track of compressed digest. + digester := crypto.SHA256.New() + // Everything read from compressed is written to digester to compute digest. + hashCompressed := io.TeeReader(compressed, digester) + + // Call io.Copy to write from the layer Reader through to the tarReader on + // the other side of the pipe. + pr, pw := io.Pipe() + var size int64 + go func() { + n, err := io.Copy(pw, hashCompressed) + if err != nil { + pw.CloseWithError(err) + return + } + size = n + + // Now close the compressed reader, to flush the gzip stream + // and calculate digest/diffID/size. This will cause pr to + // return EOF which will cause readers of the Compressed stream + // to finish reading. + pw.CloseWithError(compressed.Close()) + }() + + // Read the bytes through gzip.Reader to compute the DiffID. + uncompressed, err := gzip.NewReader(pr) + if err != nil { + return nil, err + } + diffider := crypto.SHA256.New() + hashUncompressed := io.TeeReader(uncompressed, diffider) + + // Ensure there aren't duplicate file paths. + tarReader := tar.NewReader(hashUncompressed) + files := make(map[string]struct{}) + for { + hdr, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + if _, ok := files[hdr.Name]; ok { + return nil, fmt.Errorf("duplicate file path: %s", hdr.Name) + } + files[hdr.Name] = struct{}{} + } + + // Discard any trailing padding that the tar.Reader doesn't consume. + if _, err := io.Copy(io.Discard, hashUncompressed); err != nil { + return nil, err + } + + if err := uncompressed.Close(); err != nil { + return nil, err + } + + digest := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(digester.Sum(make([]byte, 0, digester.Size()))), + } + + diffid := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(diffider.Sum(make([]byte, 0, diffider.Size()))), + } + + ur, err := layer.Uncompressed() + if err != nil { + return nil, err + } + defer ur.Close() + udiffid, usize, err := v1.SHA256(ur) + if err != nil { + return nil, err + } + + return &computedLayer{ + digest: digest, + diffid: diffid, + size: size, + uncompressedDiffid: udiffid, + uncompressedSize: usize, + }, nil +} diff --git a/pkg/v1/validate/options.go b/pkg/v1/validate/options.go new file mode 100644 index 0000000..a6bf2dc --- /dev/null +++ b/pkg/v1/validate/options.go @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +// Option is a functional option for validate. +type Option func(*options) + +type options struct { + fast bool +} + +func makeOptions(opts ...Option) options { + opt := options{ + fast: false, + } + for _, o := range opts { + o(&opt) + } + return opt +} + +// Fast causes validate to skip reading and digesting layer bytes. +func Fast(o *options) { + o.fast = true +} diff --git a/pkg/v1/zz_deepcopy_generated.go b/pkg/v1/zz_deepcopy_generated.go new file mode 100644 index 0000000..a47b747 --- /dev/null +++ b/pkg/v1/zz_deepcopy_generated.go @@ -0,0 +1,339 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Config) DeepCopyInto(out *Config) { + *out = *in + if in.Cmd != nil { + in, out := &in.Cmd, &out.Cmd + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Healthcheck != nil { + in, out := &in.Healthcheck, &out.Healthcheck + *out = new(HealthConfig) + (*in).DeepCopyInto(*out) + } + if in.Entrypoint != nil { + in, out := &in.Entrypoint, &out.Entrypoint + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.OnBuild != nil { + in, out := &in.OnBuild, &out.OnBuild + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make(map[string]struct{}, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ExposedPorts != nil { + in, out := &in.ExposedPorts, &out.ExposedPorts + *out = make(map[string]struct{}, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Shell != nil { + in, out := &in.Shell, &out.Shell + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. +func (in *Config) DeepCopy() *Config { + if in == nil { + return nil + } + out := new(Config) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigFile) DeepCopyInto(out *ConfigFile) { + *out = *in + in.Created.DeepCopyInto(&out.Created) + if in.History != nil { + in, out := &in.History, &out.History + *out = make([]History, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.RootFS.DeepCopyInto(&out.RootFS) + in.Config.DeepCopyInto(&out.Config) + if in.OSFeatures != nil { + in, out := &in.OSFeatures, &out.OSFeatures + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigFile. +func (in *ConfigFile) DeepCopy() *ConfigFile { + if in == nil { + return nil + } + out := new(ConfigFile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Descriptor) DeepCopyInto(out *Descriptor) { + *out = *in + out.Digest = in.Digest + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.URLs != nil { + in, out := &in.URLs, &out.URLs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Platform != nil { + in, out := &in.Platform, &out.Platform + *out = new(Platform) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Descriptor. +func (in *Descriptor) DeepCopy() *Descriptor { + if in == nil { + return nil + } + out := new(Descriptor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Hash) DeepCopyInto(out *Hash) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Hash. +func (in *Hash) DeepCopy() *Hash { + if in == nil { + return nil + } + out := new(Hash) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthConfig) DeepCopyInto(out *HealthConfig) { + *out = *in + if in.Test != nil { + in, out := &in.Test, &out.Test + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthConfig. +func (in *HealthConfig) DeepCopy() *HealthConfig { + if in == nil { + return nil + } + out := new(HealthConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *History) DeepCopyInto(out *History) { + *out = *in + in.Created.DeepCopyInto(&out.Created) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new History. +func (in *History) DeepCopy() *History { + if in == nil { + return nil + } + out := new(History) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IndexManifest) DeepCopyInto(out *IndexManifest) { + *out = *in + if in.Manifests != nil { + in, out := &in.Manifests, &out.Manifests + *out = make([]Descriptor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Subject != nil { + in, out := &in.Subject, &out.Subject + *out = new(Descriptor) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexManifest. +func (in *IndexManifest) DeepCopy() *IndexManifest { + if in == nil { + return nil + } + out := new(IndexManifest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Manifest) DeepCopyInto(out *Manifest) { + *out = *in + in.Config.DeepCopyInto(&out.Config) + if in.Layers != nil { + in, out := &in.Layers, &out.Layers + *out = make([]Descriptor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Subject != nil { + in, out := &in.Subject, &out.Subject + *out = new(Descriptor) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Manifest. +func (in *Manifest) DeepCopy() *Manifest { + if in == nil { + return nil + } + out := new(Manifest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Platform) DeepCopyInto(out *Platform) { + *out = *in + if in.OSFeatures != nil { + in, out := &in.OSFeatures, &out.OSFeatures + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Features != nil { + in, out := &in.Features, &out.Features + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Platform. +func (in *Platform) DeepCopy() *Platform { + if in == nil { + return nil + } + out := new(Platform) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RootFS) DeepCopyInto(out *RootFS) { + *out = *in + if in.DiffIDs != nil { + in, out := &in.DiffIDs, &out.DiffIDs + *out = make([]Hash, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RootFS. +func (in *RootFS) DeepCopy() *RootFS { + if in == nil { + return nil + } + out := new(RootFS) + in.DeepCopyInto(out) + return out +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Time. +func (in *Time) DeepCopy() *Time { + if in == nil { + return nil + } + out := new(Time) + in.DeepCopyInto(out) + return out +} -- cgit v1.2.3