From c4c467734a1b7be01ac3becbd2af2ba4dc3b38aa Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Sun, 8 Jun 2025 14:41:09 +0200 Subject: [PATCH] Fix OCI client header parsing and improve tests Signed-off-by: Philip Laine --- CHANGELOG.md | 1 + go.mod | 1 + go.sum | 8 ++- pkg/httpx/httpx.go | 5 ++ pkg/oci/client.go | 14 ++++- pkg/oci/client_test.go | 126 +++++++++++++++++------------------------ pkg/oci/oci_test.go | 2 +- 7 files changed, 78 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f49cdd..86b3237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#911](https://github.com/spegel-org/spegel/pull/911) Enforce use of request contexts and fix response closing. +- [#914](https://github.com/spegel-org/spegel/pull/914) Fix OCI client header parsing and improve tests. ### Security diff --git a/go.mod b/go.mod index e1d8c31..c938b86 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.0 toolchain go1.24.3 require ( + cuelabs.dev/go/oci/ociregistry v0.0.0-20250530080122-d0efc28a5723 github.com/alexflint/go-arg v1.5.1 github.com/containerd/containerd/api v1.9.0 github.com/containerd/containerd/v2 v2.1.1 diff --git a/go.sum b/go.sum index 5b9521f..5a026ba 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cuelabs.dev/go/oci/ociregistry v0.0.0-20250530080122-d0efc28a5723 h1:FGl278ML+6v4yF1FkYELcMXvP+BAsvsW+H3WkGQy+aE= +cuelabs.dev/go/oci/ociregistry v0.0.0-20250530080122-d0efc28a5723/go.mod h1:dqrnoZx62xbOZr11giMPrWbhlaV8euHwciXZEy3baT8= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= @@ -118,6 +120,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -427,8 +431,8 @@ github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6 github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/pkg/httpx/httpx.go b/pkg/httpx/httpx.go index c6ca113..198667f 100644 --- a/pkg/httpx/httpx.go +++ b/pkg/httpx/httpx.go @@ -21,6 +21,11 @@ const ( HeaderXForwardedFor = "X-Forwarded-For" ) +const ( + ContentTypeBinary = "application/octet-stream" + ContentTypeJSON = "application/json" +) + // BaseClient returns a http client with reasonable defaults set. func BaseClient() *http.Client { return &http.Client{ diff --git a/pkg/oci/client.go b/pkg/oci/client.go index adba660..42dfafa 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -213,10 +213,18 @@ func (c *Client) fetch(ctx context.Context, method string, dist DistributionPath return nil, ocispec.Descriptor{}, err } - if resp.Header.Get(HeaderDockerDigest) == "" { - resp.Header.Set(HeaderDockerDigest, dist.Digest.String()) + // Handle optional headers for blobs. + header := resp.Header.Clone() + if dist.Kind == DistributionKindBlob { + if header.Get(httpx.HeaderContentType) == "" { + header.Set(httpx.HeaderContentType, httpx.ContentTypeBinary) + } + if header.Get(HeaderDockerDigest) == "" { + header.Set(HeaderDockerDigest, dist.Digest.String()) + } } - desc, err := DescriptorFromHeader(resp.Header) + + desc, err := DescriptorFromHeader(header) if err != nil { httpx.DrainAndClose(resp.Body) return nil, ocispec.Descriptor{}, err diff --git a/pkg/oci/client_test.go b/pkg/oci/client_test.go index 0acc6de..610313a 100644 --- a/pkg/oci/client_test.go +++ b/pkg/oci/client_test.go @@ -1,103 +1,83 @@ package oci import ( - "encoding/json" - "errors" "net/http" "net/http/httptest" "net/url" - "runtime" + "os" + "path/filepath" "testing" + "cuelabs.dev/go/oci/ociregistry/ocimem" + "cuelabs.dev/go/oci/ociregistry/ociserver" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spegel-org/spegel/pkg/httpx" "github.com/stretchr/testify/require" ) -func TestPull(t *testing.T) { +func TestClient(t *testing.T) { t.Parallel() - srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - b, mt, err := func() ([]byte, string, error) { - switch req.URL.Path { - case "/v2/test/image/manifests/index": - idx := ocispec.Index{ - Manifests: []ocispec.Descriptor{ - { - Digest: digest.Digest("manifest"), - Platform: &ocispec.Platform{ - OS: runtime.GOOS, - Architecture: runtime.GOARCH, - }, - }, - }, - } - b, err := json.Marshal(&idx) - if err != nil { - return nil, "", err - } - return b, ocispec.MediaTypeImageIndex, nil - case "/v2/test/image/manifests/manifest": - manifest := ocispec.Manifest{ - Config: ocispec.Descriptor{ - Digest: digest.Digest("config"), - }, - Layers: []ocispec.Descriptor{ - { - Digest: digest.Digest("layer"), - }, - }, - } - b, err := json.Marshal(&manifest) - if err != nil { - return nil, "", err - } - return b, ocispec.MediaTypeImageManifest, nil - case "/v2/test/image/blobs/config": - config := ocispec.ImageConfig{ - User: "root", - } - b, err := json.Marshal(&config) - if err != nil { - return nil, "", err - } - return b, ocispec.MediaTypeImageConfig, nil - case "/v2/test/image/blobs/layer": - return []byte("hello world"), ocispec.MediaTypeImageLayer, nil - default: - return nil, "", errors.New("not found") - } - }() - if err != nil { - rw.WriteHeader(http.StatusNotFound) - return - } + img := Image{ + Repository: "test/image", + Tag: "latest", + } - rw.Header().Set(httpx.HeaderContentType, mt) - dgst := digest.SHA256.FromBytes(b) - rw.Header().Set(HeaderDockerDigest, dgst.String()) - rw.WriteHeader(http.StatusOK) - - //nolint: errcheck // Ignore error. - rw.Write(b) - })) + mem := ocimem.New() + blobs := []ocispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: digest.Digest("sha256:68b8a989a3e08ddbdb3a0077d35c0d0e59c9ecf23d0634584def8bdbb7d6824f"), + Size: 529, + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + Digest: digest.Digest("sha256:3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996"), + Size: 118, + }, + } + for _, blob := range blobs { + f, err := os.Open(filepath.Join("testdata", "blobs", "sha256", blob.Digest.Encoded())) + require.NoError(t, err) + _, err = mem.PushBlob(t.Context(), img.Repository, blob, f) + f.Close() + require.NoError(t, err) + } + manifests := []ocispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest.Digest("sha256:b6d6089ca6c395fd563c2084f5dd7bc56a2f5e6a81413558c5be0083287a77e9"), + }, + } + for _, manifest := range manifests { + b, err := os.ReadFile(filepath.Join("testdata", "blobs", "sha256", manifest.Digest.Encoded())) + require.NoError(t, err) + _, err = mem.PushManifest(t.Context(), img.Repository, img.Tag, b, manifest.MediaType) + require.NoError(t, err) + } + reg := ociserver.New(mem, nil) + srv := httptest.NewServer(reg) t.Cleanup(func() { srv.Close() }) - img := Image{ - Repository: "test/image", - Digest: digest.Digest("index"), - Registry: "example.com", - } client := NewClient() mirror, err := url.Parse(srv.URL) require.NoError(t, err) pullResults, err := client.Pull(t.Context(), img, mirror) require.NoError(t, err) + require.Len(t, pullResults, 3) - require.NotEmpty(t, pullResults) + dist := DistributionPath{ + Kind: DistributionKindBlob, + Name: img.Repository, + Digest: blobs[0].Digest, + } + desc, err := client.Head(t.Context(), dist, mirror) + require.NoError(t, err) + require.Equal(t, dist.Digest, desc.Digest) + require.Equal(t, httpx.ContentTypeBinary, desc.MediaType) } func TestDescriptorHeader(t *testing.T) { diff --git a/pkg/oci/oci_test.go b/pkg/oci/oci_test.go index 968a7ca..88e2ac1 100644 --- a/pkg/oci/oci_test.go +++ b/pkg/oci/oci_test.go @@ -21,7 +21,7 @@ import ( bolt "go.etcd.io/bbolt" ) -func TestOCIClient(t *testing.T) { +func TestStore(t *testing.T) { t.Parallel() b, err := os.ReadFile("./testdata/images.json")