Change mirror type to url and add byte range parameter (#905)

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [#905](https://github.com/spegel-org/spegel/pull/905) Change mirror type to url and add byte range parameter.
+
 ### Changed
 
 ### Deprecated

internal/web/web.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/http"
 	"net/netip"
+	"net/url"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -113,6 +114,11 @@ type pullResult struct {
 }
 
 func (w *Web) measureHandler(rw httpx.ResponseWriter, req *http.Request) {
+	mirror := &url.URL{
+		Scheme: "http",
+		Host:   "localhost:5000",
+	}
+
 	// Parse image name.
 	imgName := req.URL.Query().Get("image")
 	if imgName == "" {
@@ -145,7 +151,7 @@ func (w *Web) measureHandler(rw httpx.ResponseWriter, req *http.Request) {
 
 	if len(res.PeerResults) > 0 {
 		// Pull the image and measure performance.
-		pullMetrics, err := w.client.Pull(req.Context(), img, "http://localhost:5000")
+		pullMetrics, err := w.client.Pull(req.Context(), img, mirror)
 		if err != nil {
 			rw.WriteError(http.StatusInternalServerError, err)
 			return

pkg/httpx/httpx.go

@@ -3,5 +3,7 @@ package httpx
 const (
 	HeaderContentType   = "Content-Type"
 	HeaderContentLength = "Content-Length"
+	HeaderContentRange  = "Content-Range"
+	HeaderRange         = "Range"
 	HeaderAcceptRanges  = "Accept-Ranges"
 )

pkg/httpx/range.go

@@ -0,0 +1,26 @@
+package httpx
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ByteRange struct {
+	Start int64 `json:"start"`
+	End   int64 `json:"end"`
+}
+
+func FormatRangeHeader(byteRange ByteRange) string {
+	return fmt.Sprintf("bytes=%d-%d", byteRange.Start, byteRange.End)
+}
+
+func FormatMultipartRangeHeader(byteRanges []ByteRange) string {
+	if len(byteRanges) == 0 {
+		return ""
+	}
+	ranges := []string{}
+	for _, br := range byteRanges {
+		ranges = append(ranges, fmt.Sprintf("%d-%d", br.Start, br.End))
+	}
+	return "bytes=" + strings.Join(ranges, ", ")
+}

pkg/httpx/range_test.go

@@ -0,0 +1,35 @@
+package httpx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFormatRangeHeader(t *testing.T) {
+	t.Parallel()
+
+	br := ByteRange{Start: 10, End: 2000}
+	val := FormatRangeHeader(br)
+	require.Equal(t, "bytes=10-2000", val)
+}
+
+func TestFormatMultipartRangeHeader(t *testing.T) {
+	t.Parallel()
+
+	brr := []ByteRange{
+		{
+			Start: 10,
+			End:   100,
+		},
+		{
+			Start: 0,
+			End:   1,
+		},
+	}
+	val := FormatMultipartRangeHeader(brr)
+	require.Equal(t, "bytes=10-100, 0-1", val)
+
+	val = FormatMultipartRangeHeader(nil)
+	require.Empty(t, val)
+}

pkg/oci/client.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -43,7 +44,7 @@ type PullMetric struct {
 	Duration      time.Duration
 }
 
-func (c *Client) Pull(ctx context.Context, img Image, mirror string) ([]PullMetric, error) {
+func (c *Client) Pull(ctx context.Context, img Image, mirror *url.URL) ([]PullMetric, error) {
 	pullMetrics := []PullMetric{}
 
 	queue := []DistributionPath{
@@ -60,7 +61,7 @@ func (c *Client) Pull(ctx context.Context, img Image, mirror string) ([]PullMetr
 		queue = queue[1:]
 
 		start := time.Now()
-		rc, desc, err := c.Get(ctx, dist, mirror)
+		rc, desc, err := c.Get(ctx, dist, mirror, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -134,8 +135,8 @@ func (c *Client) Pull(ctx context.Context, img Image, mirror string) ([]PullMetr
 	return pullMetrics, nil
 }
 
-func (c *Client) Head(ctx context.Context, dist DistributionPath, mirror string) (ocispec.Descriptor, error) {
-	rc, desc, err := c.fetch(ctx, http.MethodHead, dist, mirror)
+func (c *Client) Head(ctx context.Context, dist DistributionPath, mirror *url.URL) (ocispec.Descriptor, error) {
+	rc, desc, err := c.fetch(ctx, http.MethodHead, dist, mirror, nil)
 	if err != nil {
 		return ocispec.Descriptor{}, err
 	}
@@ -147,25 +148,22 @@ func (c *Client) Head(ctx context.Context, dist DistributionPath, mirror string)
 	return desc, nil
 }
 
-func (c *Client) Get(ctx context.Context, dist DistributionPath, mirror string) (io.ReadCloser, ocispec.Descriptor, error) {
-	rc, desc, err := c.fetch(ctx, http.MethodGet, dist, mirror)
+func (c *Client) Get(ctx context.Context, dist DistributionPath, mirror *url.URL, brr []httpx.ByteRange) (io.ReadCloser, ocispec.Descriptor, error) {
+	rc, desc, err := c.fetch(ctx, http.MethodGet, dist, mirror, brr)
 	if err != nil {
 		return nil, ocispec.Descriptor{}, err
 	}
 	return rc, desc, nil
 }
 
-func (c *Client) fetch(ctx context.Context, method string, dist DistributionPath, mirror string) (io.ReadCloser, ocispec.Descriptor, error) {
+func (c *Client) fetch(ctx context.Context, method string, dist DistributionPath, mirror *url.URL, brr []httpx.ByteRange) (io.ReadCloser, ocispec.Descriptor, error) {
 	tcKey := dist.Registry + dist.Name
 
 	u := dist.URL()
-	if mirror != "" {
-		mirrorUrl, err := url.Parse(mirror)
-		if err != nil {
-			return nil, ocispec.Descriptor{}, err
-		}
-		u.Scheme = mirrorUrl.Scheme
-		u.Host = mirrorUrl.Host
+	if mirror != nil {
+		u.Scheme = mirror.Scheme
+		u.Host = mirror.Host
+		u.Path = path.Join(mirror.Path, u.Path)
 	}
 	if u.Host == "docker.io" {
 		u.Host = "registry-1.docker.io"
@@ -181,6 +179,9 @@ func (c *Client) fetch(ctx context.Context, method string, dist DistributionPath
 		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
 		req.Header.Add("Accept", "application/vnd.oci.image.index.v1+json")
 		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.list.v2+json")
+		if len(brr) > 0 {
+			req.Header.Add(httpx.HeaderRange, httpx.FormatMultipartRangeHeader(brr))
+		}
 		token, ok := c.tc.Load(tcKey)
 		if ok {
 			//nolint: errcheck // We know it will be a string.

pkg/oci/client_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"runtime"
 	"testing"
 
@@ -91,7 +92,9 @@ func TestPull(t *testing.T) {
 		Registry:   "example.com",
 	}
 	client := NewClient()
-	pullResults, err := client.Pull(t.Context(), img, srv.URL)
+	mirror, err := url.Parse(srv.URL)
+	require.NoError(t, err)
+	pullResults, err := client.Pull(t.Context(), img, mirror)
 	require.NoError(t, err)
 
 	require.NotEmpty(t, pullResults)