fix(s3): signatures for alternate hosts

config.example.yaml

@@ -133,12 +133,9 @@ storage:
   # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m".
   direct_serve_ttl: "15m"
 
-  # Public base URL to substitute into presigned URLs. Set this when the
-  # proxy reaches storage at an internal address (127.0.0.1, a Docker
-  # service name) but clients must use a public hostname. Only scheme and
-  # host are used; the signed path and query are preserved. For S3/MinIO
-  # the reverse proxy at this address must forward requests with the
-  # internal Host header or the SigV4 signature will not validate.
+  # Public base URL for presigned URLs. Set this when the proxy reaches
+  # storage at an internal address (127.0.0.1, a Docker service name, or
+  # VPC DNS) but clients must use a public hostname or CDN.
   # direct_serve_base_url: "https://minio.example.com"
 
 # Database configuration

internal/handler/handler.go

@@ -240,11 +240,7 @@ type Proxy struct {
 	NPMFullMetadata bool
 	DirectServe     bool
 	DirectServeTTL  time.Duration
-	// DirectServeBaseURL, if set, replaces the scheme and host of presigned
-	// URLs so clients receive a public address even when the proxy reaches
-	// storage at an internal one.
-	DirectServeBaseURL string
-	HTTPClient         *http.Client
+	HTTPClient      *http.Client
 
 	// RateLimits is the gate under the upstream clients. Artifact fetches
 	// check it before calling Fetcher, so held hosts never reach its breaker.
@@ -548,7 +544,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s
 	if p.DirectServe {
 		signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL)
 		if err == nil {
-			result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL)
+			result.RedirectURL = signed
 			p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
 			return result, nil
 		}
@@ -595,26 +591,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s
 	return result, nil
 }
 
-// rewriteSignedURLHost replaces the scheme and host of a signed URL with those
-// from baseURL, preserving the path and query (which carry the signature).
-// Returns signed unchanged if baseURL is empty or either URL fails to parse.
-func rewriteSignedURLHost(signed, baseURL string) string {
-	if baseURL == "" {
-		return signed
-	}
-	s, err := url.Parse(signed)
-	if err != nil {
-		return signed
-	}
-	b, err := url.Parse(baseURL)
-	if err != nil || b.Scheme == "" || b.Host == "" {
-		return signed
-	}
-	s.Scheme = b.Scheme
-	s.Host = b.Host
-	return s.String()
-}
-
 func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) {
 	_ = p.DB.RecordArtifactHit(versionPURL, filename)
 	metrics.RecordArtifactCacheHit(ecosystem)

internal/handler/handler_test.go

@@ -638,74 +638,6 @@ func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) {
 	}
 }
 
-func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) {
-	proxy, db, store, _ := setupTestProxy(t)
-	seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content")
-
-	proxy.DirectServe = true
-	proxy.DirectServeBaseURL = "https://cdn.example.com"
-	store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900"
-
-	result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900"
-	if result.RedirectURL != want {
-		t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want)
-	}
-}
-
-func TestRewriteSignedURLHost(t *testing.T) {
-	tests := []struct {
-		name    string
-		signed  string
-		baseURL string
-		want    string
-	}{
-		{
-			"empty base url is no-op",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-		},
-		{
-			"replaces scheme and host",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"https://cdn.example.com",
-			"https://cdn.example.com/bucket/key?sig=abc",
-		},
-		{
-			"preserves path and query",
-			"http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
-			"https://files.example.com",
-			"https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
-		},
-		{
-			"ignores base url path",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"https://cdn.example.com/ignored",
-			"https://cdn.example.com/bucket/key?sig=abc",
-		},
-		{
-			"invalid base url is no-op",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"://bad",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := rewriteSignedURLHost(tt.signed, tt.baseURL)
-			if got != tt.want {
-				t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want)
-			}
-		})
-	}
-}
-
 func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) {
 	proxy, db, store, _ := setupTestProxy(t)
 	seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content")

internal/server/server.go

@@ -189,7 +189,11 @@ func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server,
 		// Fall back to file:// with Path
 		storageURL = "file://" + cfg.Storage.Path //nolint:staticcheck // backwards compat
 	}
-	store, err := storage.OpenBucket(context.Background(), storageURL)
+	var storageOpts []storage.Option
+	if cfg.Storage.DirectServe && cfg.Storage.DirectServeBaseURL != "" {
+		storageOpts = append(storageOpts, storage.WithSignEndpoint(cfg.Storage.DirectServeBaseURL))
+	}
+	store, err := storage.OpenBucket(context.Background(), storageURL, storageOpts...)
 	if err != nil {
 		_ = db.Close()
 		return nil, fmt.Errorf("initializing storage: %w", err)
@@ -317,7 +321,6 @@ func (s *Server) serve(listener net.Listener) error {
 	proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
 	proxy.DirectServe = s.cfg.Storage.DirectServe
 	proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL()
-	proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL
 	proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts
 	if err := ConfigureDenylist(s.cfg, proxy); err != nil {
 		return err

internal/storage/blob.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -30,23 +31,49 @@ const osWindows = "windows"
 const (
 	uploadBufferSize  = 5 << 20
 	uploadConcurrency = 2
-)
 
-// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an
-// earlier version wrote.
-const attrsExt = ".attrs"
+	s3Scheme = "s3://"
+
+	// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an
+	// earlier version wrote.
+	attrsExt = ".attrs"
+)
 
 // Blob implements Storage using gocloud.dev/blob.
 // Supports local filesystem (file://) and S3 (s3://) URLs.
 type Blob struct {
 	bucket *blob.Bucket
 	url    string
 
+	// signBucket, when set, is used by SignedURL to sign S3 presigned URLs for
+	// a public endpoint rather than the internal storage endpoint.
+	signBucket *blob.Bucket
+
+	// signEndpoint, when set on non-S3 backends, replaces the scheme and host
+	// of presigned URLs returned by SignedURL.
+	signEndpoint string
+
 	// fileRoot is the directory backing a file:// bucket, empty for cloud
 	// backends. Used only to clear sidecars an earlier version wrote.
 	fileRoot string
 }
 
+// Option configures OpenBucket.
+type Option func(*options)
+
+type options struct {
+	signEndpoint string
+}
+
+// WithSignEndpoint configures presigned URLs to use the scheme and host of
+// signEndpoint (e.g. "https://storage.example.com") instead of the storage
+// endpoint. For S3, the URL is signed directly against signEndpoint so the
+// SigV4 Host signature matches; for other backends, the signed URL's scheme
+// and host are rewritten.
+func WithSignEndpoint(signEndpoint string) Option {
+	return func(o *options) { o.signEndpoint = signEndpoint }
+}
+
 // OpenBucket opens a blob bucket from a URL.
 //
 // Supported URL schemes:
@@ -60,9 +87,19 @@ type Blob struct {
 // For local filesystem, the directory is created if it doesn't exist.
 //
 //nolint:ireturn // The URL scheme selects the storage implementation.
-func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
+func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, error) {
+	var o options
+	for _, opt := range opts {
+		opt(&o)
+	}
+
 	if strings.HasPrefix(urlStr, "gs://") {
-		return OpenGCS(ctx, urlStr)
+		g, err := OpenGCS(ctx, urlStr)
+		if err != nil {
+			return nil, err
+		}
+		g.signEndpoint = o.signEndpoint
+		return g, nil
 	}
 
 	var fileRoot string
@@ -124,7 +161,46 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
 		return nil, fmt.Errorf("opening bucket: %w", err)
 	}
 
-	return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil
+	b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}
+
+	if o.signEndpoint != "" {
+		if strings.HasPrefix(urlStr, s3Scheme) {
+			signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint)
+			if err != nil {
+				_ = bucket.Close()
+				return nil, err
+			}
+			b.signBucket = signBucket
+		} else {
+			b.signEndpoint = o.signEndpoint
+		}
+	}
+
+	return b, nil
+}
+
+// openSignBucket opens an S3 bucket with the endpoint overridden to signEndpoint
+// while preserving all other storage URL parameters (region, use_path_style, etc.).
+func openSignBucket(ctx context.Context, urlStr, signEndpoint string) (*blob.Bucket, error) {
+	sign, err := url.Parse(signEndpoint)
+	if err != nil || sign.Scheme == "" || sign.Host == "" {
+		return nil, fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint)
+	}
+
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing storage URL: %w", err)
+	}
+	endpoint := sign.Scheme + "://" + sign.Host
+	q := u.Query()
+	q.Set("endpoint", endpoint)
+	u.RawQuery = q.Encode()
+
+	bucket, err := blob.OpenBucket(ctx, u.String())
+	if err != nil {
+		return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", endpoint, err)
+	}
+	return bucket, nil
 }
 
 // legacySidecarPath gives the ".attrs" path an earlier version wrote for key,
@@ -210,6 +286,7 @@ func (b *Blob) clearLegacySidecar(key string) {
 	}
 }
 
+// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest.
 func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
 	b.clearLegacySidecar(path)
 
@@ -246,6 +323,7 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri
 	return size, hash, nil
 }
 
+// Open returns a reader for the object stored at path.
 func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
 	if localPath := b.localPath(path); localPath != "" {
 		r, err := os.Open(localPath)
@@ -273,6 +351,7 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
 // otherwise.
 func (b *Blob) ReadsSeekable() bool { return true }
 
+// Exists reports whether an object exists at path.
 func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
 	exists, err := b.bucket.Exists(ctx, path)
 	if err != nil {
@@ -298,8 +377,16 @@ func (b *Blob) Delete(ctx context.Context, path string) error {
 	return nil
 }
 
+// SignedURL returns a presigned URL granting time-limited GET access to path.
+// When signBucket is configured (S3 with WithSignEndpoint), it signs against
+// the public endpoint; when signEndpoint is set on other backends (e.g. Azure),
+// it rewrites the signed URL's scheme and host.
 func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
-	url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
+	bucket := b.bucket
+	if b.signBucket != nil {
+		bucket = b.signBucket
+	}
+	signed, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
 		Method: http.MethodGet,
 		Expiry: expiry,
 	})
@@ -309,9 +396,33 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration)
 		}
 		return "", fmt.Errorf("signing URL: %w", err)
 	}
-	return url, nil
+	if b.signBucket == nil && b.signEndpoint != "" {
+		signed = rewriteSignedURLHost(signed, b.signEndpoint)
+	}
+	return signed, nil
 }
 
+// rewriteSignedURLHost replaces the scheme and host of a signed URL with those
+// from baseURL, preserving the path and query (which carry the signature).
+// Returns signed unchanged if baseURL is empty or either URL fails to parse.
+func rewriteSignedURLHost(signed, baseURL string) string {
+	if baseURL == "" {
+		return signed
+	}
+	s, err := url.Parse(signed)
+	if err != nil {
+		return signed
+	}
+	b, err := url.Parse(baseURL)
+	if err != nil || b.Scheme == "" || b.Host == "" {
+		return signed
+	}
+	s.Scheme = b.Scheme
+	s.Host = b.Host
+	return s.String()
+}
+
+// Size returns the size in bytes of the object at path.
 func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
 	attrs, err := b.bucket.Attributes(ctx, path)
 	if err != nil {
@@ -323,6 +434,7 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
 	return attrs.Size, nil
 }
 
+// UsedSpace returns the total bytes stored across all objects in the bucket.
 func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
 	var total int64
 
@@ -370,10 +482,18 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err
 	return objects, nil
 }
 
+// Close closes the underlying storage bucket and any separate signing bucket.
 func (b *Blob) Close() error {
-	return b.bucket.Close()
+	err := b.bucket.Close()
+	if b.signBucket != nil {
+		if signErr := b.signBucket.Close(); err == nil {
+			err = signErr
+		}
+	}
+	return err
 }
 
+// URL returns the storage backend URL.
 func (b *Blob) URL() string {
 	return b.url
 }