From 909e2e464abe1a3193ef31bbf2f72c50c6b0fac5 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 19 Oct 2026 00:02:05 -0400 Subject: [PATCH 1/3] fix(s3): signatures for alternate hosts --- config.example.yaml | 9 ++-- internal/handler/handler.go | 28 +--------- internal/handler/handler_test.go | 68 ------------------------- internal/server/server.go | 7 ++- internal/storage/blob.go | 87 +++++++++++++++++++++++++++++--- internal/storage/blob_test.go | 62 +++++++++++++++++++++++ 6 files changed, 151 insertions(+), 110 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index e43e4346..1967213d 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -133,12 +133,9 @@ storage: # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m". direct_serve_ttl: "15m" - # Public base URL to substitute into presigned URLs. Set this when the - # proxy reaches storage at an internal address (127.0.0.1, a Docker - # service name) but clients must use a public hostname. Only scheme and - # host are used; the signed path and query are preserved. For S3/MinIO - # the reverse proxy at this address must forward requests with the - # internal Host header or the SigV4 signature will not validate. + # Public base URL for presigned URLs. Set this when the proxy reaches + # storage at an internal address (127.0.0.1, a Docker service name, or + # VPC DNS) but clients must use a public hostname or CDN. # direct_serve_base_url: "https://minio.example.com" # Database configuration diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 125b1174..6d38118d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -240,11 +240,7 @@ type Proxy struct { NPMFullMetadata bool DirectServe bool DirectServeTTL time.Duration - // DirectServeBaseURL, if set, replaces the scheme and host of presigned - // URLs so clients receive a public address even when the proxy reaches - // storage at an internal one. - DirectServeBaseURL string - HTTPClient *http.Client + HTTPClient *http.Client // RateLimits is the gate under the upstream clients. Artifact fetches // check it before calling Fetcher, so held hosts never reach its breaker. @@ -548,7 +544,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s if p.DirectServe { signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { - result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) + result.RedirectURL = signed p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -595,26 +591,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s return result, nil } -// rewriteSignedURLHost replaces the scheme and host of a signed URL with those -// from baseURL, preserving the path and query (which carry the signature). -// Returns signed unchanged if baseURL is empty or either URL fails to parse. -func rewriteSignedURLHost(signed, baseURL string) string { - if baseURL == "" { - return signed - } - s, err := url.Parse(signed) - if err != nil { - return signed - } - b, err := url.Parse(baseURL) - if err != nil || b.Scheme == "" || b.Host == "" { - return signed - } - s.Scheme = b.Scheme - s.Host = b.Host - return s.String() -} - func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) metrics.RecordArtifactCacheHit(ecosystem) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8ce636c9..3ae885e1 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -638,74 +638,6 @@ func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { } } -func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) { - proxy, db, store, _ := setupTestProxy(t) - seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") - - proxy.DirectServe = true - proxy.DirectServeBaseURL = "https://cdn.example.com" - store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" - - result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" - if result.RedirectURL != want { - t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want) - } -} - -func TestRewriteSignedURLHost(t *testing.T) { - tests := []struct { - name string - signed string - baseURL string - want string - }{ - { - "empty base url is no-op", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "", - "http://127.0.0.1:9000/bucket/key?sig=abc", - }, - { - "replaces scheme and host", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "https://cdn.example.com", - "https://cdn.example.com/bucket/key?sig=abc", - }, - { - "preserves path and query", - "http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", - "https://files.example.com", - "https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", - }, - { - "ignores base url path", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "https://cdn.example.com/ignored", - "https://cdn.example.com/bucket/key?sig=abc", - }, - { - "invalid base url is no-op", - "http://127.0.0.1:9000/bucket/key?sig=abc", - "://bad", - "http://127.0.0.1:9000/bucket/key?sig=abc", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := rewriteSignedURLHost(tt.signed, tt.baseURL) - if got != tt.want { - t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want) - } - }) - } -} - func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597b..4466d06f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -189,7 +189,11 @@ func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server, // Fall back to file:// with Path storageURL = "file://" + cfg.Storage.Path //nolint:staticcheck // backwards compat } - store, err := storage.OpenBucket(context.Background(), storageURL) + var storageOpts []storage.Option + if cfg.Storage.DirectServe && cfg.Storage.DirectServeBaseURL != "" { + storageOpts = append(storageOpts, storage.WithSignEndpoint(cfg.Storage.DirectServeBaseURL)) + } + store, err := storage.OpenBucket(context.Background(), storageURL, storageOpts...) if err != nil { _ = db.Close() return nil, fmt.Errorf("initializing storage: %w", err) @@ -317,7 +321,6 @@ func (s *Server) serve(listener net.Listener) error { proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.DirectServe = s.cfg.Storage.DirectServe proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() - proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts if err := ConfigureDenylist(s.cfg, proxy); err != nil { return err diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 81af78f2..cf3ee0f6 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -30,11 +31,13 @@ const osWindows = "windows" const ( uploadBufferSize = 5 << 20 uploadConcurrency = 2 -) -// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an -// earlier version wrote. -const attrsExt = ".attrs" + s3Scheme = "s3://" + + // attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an + // earlier version wrote. + attrsExt = ".attrs" +) // Blob implements Storage using gocloud.dev/blob. // Supports local filesystem (file://) and S3 (s3://) URLs. @@ -42,11 +45,29 @@ type Blob struct { bucket *blob.Bucket url string + // signBucket, when set, is used by SignedURL to sign S3 presigned URLs for + // a public endpoint rather than the internal storage endpoint. + signBucket *blob.Bucket + // fileRoot is the directory backing a file:// bucket, empty for cloud // backends. Used only to clear sidecars an earlier version wrote. fileRoot string } +// Option configures OpenBucket. +type Option func(*options) + +type options struct { + signEndpoint string +} + +// WithSignEndpoint configures S3 presigned URLs to be signed for the scheme and +// host of signEndpoint (e.g. "https://storage.example.com") instead of the +// storage endpoint, since SigV4 includes the Host header in the signature. +func WithSignEndpoint(signEndpoint string) Option { + return func(o *options) { o.signEndpoint = signEndpoint } +} + // OpenBucket opens a blob bucket from a URL. // // Supported URL schemes: @@ -60,7 +81,12 @@ type Blob struct { // For local filesystem, the directory is created if it doesn't exist. // //nolint:ireturn // The URL scheme selects the storage implementation. -func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { +func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, error) { + var o options + for _, opt := range opts { + opt(&o) + } + if strings.HasPrefix(urlStr, "gs://") { return OpenGCS(ctx, urlStr) } @@ -124,7 +150,42 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return nil, fmt.Errorf("opening bucket: %w", err) } - return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil + b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot} + + if o.signEndpoint != "" && strings.HasPrefix(urlStr, s3Scheme) { + signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint) + if err != nil { + _ = bucket.Close() + return nil, err + } + b.signBucket = signBucket + } + + return b, nil +} + +// openSignBucket opens an S3 bucket with the endpoint overridden to signEndpoint +// while preserving all other storage URL parameters (region, use_path_style, etc.). +func openSignBucket(ctx context.Context, urlStr, signEndpoint string) (*blob.Bucket, error) { + sign, err := url.Parse(signEndpoint) + if err != nil || sign.Scheme == "" || sign.Host == "" { + return nil, fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint) + } + + u, err := url.Parse(urlStr) + if err != nil { + return nil, fmt.Errorf("parsing storage URL: %w", err) + } + endpoint := sign.Scheme + "://" + sign.Host + q := u.Query() + q.Set("endpoint", endpoint) + u.RawQuery = q.Encode() + + bucket, err := blob.OpenBucket(ctx, u.String()) + if err != nil { + return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", endpoint, err) + } + return bucket, nil } // legacySidecarPath gives the ".attrs" path an earlier version wrote for key, @@ -299,7 +360,11 @@ func (b *Blob) Delete(ctx context.Context, path string) error { } func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { - url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + bucket := b.bucket + if b.signBucket != nil { + bucket = b.signBucket + } + url, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, Expiry: expiry, }) @@ -371,7 +436,13 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err } func (b *Blob) Close() error { - return b.bucket.Close() + err := b.bucket.Close() + if b.signBucket != nil { + if signErr := b.signBucket.Close(); err == nil { + err = signErr + } + } + return err } func (b *Blob) URL() string { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 1a764de7..4811c7eb 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "io/fs" + "net/url" "os" "path/filepath" "runtime" @@ -260,6 +261,67 @@ func TestBlobSignedURLUnsupported(t *testing.T) { } } +func TestBlobSignedURLSignEndpoint(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + + const ( + storageURL = "s3://bucket?region=us-1&endpoint=http://127.0.0.1:9000&use_path_style=true" + signEndpoint = "https://cdn.example.com" + artifactPath = "npm/lodash/4.17.21/lodash-4.17.21.tgz" + ) + + tests := []struct { + name string + opts []Option + host string + }{ + { + "storage endpoint without a sign endpoint", + nil, + "127.0.0.1:9000", + }, + { + "configured sign endpoint, reached over another host", + []Option{WithSignEndpoint(signEndpoint)}, + "cdn.example.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + b, err := OpenBucket(ctx, storageURL, tt.opts...) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + defer func() { _ = b.Close() }() + + signed, err := b.SignedURL(ctx, artifactPath, time.Minute) + if err != nil { + t.Fatalf("SignedURL failed: %v", err) + } + + u, err := url.Parse(signed) + if err != nil { + t.Fatalf("parsing signed URL %q: %v", signed, err) + } + if u.Host != tt.host { + t.Errorf("signed URL %q host = %q, want %q", signed, u.Host, tt.host) + } + // The bucket and key stay where the storage URL puts them, so the + // request the client makes still names the object. + if want := "/bucket/" + artifactPath; u.Path != want { + t.Errorf("signed URL path = %q, want %q", u.Path, want) + } + if u.Query().Get("X-Amz-Signature") == "" { + t.Errorf("signed URL %q carries no signature", signed) + } + }) + } +} + func TestBlobOverwrite(t *testing.T) { b := createTestBlob(t) ctx := context.Background() From f82ec690068dce9ec4e5edcad407ea35795b8025 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 6 Nov 2026 20:06:36 -0500 Subject: [PATCH 2/3] fix(storage): restore Azure/GCS direct serve signing --- internal/storage/blob.go | 62 +++++++++++++++++++++------ internal/storage/blob_test.go | 79 +++++++++++++++++++++++++++++++++++ internal/storage/gcs.go | 10 +++-- internal/storage/gcs_test.go | 56 +++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 15 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index cf3ee0f6..eeb8b3b9 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -49,6 +49,10 @@ type Blob struct { // a public endpoint rather than the internal storage endpoint. signBucket *blob.Bucket + // signEndpoint, when set on non-S3 backends, replaces the scheme and host + // of presigned URLs returned by SignedURL. + signEndpoint string + // fileRoot is the directory backing a file:// bucket, empty for cloud // backends. Used only to clear sidecars an earlier version wrote. fileRoot string @@ -61,9 +65,11 @@ type options struct { signEndpoint string } -// WithSignEndpoint configures S3 presigned URLs to be signed for the scheme and -// host of signEndpoint (e.g. "https://storage.example.com") instead of the -// storage endpoint, since SigV4 includes the Host header in the signature. +// WithSignEndpoint configures presigned URLs to use the scheme and host of +// signEndpoint (e.g. "https://storage.example.com") instead of the storage +// endpoint. For S3, the URL is signed directly against signEndpoint so the +// SigV4 Host signature matches; for other backends, the signed URL's scheme +// and host are rewritten. func WithSignEndpoint(signEndpoint string) Option { return func(o *options) { o.signEndpoint = signEndpoint } } @@ -88,7 +94,12 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er } if strings.HasPrefix(urlStr, "gs://") { - return OpenGCS(ctx, urlStr) + g, err := OpenGCS(ctx, urlStr) + if err != nil { + return nil, err + } + g.signEndpoint = o.signEndpoint + return g, nil } var fileRoot string @@ -152,13 +163,17 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot} - if o.signEndpoint != "" && strings.HasPrefix(urlStr, s3Scheme) { - signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint) - if err != nil { - _ = bucket.Close() - return nil, err + if o.signEndpoint != "" { + if strings.HasPrefix(urlStr, s3Scheme) { + signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint) + if err != nil { + _ = bucket.Close() + return nil, err + } + b.signBucket = signBucket + } else { + b.signEndpoint = o.signEndpoint } - b.signBucket = signBucket } return b, nil @@ -364,7 +379,7 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) if b.signBucket != nil { bucket = b.signBucket } - url, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + signed, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, Expiry: expiry, }) @@ -374,7 +389,30 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) } return "", fmt.Errorf("signing URL: %w", err) } - return url, nil + if b.signBucket == nil && b.signEndpoint != "" { + signed = rewriteSignedURLHost(signed, b.signEndpoint) + } + return signed, nil +} + +// rewriteSignedURLHost replaces the scheme and host of a signed URL with those +// from baseURL, preserving the path and query (which carry the signature). +// Returns signed unchanged if baseURL is empty or either URL fails to parse. +func rewriteSignedURLHost(signed, baseURL string) string { + if baseURL == "" { + return signed + } + s, err := url.Parse(signed) + if err != nil { + return signed + } + b, err := url.Parse(baseURL) + if err != nil || b.Scheme == "" || b.Host == "" { + return signed + } + s.Scheme = b.Scheme + s.Host = b.Host + return s.String() } func (b *Blob) Size(ctx context.Context, path string) (int64, error) { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 4811c7eb..bffe09b0 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -322,6 +322,85 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { } } +func TestAzureSignedURLSignEndpoint(t *testing.T) { + t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") + t.Setenv("AZURE_STORAGE_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==") + t.Setenv("AZURE_STORAGE_PROTOCOL", "http") + t.Setenv("AZURE_STORAGE_IS_LOCAL_EMULATOR", "true") + + ctx := context.Background() + b, err := OpenBucket(ctx, "azblob://bucket?domain=127.0.0.1:10000", WithSignEndpoint("https://cdn.example.com")) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + defer func() { _ = b.Close() }() + + signed, err := b.SignedURL(ctx, "npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) + if err != nil { + t.Fatalf("SignedURL failed: %v", err) + } + + u, err := url.Parse(signed) + if err != nil { + t.Fatalf("parsing signed URL %q: %v", signed, err) + } + if u.Scheme != "https" || u.Host != "cdn.example.com" { + t.Errorf("signed URL = %q, want https://cdn.example.com", signed) + } + if u.Query().Get("sig") == "" { + t.Errorf("signed URL %q carries no SAS signature", signed) + } +} + +func TestRewriteSignedURLHost(t *testing.T) { + tests := []struct { + name string + signed string + baseURL string + want string + }{ + { + "empty base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + { + "replaces scheme and host", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "preserves path and query", + "http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + "https://files.example.com", + "https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + }, + { + "ignores base url path", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com/ignored", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "invalid base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "://bad", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rewriteSignedURLHost(tt.signed, tt.baseURL) + if got != tt.want { + t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want) + } + }) + } +} + func TestBlobOverwrite(t *testing.T) { b := createTestBlob(t) ctx := context.Background() diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index a096c18c..43b14e55 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -13,8 +13,9 @@ import ( // GCS adapts a Google Cloud Storage bucket to Storage. type GCS struct { - bucket *gcstorage.Bucket - url string + bucket *gcstorage.Bucket + url string + signEndpoint string } // OpenGCS opens a Google Cloud Storage bucket from a gs:// URL. @@ -64,7 +65,10 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { return "", ErrSignedURLUnsupported } - return u, err + if err != nil { + return "", err + } + return rewriteSignedURLHost(u, g.signEndpoint), nil } func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go index 1bd8a407..461fb866 100644 --- a/internal/storage/gcs_test.go +++ b/internal/storage/gcs_test.go @@ -2,14 +2,20 @@ package storage import ( "context" + "crypto/rand" + "crypto/rsa" "crypto/sha256" + "crypto/x509" "encoding/hex" "encoding/json" + "encoding/pem" "errors" "io" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "sort" "strconv" "strings" @@ -89,6 +95,56 @@ func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { } } +func TestGCSSignedURLSignEndpoint(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generating RSA key: %v", err) + } + pemBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + credsJSON, err := json.Marshal(map[string]string{ + "type": "service_account", + "client_email": "test@example.iam.gserviceaccount.com", + "private_key": string(pemBytes), + }) + if err != nil { + t.Fatalf("marshaling creds: %v", err) + } + credsFile := filepath.Join(t.TempDir(), "creds.json") + if err := os.WriteFile(credsFile, credsJSON, 0o600); err != nil { + t.Fatalf("writing creds: %v", err) + } + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credsFile) + t.Setenv("STORAGE_EMULATOR_HOST", "") + + ctx := context.Background() + store, err := OpenBucket(ctx, "gs://test-bucket", WithSignEndpoint("https://cdn.example.com")) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + defer func() { _ = store.Close() }() + + signed, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute) + if err != nil { + t.Fatalf("SignedURL failed: %v", err) + } + u, err := url.Parse(signed) + if err != nil { + t.Fatalf("parsing signed URL %q: %v", signed, err) + } + if u.Scheme != "https" || u.Host != "cdn.example.com" { + t.Errorf("signed URL = %q, want https://cdn.example.com", signed) + } + if u.Path != "/test-bucket/npm/pkg/file.tgz" { + t.Errorf("signed URL path = %q, want /test-bucket/npm/pkg/file.tgz", u.Path) + } + if u.Query().Get("Signature") == "" { + t.Errorf("signed URL %q carries no signature", signed) + } +} + type fakeGCSServer struct { t *testing.T objects map[string]string From 5c1eaa9fe5a1189476089a414178da0c910690db Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 13:57:40 -0500 Subject: [PATCH 3/3] chore: test reliability / godoc on methods Code review feedback --- internal/storage/blob.go | 11 +++++++++++ internal/storage/blob_test.go | 14 ++++++++++++++ internal/storage/gcs.go | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index eeb8b3b9..7519da8e 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -286,6 +286,7 @@ func (b *Blob) clearLegacySidecar(key string) { } } +// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { b.clearLegacySidecar(path) @@ -322,6 +323,7 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri return size, hash, nil } +// Open returns a reader for the object stored at path. func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { if localPath := b.localPath(path); localPath != "" { r, err := os.Open(localPath) @@ -349,6 +351,7 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { // otherwise. func (b *Blob) ReadsSeekable() bool { return true } +// Exists reports whether an object exists at path. func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { exists, err := b.bucket.Exists(ctx, path) if err != nil { @@ -374,6 +377,10 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return nil } +// SignedURL returns a presigned URL granting time-limited GET access to path. +// When signBucket is configured (S3 with WithSignEndpoint), it signs against +// the public endpoint; when signEndpoint is set on other backends (e.g. Azure), +// it rewrites the signed URL's scheme and host. func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { bucket := b.bucket if b.signBucket != nil { @@ -415,6 +422,7 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } +// Size returns the size in bytes of the object at path. func (b *Blob) Size(ctx context.Context, path string) (int64, error) { attrs, err := b.bucket.Attributes(ctx, path) if err != nil { @@ -426,6 +434,7 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) { return attrs.Size, nil } +// UsedSpace returns the total bytes stored across all objects in the bucket. func (b *Blob) UsedSpace(ctx context.Context) (int64, error) { var total int64 @@ -473,6 +482,7 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err return objects, nil } +// Close closes the underlying storage bucket and any separate signing bucket. func (b *Blob) Close() error { err := b.bucket.Close() if b.signBucket != nil { @@ -483,6 +493,7 @@ func (b *Blob) Close() error { return err } +// URL returns the storage backend URL. func (b *Blob) URL() string { return b.url } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index bffe09b0..7a46413b 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -262,6 +262,19 @@ func TestBlobSignedURLUnsupported(t *testing.T) { } func TestBlobSignedURLSignEndpoint(t *testing.T) { + awsDir := t.TempDir() + awsConfigFile := filepath.Join(awsDir, "config") + awsCredsFile := filepath.Join(awsDir, "credentials") + if err := os.WriteFile(awsConfigFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS config file: %v", err) + } + if err := os.WriteFile(awsCredsFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS credentials file: %v", err) + } + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", awsConfigFile) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", awsCredsFile) t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") @@ -323,6 +336,7 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { } func TestAzureSignedURLSignEndpoint(t *testing.T) { + t.Setenv("AZURE_STORAGE_SAS_TOKEN", "") t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") t.Setenv("AZURE_STORAGE_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==") t.Setenv("AZURE_STORAGE_PROTOCOL", "http") diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index 43b14e55..c2c98c0b 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -27,6 +27,7 @@ func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { return &GCS{bucket: bucket, url: urlStr}, nil } +// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { h := sha256.New() size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h)) @@ -36,6 +37,7 @@ func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, strin return size, hex.EncodeToString(h.Sum(nil)), nil } +// Open returns a reader for the object stored at path. func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { r, err := g.bucket.Open(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -44,14 +46,17 @@ func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { return r, err } +// Exists reports whether an object exists at path. func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { return g.bucket.Exists(ctx, path) } +// Delete removes the object at path. func (g *GCS) Delete(ctx context.Context, path string) error { return g.bucket.Delete(ctx, path) } +// Size returns the size in bytes of the object at path. func (g *GCS) Size(ctx context.Context, path string) (int64, error) { size, err := g.bucket.Size(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -60,6 +65,8 @@ func (g *GCS) Size(ctx context.Context, path string) (int64, error) { return size, err } +// SignedURL returns a presigned URL granting time-limited GET access to path, +// rewriting the scheme and host to signEndpoint when configured. func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { u, err := g.bucket.SignedURL(ctx, path, expiry) if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { @@ -71,10 +78,12 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) return rewriteSignedURLHost(u, g.signEndpoint), nil } +// UsedSpace returns the total bytes stored across all objects in the bucket. func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { return g.bucket.UsedSpace(ctx) } +// ListPrefix returns object metadata for keys under a prefix. func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { objects, err := g.bucket.ListPrefix(ctx, prefix) if err != nil { @@ -92,10 +101,12 @@ func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, erro return result, nil } +// Close releases any resources held by the GCS client. func (g *GCS) Close() error { return nil } +// URL returns the storage backend URL. func (g *GCS) URL() string { return g.url }