diff --git a/README.md b/README.md index 2a08eaac..2eea25dc 100644 --- a/README.md +++ b/README.md @@ -1194,6 +1194,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 # Mirror a container image for one platform proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 + +# Mirror a commit of an upstream.git repository into the git cache +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 ``` The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 70ae8e6a..f9ab8491 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -595,7 +595,9 @@ func runMirror() { return } + closeGit := useCommandGit(cfg, proxy, m, logger) progress, err := m.Run(ctx, source) + closeGit() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) @@ -634,6 +636,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. return m } +// useCommandGit opens the git cache for the pkg:git items of proxy mirror, +// when upstream.git is configured, and returns the function that closes it. +// Where it can't be opened, such as while the running server holds +// git.cache_dir, those items fail with the reason and the rest are mirrored. +func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() { + git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger) + if err != nil { + m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)} + return func() {} + } + if git == nil { + return func() {} + } + m.Git = git + return func() { + if err := closeGit(); err != nil { + logger.Warn("closing the git cache", "error", err) + } + } +} + // mirrorExitCode returns the exit status for a finished mirror run: 1 when any // package version failed to mirror, 0 otherwise. func mirrorExitCode(progress *mirror.Progress) int { diff --git a/cmd/proxy/mirror_git_test.go b/cmd/proxy/mirror_git_test.go new file mode 100644 index 00000000..27895368 --- /dev/null +++ b/cmd/proxy/mirror_git_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// gitMirrorUpstream serves team/repo, with one commit on main, from git +// http-backend, and returns the server and the commit. +func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) { + t.Helper() + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + backend := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) { + backend = path + } + } + if backend == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + git("", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + git("", "clone", "--quiet", bare, work) + git(work, "commit", "--quiet", "--allow-empty", "-m", "first") + git(work, "push", "--quiet", "origin", "HEAD:main") + head := git(work, "rev-parse", "HEAD") + + server := httptest.NewServer(&cgi.Handler{ + Path: backend, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + }) + t.Cleanup(server.Close) + return server, head +} + +func fileExistsForTest(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// writeGitMirrorConfig writes a config with upstream.git "up" at +// upstreamURL and the git cache, storage and database under dir. +func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " allow_loopback: true\n" + + " git:\n" + + " up: " + upstreamURL + "\n" + + "git:\n" + + " cache_dir: " + filepath.Join(dir, "git") + "\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func TestMirrorCommandCachesGitCommit(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git")) + if len(cached) != 1 { + t.Errorf("cached repositories %v, want team/repo.git", cached) + } + + output = runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 skipped") { + t.Errorf("second run output:\n%s", output) + } +} + +// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the +// mirror API, while the running server holds git.cache_dir. +func TestMirrorCommandGitCacheInUse(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: filepath.Join(dir, "git"), + RefreshWait: time.Second, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = cache.Close() }() + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 1, "-config", configPath, purl) + for _, want := range []string{"in use by another process", "mirror API", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index d8b294c5..655b3098 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 38765e08..d447c0c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1877,6 +1877,20 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 - Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. - Casks, source archives and taps other than homebrew/core aren't mirrored. +### Git repositories + +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: + +```bash +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl +``` + +- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. +- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. + ### Scheduled refresh Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go new file mode 100644 index 00000000..b5bde40f --- /dev/null +++ b/internal/gitcache/mirror.go @@ -0,0 +1,118 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "strings" + "time" +) + +// MirrorResult is what Mirror did. +type MirrorResult struct { + // Cached reports that the repository, and the commit if one was given, + // were cached already. + Cached bool + // Bytes is how much the cached repository grew. + Bytes int64 +} + +// Mirror brings the repository cached under key up to date with repoURL, +// cloning it if needed, and waits for that to finish. With commit, an object +// ID, it also makes sure the commit is cached. One on no branch or tag is +// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and +// is kept as long as a commit a client fetched by ID. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { + if commit != "" && !isObjectID(commit) { + return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) + } + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) + if err != nil { + return MirrorResult{}, err + } + defer lease.Release() + rp := lease.repo + cached := lease.hit + before := int64(0) + var refreshErr error + // Open cloned a repository that wasn't cached; one that was is + // refreshed here, waiting for the fetch a client would not wait for. + if cached { + before = rp.size.Load() + refresh := c.startRefresh(rp, repoURL) + select { + case <-refresh.done: + refreshErr = refresh.err + case <-ctx.Done(): + return MirrorResult{}, ctx.Err() + } + } + + // A refresh past max_repo_size gets the repository dropped once it is + // released, commit and all. + if errors.Is(refreshErr, ErrTooLarge) { + return MirrorResult{}, refreshErr + } + if commit != "" { + if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { + return MirrorResult{}, err + } + } + // A refresh that failed leaves the commit cached all the same. + if refreshErr != nil && commit == "" { + return MirrorResult{}, refreshErr + } + grown := max(rp.size.Load()-before, 0) + return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil +} + +// ensureCommit makes sure the repository has commit, fetching it by ID when +// it doesn't and fetchWants allows it. A commit on no branch or tag that is +// cached already gets a new wanted ref, so it is kept for wantedMaxAge from +// this run on. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error { + missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) + if err != nil { + return err + } + if len(missing) == 0 { + return c.renewWanted(ctx, rp, commit) + } + if !fetchWants { + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { + return err + } + // The upstream refuses a commit it doesn't have. + return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + } + // fetchMissingWants skips a commit that failed moments ago. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { + return err + } + if len(missing) > 0 { + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + return nil +} + +// renewWanted writes a new wanted ref for commit when no branch or tag +// includes it. +func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit}) + if err != nil || !unmerged[commit] { + return err + } + cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go new file mode 100644 index 00000000..86f652f9 --- /dev/null +++ b/internal/gitcache/mirror_test.go @@ -0,0 +1,214 @@ +package gitcache + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func TestMirrorClonesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("first mirror = %+v, want fetched bytes", got) + } + if !strings.Contains(tc.cachedRefs(t), "refs/tags/v1") { + t.Errorf("cached refs %q lack the tag", tc.cachedRefs(t)) + } + + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } +} + +func TestMirrorFetchesCommitOnNoBranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("mirror = %+v, want fetched", got) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Errorf("wanted refs %v, want one", refs) + } + + sessions := upstream.gitSessions.Load() + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if n := upstream.gitSessions.Load() - sessions; n != 0 { + t.Errorf("second mirror fetched %d times, want none", n) + } +} + +func TestMirrorFindsCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if err != nil || got.Cached { + t.Errorf("mirror = %+v, %v, want fetched", got, err) + } +} + +func TestMirrorDoesNotFetchByIDWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit on no branch with fetching by ID off = %v, want ErrNotFound", err) + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none", refs) + } +} + +func TestMirrorReportsMissingCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.Repeat("ab", 20), true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit the upstream lacks = %v, want ErrNotFound", err) + } +} + +func TestMirrorRefusesInvalidCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "main", true); err == nil { + t.Error("mirrored a branch name as a commit") + } +} + +func TestMirrorCountsRefreshThatFetched(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("mirror after a push = %+v, want fetched bytes", got) + } +} + +// TestMirrorRenewsWantedCommit checks mirroring a commit on no branch again +// makes its 7 days count from that run, not from the first fetch. +func TestMirrorRenewsWantedCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + runGit(t, cached, "update-ref", "-d", ref) + } + old := wantedRef(sha, time.Now().Add(-wantedMaxAge+time.Hour)) + runGit(t, cached, "update-ref", old, sha) + + start := time.Now().Add(-time.Second) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Fatalf("second mirror = %+v, %v, want cached", got, err) + } + if !slices.ContainsFunc(tc.wantedRefs(t), func(ref string) bool { + return ref != old && wantedRefTime(t, ref, sha).After(start) + }) { + t.Errorf("wanted refs %v, want one written by the second mirror", tc.wantedRefs(t)) + } +} + +func TestMirrorDoesNotPinCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + for range 2 { + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none for a commit on a branch", refs) + } +} + +func wantedRefTime(t *testing.T, ref, oid string) time.Time { + t.Helper() + created, got, _ := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if err != nil || got != oid { + return time.Time{} + } + return time.Unix(unix, 0) +} + +// TestMirrorReportsRefreshTooLarge checks a commit that is cached still +// fails to mirror when the refresh doesn't fit in max_repo_size, since the +// repository is dropped once released. +func TestMirrorReportsRefreshTooLarge(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true) + if !errors.Is(err, ErrTooLarge) { + t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) + } +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index c715f2f3..8755ba1b 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -161,19 +161,10 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe } var mergedErr error if fetched && len(keep) > 0 { - // rev-list prints what the wanted commits, read from stdin, reach - // and no branch or tag does, so a wanted commit it leaves out is - // merged. - cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") - cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") - out, err := cmd.Output() + unmerged, err := c.unmergedCommits(ctx, dir, slices.Sorted(maps.Keys(keep))) if err != nil { mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) } else { - unmerged := make(map[string]bool) - for line := range strings.Lines(string(out)) { - unmerged[strings.TrimSpace(line)] = true - } for oid, ref := range keep { if !unmerged[oid] { drop = append(drop, ref) @@ -196,6 +187,24 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe return mergedErr } +// unmergedCommits returns which of commits no cached branch or tag +// includes. +func (c *Cache) unmergedCommits(ctx context.Context, dir string, commits []string) (map[string]bool, error) { + // rev-list prints what the commits, read from stdin, reach and no + // branch or tag does, so a commit it leaves out is merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(commits, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + return unmerged, nil +} + // withoutFailedWants drops the object IDs a recent fetch for the // repository could not get. func (c *Cache) withoutFailedWants(key string, oids []string) []string { diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go new file mode 100644 index 00000000..709df313 --- /dev/null +++ b/internal/handler/git_mirror.go @@ -0,0 +1,57 @@ +package handler + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// Mirror caches the repository at location, a host and path such as +// github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git +// host serving it, and with commit, a full commit ID, that commit too. +// Clones through /git are then served from that copy. +func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { + repo, err := h.mirrorRepository(location) + if err != nil { + return gitcache.MirrorResult{}, err + } + return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) +} + +// mirrorRepository resolves location to a repository of the upstream.git +// host whose URL it starts with, the longest such URL if several do. +func (h *GitHandler) mirrorRepository(location string) (gitRepository, error) { + var best, bestPrefix, rest string + for name, upstream := range h.upstreams { + prefix := gitLocationPrefix(upstream.URL) + path, ok := strings.CutPrefix(location, prefix) + if prefix == "" || !ok || path == "" { + continue + } + if len(prefix) > len(bestPrefix) || (len(prefix) == len(bestPrefix) && name < best) { + best, bestPrefix, rest = name, prefix, path + } + } + if best == "" { + return gitRepository{}, fmt.Errorf("no upstream.git host serves %s", location) + } + repo, status, msg := h.repository(best + "/" + rest) + if status != 0 { + return gitRepository{}, fmt.Errorf("%s: %s", location, msg) + } + return repo, nil +} + +// gitLocationPrefix is how a pkg:git PURL names the repositories under an +// upstream URL: its host and path, ending in a slash. Empty for a URL +// without a host. +func gitLocationPrefix(upstreamURL string) string { + u, err := url.Parse(upstreamURL) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) + strings.TrimSuffix(u.Path, "/") + "/" +} diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go new file mode 100644 index 00000000..4ebcf982 --- /dev/null +++ b/internal/handler/git_mirror_test.go @@ -0,0 +1,73 @@ +package handler + +import ( + "context" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGitMirrorLocation(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/**"}}, + "sub": {URL: "https://git.example/r/"}, + "nested": {URL: "https://git.example/r/deep"}, + }}) + tests := []struct { + location string + ok bool + key, repoURL string + }{ + {"github.com/acme/widgets", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets"}, + {"github.com/acme/widgets.git", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets.git"}, + {"gitlab.wikimedia.org/mediawiki/skins/Vector", true, h.upstreams["wikimedia"].dir + "/mediawiki/skins/Vector.git", "https://gitlab.wikimedia.org/mediawiki/skins/Vector"}, + {"gitlab.wikimedia.org/other/repo", false, "", ""}, + {"git.example/r/a", true, h.upstreams["sub"].dir + "/a.git", "https://git.example/r/a"}, + // The longest matching upstream URL wins. + {"git.example/r/deep/a", true, h.upstreams["nested"].dir + "/a.git", "https://git.example/r/deep/a"}, + {"git.example/other", false, "", ""}, + {"git.example/r", false, "", ""}, + {"github.com.evil/acme/widgets", false, "", ""}, + {"github.com/acme/../widgets", false, "", ""}, + {"codeberg.org/forgejo/forgejo", false, "", ""}, + } + for _, tt := range tests { + t.Run(tt.location, func(t *testing.T) { + repo, err := h.mirrorRepository(tt.location) + if (err == nil) != tt.ok { + t.Fatalf("error %v, want ok %v", err, tt.ok) + } + if tt.ok && (repo.key != tt.key || repo.url != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", repo.key, repo.url, tt.key, tt.repoURL) + } + }) + } +} + +func TestGitMirrorCachesCommitForClones(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + got, err = g.handler.Mirror(context.Background(), location, sha) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + + // With the upstream gone, a clone through the proxy gets the mirrored copy. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.url(), clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != sha { + t.Errorf("cloned HEAD %s, want %s", head, sha) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index 0aeff4ba..cf553d60 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -131,7 +131,12 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { type gitTestSetup struct { work string server *httptest.Server - logs *syncLog + // upstream serves the repository, at upstreamURL/repo, and handler is + // the GitHandler under test. + upstream *httptest.Server + upstreamURL string + handler *GitHandler + logs *syncLog // slow delays every upstream request by half a second. slow atomic.Bool // active counts the requests the handler is serving. @@ -172,6 +177,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe backend.ServeHTTP(w, r) })) t.Cleanup(upstream.Close) + g.upstream, g.upstreamURL = upstream, upstream.URL logger := slog.New(slog.NewTextHandler(g.logs, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -190,6 +196,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe if writeStall > 0 { h.writeStall = writeStall } + g.handler = h routes := http.StripPrefix("/git", h.Routes()) g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { g.active.Add(1) diff --git a/internal/mirror/git.go b/internal/mirror/git.go new file mode 100644 index 00000000..a4908dc5 --- /dev/null +++ b/internal/mirror/git.go @@ -0,0 +1,49 @@ +package mirror + +import ( + "context" + "errors" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// GitMirrorer caches Git repositories for pkg:git items. location is the +// repository's host and path, such as codeberg.org/forgejo/forgejo, and +// commit a full commit ID, or empty to cache the repository's branches and +// tags only. +type GitMirrorer interface { + Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) +} + +// UnavailableGit fails every pkg:git item with Reason. +type UnavailableGit struct { + Reason string +} + +func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.MirrorResult, error) { + return gitcache.MirrorResult{}, errors.New(u.Reason) +} + +var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") + +// mirrorGit caches a pkg:git repository, and the commit its version names, +// in the cache /git serves clones from. +func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + if m.Git == nil { + m.recordFailure(tracker, pv, errNoGitUpstream) + return + } + result, err := m.Git.Mirror(ctx, pv.Name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if result.Cached { + tracker.skipped.Add(1) + m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + return + } + tracker.completed.Add(1) + tracker.bytes.Add(result.Bytes) + m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) +} diff --git a/internal/mirror/git_test.go b/internal/mirror/git_test.go new file mode 100644 index 00000000..aa3aed4e --- /dev/null +++ b/internal/mirror/git_test.go @@ -0,0 +1,92 @@ +package mirror + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +type gitCall struct{ location, commit string } + +type fakeGit struct { + mu sync.Mutex + calls []gitCall + result gitcache.MirrorResult + err error +} + +func (f *fakeGit) Mirror(_ context.Context, location, commit string) (gitcache.MirrorResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, gitCall{location, commit}) + return f.result, f.err +} + +const gitTestCommit = "a72d2c07cfca03b55371089de6aa230d8c951fa0" + +func TestMirrorRunCountsGitRepositories(t *testing.T) { + tests := []struct { + name string + result gitcache.MirrorResult + err error + wantCompleted, wantSkipped, wantFailed int64 + }{ + {name: "fetched", result: gitcache.MirrorResult{Bytes: 42}, wantCompleted: 1}, + {name: "cached", result: gitcache.MirrorResult{Cached: true}, wantSkipped: 1}, + {name: "failed", err: errors.New("boom"), wantFailed: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{result: tt.result, err: tt.err} + m.Git = git + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { + t.Errorf("progress = %+v", progress) + } + if progress.Bytes != tt.result.Bytes { + t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) + } + want := gitCall{"codeberg.org/forgejo/forgejo", gitTestCommit} + if len(git.calls) != 1 || git.calls[0] != want { + t.Errorf("calls = %+v, want %+v", git.calls, want) + } + }) + } +} + +// TestMirrorRunGitRepositoryWithoutCommit checks a pkg:git PURL without a +// version mirrors the repository instead of listing versions. +func TestMirrorRunGitRepositoryWithoutCommit(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{} + m.Git = git + versions := &fakeVersions{err: errors.New("listed versions")} + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo"}, Versions: versions}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(versions.asked) != 0 { + t.Errorf("listed versions of %v", versions.asked) + } + if progress.Failed != 0 || len(git.calls) != 1 || git.calls[0] != (gitCall{"codeberg.org/forgejo/forgejo", ""}) { + t.Errorf("progress = %+v, calls = %+v", progress, git.calls) + } +} + +func TestMirrorRunGitWithoutUpstream(t *testing.T) { + m := setupTestMirror(t, 1) + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 { + t.Errorf("progress = %+v, want one failure", progress) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index d5ec9a59..c08b1fbd 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -32,6 +32,10 @@ type Mirror struct { // Charts mirrors helm charts. Without it they fail. Charts ChartMirrorer + // Git mirrors pkg:git repositories into the upstream.git cache. Without + // it they fail. + Git GitMirrorer + // Bottles resolves Homebrew formula versions to the images their // bottles are published under, which Images mirrors. Without it brew // PURLs fail. @@ -279,6 +283,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "git" { + m.mirrorGit(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 2c417053..c0441c4e 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -70,8 +70,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Images are never expanded to all tags, and conda packages without - // a channel aren't listed; both fail when they are mirrored. - if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + // a channel aren't listed; both fail when they are mirrored. A Git + // repository without a commit is mirrored as a whole. + if p.Version != "" || isImageEcosystem(ecosystem) || ecosystem == "git" || (ecosystem == "conda" && !strings.Contains(name, "/")) { pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() diff --git a/internal/server/git_test.go b/internal/server/git_test.go index d52f1a95..e59f30df 100644 --- a/internal/server/git_test.go +++ b/internal/server/git_test.go @@ -3,6 +3,7 @@ package server import ( "bytes" "context" + "encoding/json" "errors" "log/slog" "net" @@ -18,6 +19,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/mirror" ) func gitTestEnv(home string) []string { @@ -75,11 +77,11 @@ func (b *syncBuffer) String() string { return b.buf.String() } -// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole -// server, with credentials from upstream.auth that only the proxy holds. -func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { - root := t.TempDir() - bare := filepath.Join(root, "team", "repo.git") +// newGitTestRepo creates the bare repository {root}/{path}.git with one +// commit on main and returns the commit. +func newGitTestRepo(t *testing.T, root, path string) string { + t.Helper() + bare := filepath.Join(root, filepath.FromSlash(path)+".git") gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) work := t.TempDir() gitCommand(t, "", "clone", "--quiet", bare, work) @@ -89,21 +91,22 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { gitCommand(t, work, "add", "file.txt") gitCommand(t, work, "commit", "--quiet", "-m", "first") gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") - head := gitCommand(t, work, "rev-parse", "HEAD") + return gitCommand(t, work, "rev-parse", "HEAD") +} - backend := &cgi.Handler{ +// gitBackend serves the repositories under root with git http-backend. +func gitBackend(t *testing.T, root string) http.Handler { + t.Helper() + return &cgi.Handler{ Path: gitHTTPBackend(t), Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer git-secret" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - backend.ServeHTTP(w, r) - })) - defer upstream.Close() +} +// startGitTestServer starts the whole server with the git cache, after +// configure has set up cfg, and returns cfg and the server's log. +func startGitTestServer(t *testing.T, configure func(*config.Config)) (*config.Config, *syncBuffer) { + t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -115,21 +118,20 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { cfg.Database.Path = filepath.Join(tempDir, "proxy.db") cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") cfg.Upstream.AllowLoopback = true - cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} - cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} cfg.Git.CacheDir = filepath.Join(tempDir, "git") + configure(cfg) if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) } - var logs syncBuffer - proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + logs := &syncBuffer{} + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(logs, nil)), BuildInfo{Version: "test", Commit: "test"}) if err != nil { t.Fatalf("creating server: %v", err) } startErr := make(chan error, 1) go func() { startErr <- proxyServer.Start(listener) }() - defer func() { + t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := proxyServer.Shutdown(ctx); err != nil { @@ -138,7 +140,29 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) } - }() + }) + return cfg, logs +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + backend := gitBackend(t, root) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + cfg, logs := startGitTestServer(t, func(cfg *config.Config) { + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + }) clone := filepath.Join(t.TempDir(), "clone") gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) @@ -158,3 +182,60 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) } } + +// TestMirrorAPICachesGitCommit mirrors a pkg:git commit through the mirror +// API, then clones it through /git while the upstream is down. +func TestMirrorAPICachesGitCommit(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + upstream := httptest.NewServer(gitBackend(t, root)) + defer upstream.Close() + cfg, _ := startGitTestServer(t, func(cfg *config.Config) { + cfg.MirrorAPI = true + cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: upstream.URL}} + }) + + location := strings.TrimPrefix(upstream.URL, "http://") + "/team/repo" + body, err := json.Marshal(map[string][]string{"purls": {"pkg:git/" + location + "@" + head}}) + if err != nil { + t.Fatal(err) + } + resp, err := http.Post(cfg.BaseURL+"/api/mirror", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var created struct{ ID string } + err = json.NewDecoder(resp.Body).Decode(&created) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusAccepted { + t.Fatalf("creating job: status %d, %v", resp.StatusCode, err) + } + + var job mirror.Job + deadline := time.Now().Add(30 * time.Second) + for job.State != mirror.JobStateComplete && job.State != mirror.JobStateFailed && job.State != mirror.JobStateCanceled { + if time.Now().After(deadline) { + t.Fatalf("job did not finish: %+v", job) + } + time.Sleep(50 * time.Millisecond) + resp, err := http.Get(cfg.BaseURL + "/api/mirror/" + created.ID) + if err != nil { + t.Fatal(err) + } + err = json.NewDecoder(resp.Body).Decode(&job) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + } + if job.State != mirror.JobStateComplete || job.Progress.Completed != 1 { + t.Fatalf("job = %+v, want one repository mirrored", job) + } + + upstream.Close() + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/up/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 6d4599e7..81491d95 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -32,6 +32,9 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m.Images = s.newContainerHandler(proxy) m.Bottles = handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) + if s.git != nil { + m.Git = handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + } UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597b..86b4a4c2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -626,6 +626,21 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, return proxy, s.newContainerHandler(proxy) } +// NewMirrorGit opens the git cache for proxy mirror, with the upstream client +// serve uses, and returns the handler that mirrors pkg:git items into it and +// the function that closes it. Both are nil without upstream.git. +func NewMirrorGit(cfg *config.Config, proxy *handler.Proxy, logger *slog.Logger) (*handler.GitHandler, func() error, error) { + if len(cfg.Upstream.Git) == 0 { + return nil, nil, nil + } + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + _, artifactClient := s.newUpstreamClients(0) + if err := s.configureGitCache(artifactClient); err != nil { + return nil, nil, err + } + return handler.NewGitHandler(proxy, s.git, cfg.Upstream), s.git.Close, nil +} + // newContainerHandler builds the OCI handler with the configured default // registry, named upstreams, mirrored registries and Homebrew routes. func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler {