Mirror pkg:git commits into the git cache

README.md

@@ -1194,6 +1194,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
 
 # Mirror a container image for one platform
 proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
+
+# Mirror a commit of an upstream.git repository into the git cache
+proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
 ```
 
 The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror.

cmd/proxy/main.go

@@ -595,7 +595,9 @@ func runMirror() {
 		return
 	}
 
+	closeGit := useCommandGit(cfg, proxy, m, logger)
 	progress, err := m.Run(ctx, source)
+	closeGit()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)
@@ -634,6 +636,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.
 	return m
 }
 
+// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
+// when upstream.git is configured, and returns the function that closes it.
+// Where it can't be opened, such as while the running server holds
+// git.cache_dir, those items fail with the reason and the rest are mirrored.
+func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
+	git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
+	if err != nil {
+		m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
+		return func() {}
+	}
+	if git == nil {
+		return func() {}
+	}
+	m.Git = git
+	return func() {
+		if err := closeGit(); err != nil {
+			logger.Warn("closing the git cache", "error", err)
+		}
+	}
+}
+
 // mirrorExitCode returns the exit status for a finished mirror run: 1 when any
 // package version failed to mirror, 0 otherwise.
 func mirrorExitCode(progress *mirror.Progress) int {

cmd/proxy/mirror_git_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"net/http"
+	"net/http/cgi"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/gitcache"
+)
+
+// gitMirrorUpstream serves team/repo, with one commit on main, from git
+// http-backend, and returns the server and the commit.
+func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
+	t.Helper()
+	git := func(dir string, args ...string) string {
+		t.Helper()
+		cmd := exec.Command("git", args...)
+		cmd.Dir = dir
+		cmd.Env = []string{
+			"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
+			"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
+			"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
+		}
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
+		}
+		return strings.TrimSpace(string(out))
+	}
+	backend := ""
+	for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
+		if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
+			backend = path
+		}
+	}
+	if backend == "" {
+		t.Skip("git http-backend is not installed")
+	}
+	root := t.TempDir()
+	bare := filepath.Join(root, "team", "repo.git")
+	git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
+	work := t.TempDir()
+	git("", "clone", "--quiet", bare, work)
+	git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
+	git(work, "push", "--quiet", "origin", "HEAD:main")
+	head := git(work, "rev-parse", "HEAD")
+
+	server := httptest.NewServer(&cgi.Handler{
+		Path: backend,
+		Env:  []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
+	})
+	t.Cleanup(server.Close)
+	return server, head
+}
+
+func fileExistsForTest(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// writeGitMirrorConfig writes a config with upstream.git "up" at
+// upstreamURL and the git cache, storage and database under dir.
+func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
+	t.Helper()
+	cfg := "storage:\n" +
+		"  url: file://" + filepath.Join(dir, "artifacts") + "\n" +
+		"database:\n" +
+		"  path: " + filepath.Join(dir, "proxy.db") + "\n" +
+		"upstream:\n" +
+		"  allow_loopback: true\n" +
+		"  git:\n" +
+		"    up: " + upstreamURL + "\n" +
+		"git:\n" +
+		"  cache_dir: " + filepath.Join(dir, "git") + "\n"
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
+		t.Fatalf("writing config: %v", err)
+	}
+	return path
+}
+
+func TestMirrorCommandCachesGitCommit(t *testing.T) {
+	upstream, head := gitMirrorUpstream(t)
+	dir := t.TempDir()
+	configPath := writeGitMirrorConfig(t, dir, upstream.URL)
+	purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
+
+	output := runMirrorProcess(t, 0, "-config", configPath, purl)
+	if !strings.Contains(output, "1 downloaded") {
+		t.Errorf("output:\n%s", output)
+	}
+	cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
+	if len(cached) != 1 {
+		t.Errorf("cached repositories %v, want team/repo.git", cached)
+	}
+
+	output = runMirrorProcess(t, 0, "-config", configPath, purl)
+	if !strings.Contains(output, "1 skipped") {
+		t.Errorf("second run output:\n%s", output)
+	}
+}
+
+// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
+// mirror API, while the running server holds git.cache_dir.
+func TestMirrorCommandGitCacheInUse(t *testing.T) {
+	upstream, head := gitMirrorUpstream(t)
+	dir := t.TempDir()
+	configPath := writeGitMirrorConfig(t, dir, upstream.URL)
+	cache, err := gitcache.New(gitcache.Options{
+		CacheDir:    filepath.Join(dir, "git"),
+		RefreshWait: time.Second,
+		Client:      &http.Client{},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = cache.Close() }()
+	purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
+
+	output := runMirrorProcess(t, 1, "-config", configPath, purl)
+	for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
+		if !strings.Contains(output, want) {
+			t.Errorf("output missing %q:\n%s", want, output)
+		}
+	}
+}

docs/architecture.md

@@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g
 
 ### `internal/mirror`
 
-Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads.
+Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves.
 
 The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
 

docs/configuration.md

@@ -1877,6 +1877,20 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
 - Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`.
 - Casks, source archives and taps other than homebrew/core aren't mirrored.
 
+### Git repositories
+
+`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID:
+
+```bash
+proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
+proxy mirror pkg:git/github.com/git-pkgs/purl
+```
+
+- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail.
+- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails.
+- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID.
+- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead.
+
 ### Scheduled refresh
 
 Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later:

internal/gitcache/mirror.go

@@ -0,0 +1,118 @@
+package gitcache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// MirrorResult is what Mirror did.
+type MirrorResult struct {
+	// Cached reports that the repository, and the commit if one was given,
+	// were cached already.
+	Cached bool
+	// Bytes is how much the cached repository grew.
+	Bytes int64
+}
+
+// Mirror brings the repository cached under key up to date with repoURL,
+// cloning it if needed, and waits for that to finish. With commit, an object
+// ID, it also makes sure the commit is cached. One on no branch or tag is
+// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and
+// is kept as long as a commit a client fetched by ID.
+func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) {
+	if commit != "" && !isObjectID(commit) {
+		return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit)
+	}
+	lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants})
+	if err != nil {
+		return MirrorResult{}, err
+	}
+	defer lease.Release()
+	rp := lease.repo
+	cached := lease.hit
+	before := int64(0)
+	var refreshErr error
+	// Open cloned a repository that wasn't cached; one that was is
+	// refreshed here, waiting for the fetch a client would not wait for.
+	if cached {
+		before = rp.size.Load()
+		refresh := c.startRefresh(rp, repoURL)
+		select {
+		case <-refresh.done:
+			refreshErr = refresh.err
+		case <-ctx.Done():
+			return MirrorResult{}, ctx.Err()
+		}
+	}
+
+	// A refresh past max_repo_size gets the repository dropped once it is
+	// released, commit and all.
+	if errors.Is(refreshErr, ErrTooLarge) {
+		return MirrorResult{}, refreshErr
+	}
+	if commit != "" {
+		if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil {
+			return MirrorResult{}, err
+		}
+	}
+	// A refresh that failed leaves the commit cached all the same.
+	if refreshErr != nil && commit == "" {
+		return MirrorResult{}, refreshErr
+	}
+	grown := max(rp.size.Load()-before, 0)
+	return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil
+}
+
+// ensureCommit makes sure the repository has commit, fetching it by ID when
+// it doesn't and fetchWants allows it. A commit on no branch or tag that is
+// cached already gets a new wanted ref, so it is kept for wantedMaxAge from
+// this run on.
+func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error {
+	missing, err := c.missingObjects(ctx, rp.dir, []string{commit})
+	if err != nil {
+		return err
+	}
+	if len(missing) == 0 {
+		return c.renewWanted(ctx, rp, commit)
+	}
+	if !fetchWants {
+		return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
+	}
+	if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil {
+		if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) {
+			return err
+		}
+		// The upstream refuses a commit it doesn't have.
+		return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err)
+	}
+	// fetchMissingWants skips a commit that failed moments ago.
+	if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil {
+		return err
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
+	}
+	return nil
+}
+
+// renewWanted writes a new wanted ref for commit when no branch or tag
+// includes it.
+func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error {
+	unlock, err := rp.lockWrites(ctx)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+	unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit})
+	if err != nil || !unmerged[commit] {
+		return err
+	}
+	cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return nil
+}