fix(server): route pacman database signatures through the crawl

- Classify core.db.sig as an Arch entry point so it is gated on the database it signs. Served as a plain file it could be revalidated on its own, and pacman fetching core.db then core.db.sig would receive a rotated signature paired with the database the mirror still holds, failing verification on every client until the next crawl.
- Keep a repository registered when its crawl verified and published the tree but the requested entry point is absent upstream. Deregistering on that miss dropped every member file to the generic path, which refreshes files individually with no checksum or signature check.
- Drop the generic state entry for a file a registered repository now owns instead of evicting the file. The entry is left by a request that predates the registration and nothing refreshes it, so expiry deleted a file out of the verified tree and forced the next request into a synchronous crawl.
- Release 0.1.2.

Claude-Session: https://claude.ai/code/session_01FkruwxzDGY4BoXp1Zzoott

VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

server/classify.go

@@ -67,8 +67,10 @@ func classifyRequest(reqPath string) resource {
 		}
 	}
 
-	// Pacman repositories are identified by their database file.
-	if strings.HasSuffix(base, ".db") {
+	// Pacman repositories are identified by their database file. Its
+	// detached signature is an entry point too: served as a plain file it
+	// could be refreshed apart from the database it signs.
+	if strings.HasSuffix(base, ".db") || strings.HasSuffix(base, ".db.sig") {
 		return resource{
 			Kind:    string(mirror.RepoArch),
 			Key:     string(mirror.RepoArch) + ":" + dir,

server/classify_test.go

@@ -22,6 +22,8 @@ func TestClassifyRequest(t *testing.T) {
 		{"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"},
 		{"/flat/Release", "deb", "/flat", "/flat"},
 		{"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"},
+		{"/archlinux/core/os/x86_64/core.db.sig", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"},
+		{"/archlinux/core/os/x86_64/zlib-1.3-1-x86_64.pkg.tar.zst.sig", kindGeneric, "/archlinux/core/os/x86_64/zlib-1.3-1-x86_64.pkg.tar.zst.sig", ""},
 		{"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"},
 		{"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""},
 		{"/notes/README.txt", kindGeneric, "/notes/README.txt", ""},

server/crawl_loop.go

@@ -248,10 +248,12 @@ func signaturePolicyArtifactsPresent(conf *cfg.Config, res resource, mode mirror
 		_, exists := regularFile(signature)
 		return exists
 	case mirror.RepoArch:
-		if !strings.HasSuffix(res.ReqPath, ".db") {
+		// A signature request is gated on the database it signs.
+		name := strings.TrimSuffix(res.ReqPath, ".sig")
+		if !strings.HasSuffix(name, ".db") {
 			return false
 		}
-		database, err := fetch.LocalJoin(conf.OnlineDomain().Root, res.ReqPath)
+		database, err := fetch.LocalJoin(conf.OnlineDomain().Root, name)
 		if err != nil {
 			return false
 		}
@@ -317,6 +319,20 @@ func pathBelow(p, base string) bool {
 	return strings.HasPrefix(p, base+"/")
 }
 
+// repositoryOwns reports whether a registered repository's root covers a
+// request path, meaning its crawl is what keeps the file there current.
+func repositoryOwns(reqPath string) bool {
+	for _, entry := range state.S.Snapshot() {
+		if entry.Kind == kindGeneric || entry.Root == "" {
+			continue
+		}
+		if pathBelow(reqPath, entry.Root) {
+			return true
+		}
+	}
+	return false
+}
+
 // prunableTree reports whether a crawl may prune its destination tree. Every
 // sync prunes the repository directory, which for deb is the suite directory
 // and otherwise the repository's own path. A crawl's keep set covers only
@@ -594,6 +610,14 @@ func evictStale(key string, entry state.Entry) {
 	if ok && current.LastRequested.After(entry.LastRequested) {
 		return
 	}
+	// A plain file requested before its repository was registered keeps a
+	// generic entry that nothing refreshes. The file belongs to the
+	// repository's verified tree now, so only the bookkeeping goes.
+	if entry.Kind == kindGeneric && repositoryOwns(entry.Path) {
+		log.WithFields(log.Fields{"key": key, "path": entry.Path}).Debug("Dropped a generic entry for a repository member.")
+		state.S.Delete(key)
+		return
+	}
 	target, err := fetch.LocalJoin(cfg.C.OnlineDomain().Root, entry.Path)
 	if err == nil && entry.Path != "/" && entry.Path != "" {
 		if err := os.RemoveAll(target); err != nil {

server/serve.go

@@ -212,6 +212,10 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi
 		// fetched on demand until its crawl completes.
 		members := state.S.TouchRepoMembers(reqPath, now)
 		if len(members) > 0 {
+			// The repository's crawl owns the file from here on; a generic
+			// entry left by a request that predates the registration would
+			// otherwise expire and evict the file from the verified tree.
+			state.S.Delete(res.Key)
 			if _, exists := regularFile(local); !exists {
 				var protectedKey string
 				var protectedEntry state.Entry
@@ -275,11 +279,16 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi
 	needsSignatureCheck := protected && (signatureSettingsErr != nil || entry.SignaturePolicy != signatureSettings.policy || !artifactsPresent)
 	if !exists || needsSignatureCheck {
 		var err error
+		// A crawl that verified and published the repository proves the
+		// path is one, whether or not the requested entry point exists
+		// upstream.
+		verified := false
 		if protected {
 			// A protected entry point cannot be served until the crawl has
 			// verified and published its metadata generation.
 			err = crawlProtectedRepository(r.Context(), res, artifactsPresent, signatureSettings, signatureSettingsErr)
 			needCrawl = false
+			verified = err == nil
 			_, exists = regularFile(local)
 			if err == nil && !exists {
 				err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
@@ -294,7 +303,9 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi
 			// earlier request established are kept: every entry point of one
 			// repository shares a key, so a repository serving Release but
 			// not InRelease would otherwise deregister itself on the miss.
-			if !tracked {
+			// A verified repository is kept for the same reason, or its
+			// member files would fall to the generic path unchecked.
+			if !tracked && !verified {
 				state.S.Delete(res.Key)
 			}
 			writeFetchError(w, err)

server/serve_test.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"bytes"
+	"compress/gzip"
 	"fmt"
 	"io"
 	"net/http"
@@ -585,3 +586,141 @@ func TestServeUnresolvedEntryPoint(t *testing.T) {
 		}
 	}
 }
+
+// archSignatureFixture serves one pacman repository, signed when a key is
+// given, behind a mirror enforcing the given signature mode. Upstream files
+// carry an hour-old modification time so a later rotation is visible to
+// conditional requests, which http.FileServer compares at second precision.
+func archSignatureFixture(t *testing.T, key *testrepos.SigningKey, mode string) (*httptest.Server, string, string) {
+	t.Helper()
+	www := t.TempDir()
+	repoDir := filepath.Join(www, "archlinux", "core", "os", "x86_64")
+	packages := testrepos.BuildArchRepo(t, repoDir, "core")
+	if key != nil {
+		key.SignArchRepo(t, repoDir, "core", packages)
+	} else {
+		// The fixture's placeholder package signatures would fail
+		// verification; an unsigned upstream publishes none.
+		for filename := range packages {
+			require.NoError(t, os.Remove(filepath.Join(repoDir, filename+".sig")))
+		}
+	}
+	old := time.Now().Add(-time.Hour)
+	entries, err := os.ReadDir(repoDir)
+	require.NoError(t, err)
+	for _, entry := range entries {
+		require.NoError(t, os.Chtimes(filepath.Join(repoDir, entry.Name()), old, old))
+	}
+	upstream := testrepos.ServeDir(t, www)
+
+	onlineRoot := t.TempDir()
+	confDir := t.TempDir()
+	keys := "[]"
+	if key != nil {
+		keyPath := filepath.Join(confDir, "repository.asc")
+		testrepos.WriteFile(t, keyPath, key.PublicKey(t))
+		keys = "[" + keyPath + "]"
+	}
+	confPath := filepath.Join(confDir, "config.yaml")
+	testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(`
+state_path: %s/state.yaml
+domains:
+  - {domain: 127.0.0.1, role: online, root: %s}
+mounts:
+  - {path: /, upstream: %s}
+crawler:
+  signature_mode: %s
+  gpg_keys: %s
+  keyservers: []
+`, confDir, onlineRoot, upstream.URL, mode, keys)))
+	require.NoError(t, cfg.Init(confPath))
+	require.NoError(t, state.Load())
+	srv := httptest.NewServer(Handler())
+	t.Cleanup(srv.Close)
+	t.Cleanup(WaitCrawls)
+	return srv, onlineRoot, repoDir
+}
+
+// TestServeArchSignatureEntryPoint verifies a pacman database signature is
+// never refreshed on its own. pacman fetches core.db and then core.db.sig,
+// so a mirror that revalidated the signature as a plain file after losing
+// the repository's registration would pair a rotated signature with the
+// database it still holds, and every client would fail verification until
+// the next crawl. The signature must take the same verified path as
+// Release.gpg and repomd.xml.asc.
+func TestServeArchSignatureEntryPoint(t *testing.T) {
+	key := testrepos.NewSigningKey(t)
+	srv, onlineRoot, repoDir := archSignatureFixture(t, key, "required")
+	repo := "/archlinux/core/os/x86_64"
+	localDB := filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "core.db")
+	localSig := localDB + ".sig"
+
+	resp, _ := get(t, srv, "", repo+"/core.db")
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.FileExists(t, localSig, "the verifying crawl publishes the database signature")
+
+	// Model a restart that lost the state file: the tree is kept, the
+	// registration is gone.
+	state.S.Delete("arch:" + repo)
+
+	// Rotate the upstream: the database gains an empty trailing gzip member,
+	// which changes its bytes without changing its contents, and is re-signed.
+	upstreamDB := filepath.Join(repoDir, "core.db")
+	rotated, err := os.ReadFile(upstreamDB)
+	require.NoError(t, err)
+	var trailer bytes.Buffer
+	require.NoError(t, gzip.NewWriter(&trailer).Close())
+	rotated = append(rotated, trailer.Bytes()...)
+	require.NoError(t, os.WriteFile(upstreamDB, rotated, 0644))
+	key.SignArchRepo(t, repoDir, "core", nil)
+	rotatedSig, err := os.ReadFile(upstreamDB + ".sig")
+	require.NoError(t, err)
+
+	resp, body := get(t, srv, "", repo+"/core.db.sig")
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	gotDB, err := os.ReadFile(localDB)
+	require.NoError(t, err)
+	assert.Equal(t, rotated, gotDB, "a signature request must publish the database it was verified against")
+	assert.Equal(t, rotatedSig, body, "the rotated signature is served once its pair is verified")
+	_, registered := state.S.Entry("arch:" + repo)
+	assert.True(t, registered, "a signature request registers the repository it belongs to")
+}
+
+// TestServeProtectedFirstContactMiss verifies a repository whose first
+// request is an entry point the upstream does not publish stays registered
+// once its crawl succeeded. Deregistering it would hand every member file
+// to the generic path, which refreshes files individually with no checksum
+// or signature check, until a later entry-point request re-registers it.
+func TestServeProtectedFirstContactMiss(t *testing.T) {
+	srv, onlineRoot, _ := archSignatureFixture(t, nil, "if-present")
+	repo := "/archlinux/core/os/x86_64"
+
+	resp, _ := get(t, srv, "", repo+"/core.db.sig")
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode, "the upstream publishes no database signature")
+	assert.FileExists(t, filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "core.db"), "the crawl published the repository")
+	_, registered := state.S.Entry("arch:" + repo)
+	assert.True(t, registered, "a crawled repository stays registered after a sibling entry-point miss")
+}
+
+// TestEvictStaleKeepsRepositoryMember verifies evicting a plain-file entry
+// leaves the file alone when a registered repository owns it. A file
+// requested before its repository was registered is tracked as generic;
+// deleting it from under the verified tree would force the next request into
+// a full synchronous crawl to restore it.
+func TestEvictStaleKeepsRepositoryMember(t *testing.T) {
+	_, onlineRoot, _ := serverFixture(t)
+	repo := "/archlinux/core/os/x86_64"
+	member := repo + "/zlib-1.3-1-x86_64.pkg.tar.zst"
+	local := filepath.Join(onlineRoot, filepath.FromSlash(member[1:]))
+	testrepos.WriteFile(t, local, []byte("package"))
+
+	now := time.Now()
+	state.S.MarkRequested(kindGeneric, kindGeneric+":"+member, member, "", now.Add(-48*time.Hour))
+	state.S.MarkRequested(string(mirror.RepoArch), "arch:"+repo, repo, repo, now)
+	entry, _ := state.S.Entry(kindGeneric + ":" + member)
+	evictStale(kindGeneric+":"+member, entry)
+
+	assert.FileExists(t, local, "a repository member must survive eviction of its stale generic entry")
+	_, tracked := state.S.Entry(kindGeneric + ":" + member)
+	assert.False(t, tracked, "the stale generic entry is dropped")
+}