diff --git a/VERSION b/VERSION index 17e51c3..d917d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.1 +0.1.2 diff --git a/server/classify.go b/server/classify.go index eac42c1..96a57a2 100644 --- a/server/classify.go +++ b/server/classify.go @@ -67,8 +67,10 @@ func classifyRequest(reqPath string) resource { } } - // Pacman repositories are identified by their database file. - if strings.HasSuffix(base, ".db") { + // Pacman repositories are identified by their database file. Its + // detached signature is an entry point too: served as a plain file it + // could be refreshed apart from the database it signs. + if strings.HasSuffix(base, ".db") || strings.HasSuffix(base, ".db.sig") { return resource{ Kind: string(mirror.RepoArch), Key: string(mirror.RepoArch) + ":" + dir, diff --git a/server/classify_test.go b/server/classify_test.go index 202e8c4..3d3375a 100644 --- a/server/classify_test.go +++ b/server/classify_test.go @@ -22,6 +22,8 @@ func TestClassifyRequest(t *testing.T) { {"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"}, {"/flat/Release", "deb", "/flat", "/flat"}, {"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"}, + {"/archlinux/core/os/x86_64/core.db.sig", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"}, + {"/archlinux/core/os/x86_64/zlib-1.3-1-x86_64.pkg.tar.zst.sig", kindGeneric, "/archlinux/core/os/x86_64/zlib-1.3-1-x86_64.pkg.tar.zst.sig", ""}, {"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"}, {"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""}, {"/notes/README.txt", kindGeneric, "/notes/README.txt", ""}, diff --git a/server/crawl_loop.go b/server/crawl_loop.go index 3e9123e..b54784b 100644 --- a/server/crawl_loop.go +++ b/server/crawl_loop.go @@ -248,10 +248,12 @@ func signaturePolicyArtifactsPresent(conf *cfg.Config, res resource, mode mirror _, exists := regularFile(signature) return exists case mirror.RepoArch: - if !strings.HasSuffix(res.ReqPath, ".db") { + // A signature request is gated on the database it signs. + name := strings.TrimSuffix(res.ReqPath, ".sig") + if !strings.HasSuffix(name, ".db") { return false } - database, err := fetch.LocalJoin(conf.OnlineDomain().Root, res.ReqPath) + database, err := fetch.LocalJoin(conf.OnlineDomain().Root, name) if err != nil { return false } @@ -317,6 +319,20 @@ func pathBelow(p, base string) bool { return strings.HasPrefix(p, base+"/") } +// repositoryOwns reports whether a registered repository's root covers a +// request path, meaning its crawl is what keeps the file there current. +func repositoryOwns(reqPath string) bool { + for _, entry := range state.S.Snapshot() { + if entry.Kind == kindGeneric || entry.Root == "" { + continue + } + if pathBelow(reqPath, entry.Root) { + return true + } + } + return false +} + // prunableTree reports whether a crawl may prune its destination tree. Every // sync prunes the repository directory, which for deb is the suite directory // and otherwise the repository's own path. A crawl's keep set covers only @@ -594,6 +610,14 @@ func evictStale(key string, entry state.Entry) { if ok && current.LastRequested.After(entry.LastRequested) { return } + // A plain file requested before its repository was registered keeps a + // generic entry that nothing refreshes. The file belongs to the + // repository's verified tree now, so only the bookkeeping goes. + if entry.Kind == kindGeneric && repositoryOwns(entry.Path) { + log.WithFields(log.Fields{"key": key, "path": entry.Path}).Debug("Dropped a generic entry for a repository member.") + state.S.Delete(key) + return + } target, err := fetch.LocalJoin(cfg.C.OnlineDomain().Root, entry.Path) if err == nil && entry.Path != "/" && entry.Path != "" { if err := os.RemoveAll(target); err != nil { diff --git a/server/serve.go b/server/serve.go index d5840c2..08759b4 100644 --- a/server/serve.go +++ b/server/serve.go @@ -212,6 +212,10 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi // fetched on demand until its crawl completes. members := state.S.TouchRepoMembers(reqPath, now) if len(members) > 0 { + // The repository's crawl owns the file from here on; a generic + // entry left by a request that predates the registration would + // otherwise expire and evict the file from the verified tree. + state.S.Delete(res.Key) if _, exists := regularFile(local); !exists { var protectedKey string var protectedEntry state.Entry @@ -275,11 +279,16 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi needsSignatureCheck := protected && (signatureSettingsErr != nil || entry.SignaturePolicy != signatureSettings.policy || !artifactsPresent) if !exists || needsSignatureCheck { var err error + // A crawl that verified and published the repository proves the + // path is one, whether or not the requested entry point exists + // upstream. + verified := false if protected { // A protected entry point cannot be served until the crawl has // verified and published its metadata generation. err = crawlProtectedRepository(r.Context(), res, artifactsPresent, signatureSettings, signatureSettingsErr) needCrawl = false + verified = err == nil _, exists = regularFile(local) if err == nil && !exists { err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound) @@ -294,7 +303,9 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi // earlier request established are kept: every entry point of one // repository shares a key, so a repository serving Release but // not InRelease would otherwise deregister itself on the miss. - if !tracked { + // A verified repository is kept for the same reason, or its + // member files would fall to the generic path unchecked. + if !tracked && !verified { state.S.Delete(res.Key) } writeFetchError(w, err) diff --git a/server/serve_test.go b/server/serve_test.go index 7c4ce4d..dbc37e3 100644 --- a/server/serve_test.go +++ b/server/serve_test.go @@ -2,6 +2,7 @@ package server import ( "bytes" + "compress/gzip" "fmt" "io" "net/http" @@ -585,3 +586,141 @@ func TestServeUnresolvedEntryPoint(t *testing.T) { } } } + +// archSignatureFixture serves one pacman repository, signed when a key is +// given, behind a mirror enforcing the given signature mode. Upstream files +// carry an hour-old modification time so a later rotation is visible to +// conditional requests, which http.FileServer compares at second precision. +func archSignatureFixture(t *testing.T, key *testrepos.SigningKey, mode string) (*httptest.Server, string, string) { + t.Helper() + www := t.TempDir() + repoDir := filepath.Join(www, "archlinux", "core", "os", "x86_64") + packages := testrepos.BuildArchRepo(t, repoDir, "core") + if key != nil { + key.SignArchRepo(t, repoDir, "core", packages) + } else { + // The fixture's placeholder package signatures would fail + // verification; an unsigned upstream publishes none. + for filename := range packages { + require.NoError(t, os.Remove(filepath.Join(repoDir, filename+".sig"))) + } + } + old := time.Now().Add(-time.Hour) + entries, err := os.ReadDir(repoDir) + require.NoError(t, err) + for _, entry := range entries { + require.NoError(t, os.Chtimes(filepath.Join(repoDir, entry.Name()), old, old)) + } + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + confDir := t.TempDir() + keys := "[]" + if key != nil { + keyPath := filepath.Join(confDir, "repository.asc") + testrepos.WriteFile(t, keyPath, key.PublicKey(t)) + keys = "[" + keyPath + "]" + } + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - {domain: 127.0.0.1, role: online, root: %s} +mounts: + - {path: /, upstream: %s} +crawler: + signature_mode: %s + gpg_keys: %s + keyservers: [] +`, confDir, onlineRoot, upstream.URL, mode, keys))) + require.NoError(t, cfg.Init(confPath)) + require.NoError(t, state.Load()) + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + t.Cleanup(WaitCrawls) + return srv, onlineRoot, repoDir +} + +// TestServeArchSignatureEntryPoint verifies a pacman database signature is +// never refreshed on its own. pacman fetches core.db and then core.db.sig, +// so a mirror that revalidated the signature as a plain file after losing +// the repository's registration would pair a rotated signature with the +// database it still holds, and every client would fail verification until +// the next crawl. The signature must take the same verified path as +// Release.gpg and repomd.xml.asc. +func TestServeArchSignatureEntryPoint(t *testing.T) { + key := testrepos.NewSigningKey(t) + srv, onlineRoot, repoDir := archSignatureFixture(t, key, "required") + repo := "/archlinux/core/os/x86_64" + localDB := filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "core.db") + localSig := localDB + ".sig" + + resp, _ := get(t, srv, "", repo+"/core.db") + require.Equal(t, http.StatusOK, resp.StatusCode) + require.FileExists(t, localSig, "the verifying crawl publishes the database signature") + + // Model a restart that lost the state file: the tree is kept, the + // registration is gone. + state.S.Delete("arch:" + repo) + + // Rotate the upstream: the database gains an empty trailing gzip member, + // which changes its bytes without changing its contents, and is re-signed. + upstreamDB := filepath.Join(repoDir, "core.db") + rotated, err := os.ReadFile(upstreamDB) + require.NoError(t, err) + var trailer bytes.Buffer + require.NoError(t, gzip.NewWriter(&trailer).Close()) + rotated = append(rotated, trailer.Bytes()...) + require.NoError(t, os.WriteFile(upstreamDB, rotated, 0644)) + key.SignArchRepo(t, repoDir, "core", nil) + rotatedSig, err := os.ReadFile(upstreamDB + ".sig") + require.NoError(t, err) + + resp, body := get(t, srv, "", repo+"/core.db.sig") + require.Equal(t, http.StatusOK, resp.StatusCode) + gotDB, err := os.ReadFile(localDB) + require.NoError(t, err) + assert.Equal(t, rotated, gotDB, "a signature request must publish the database it was verified against") + assert.Equal(t, rotatedSig, body, "the rotated signature is served once its pair is verified") + _, registered := state.S.Entry("arch:" + repo) + assert.True(t, registered, "a signature request registers the repository it belongs to") +} + +// TestServeProtectedFirstContactMiss verifies a repository whose first +// request is an entry point the upstream does not publish stays registered +// once its crawl succeeded. Deregistering it would hand every member file +// to the generic path, which refreshes files individually with no checksum +// or signature check, until a later entry-point request re-registers it. +func TestServeProtectedFirstContactMiss(t *testing.T) { + srv, onlineRoot, _ := archSignatureFixture(t, nil, "if-present") + repo := "/archlinux/core/os/x86_64" + + resp, _ := get(t, srv, "", repo+"/core.db.sig") + assert.Equal(t, http.StatusNotFound, resp.StatusCode, "the upstream publishes no database signature") + assert.FileExists(t, filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "core.db"), "the crawl published the repository") + _, registered := state.S.Entry("arch:" + repo) + assert.True(t, registered, "a crawled repository stays registered after a sibling entry-point miss") +} + +// TestEvictStaleKeepsRepositoryMember verifies evicting a plain-file entry +// leaves the file alone when a registered repository owns it. A file +// requested before its repository was registered is tracked as generic; +// deleting it from under the verified tree would force the next request into +// a full synchronous crawl to restore it. +func TestEvictStaleKeepsRepositoryMember(t *testing.T) { + _, onlineRoot, _ := serverFixture(t) + repo := "/archlinux/core/os/x86_64" + member := repo + "/zlib-1.3-1-x86_64.pkg.tar.zst" + local := filepath.Join(onlineRoot, filepath.FromSlash(member[1:])) + testrepos.WriteFile(t, local, []byte("package")) + + now := time.Now() + state.S.MarkRequested(kindGeneric, kindGeneric+":"+member, member, "", now.Add(-48*time.Hour)) + state.S.MarkRequested(string(mirror.RepoArch), "arch:"+repo, repo, repo, now) + entry, _ := state.S.Entry(kindGeneric + ":" + member) + evictStale(kindGeneric+":"+member, entry) + + assert.FileExists(t, local, "a repository member must survive eviction of its stale generic entry") + _, tracked := state.S.Entry(kindGeneric + ":" + member) + assert.False(t, tracked, "the stale generic entry is dropped") +}