James Coleman
1e6b8ed0ce
Some checks failed
Go package / build (push) Has been cancelled
- Classify core.db.sig as an Arch entry point so it is gated on the database it signs. Served as a plain file it could be revalidated on its own, and pacman fetching core.db then core.db.sig would receive a rotated signature paired with the database the mirror still holds, failing verification on every client until the next crawl. - Keep a repository registered when its crawl verified and published the tree but the requested entry point is absent upstream. Deregistering on that miss dropped every member file to the generic path, which refreshes files individually with no checksum or signature check. - Drop the generic state entry for a file a registered repository now owns instead of evicting the file. The entry is left by a request that predates the registration and nothing refreshes it, so expiry deleted a file out of the verified tree and forced the next request into a synchronous crawl. - Release 0.1.2. Claude-Session: https://claude.ai/code/session_01FkruwxzDGY4BoXp1Zzoott
381 lines
13 KiB
Go
381 lines
13 KiB
Go
// Package server runs the caching mirror: it answers requests from the
|
|
// local tree, fetches misses through the configured upstream mounts, and
|
|
// crawls the repositories it discovers in the background so they stay
|
|
// current without holding a request open.
|
|
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
cfg "github.com/grmrgecko/repo-sync/config"
|
|
"github.com/grmrgecko/repo-sync/fetch"
|
|
"github.com/grmrgecko/repo-sync/mirror"
|
|
"github.com/grmrgecko/repo-sync/state"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// Handler returns the mirror server's HTTP handler.
|
|
func Handler() http.Handler {
|
|
return http.HandlerFunc(serveMirror)
|
|
}
|
|
|
|
// regularFile stats a path, reporting only regular files as present.
|
|
func regularFile(name string) (os.FileInfo, bool) {
|
|
info, err := os.Stat(name)
|
|
if err != nil || !info.Mode().IsRegular() {
|
|
return nil, false
|
|
}
|
|
return info, true
|
|
}
|
|
|
|
// serveLocal resolves a request path under a domain root, mapping
|
|
// directory requests to a cached index.html.
|
|
func serveLocal(root, reqPath string, isDir bool) (string, error) {
|
|
p := reqPath
|
|
if isDir || p == "/" {
|
|
p = strings.TrimSuffix(p, "/") + "/index.html"
|
|
}
|
|
return fetch.LocalJoin(root, p)
|
|
}
|
|
|
|
// internalFile reports paths that belong to the mirror's own bookkeeping
|
|
// and must never be served.
|
|
func internalFile(reqPath string) bool {
|
|
base := path.Base(reqPath)
|
|
return base == fetch.LockFileName || base == fetch.PruneStateName ||
|
|
base == fetch.MissingStateName || base == fetch.DiscoverStateName ||
|
|
strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix)
|
|
}
|
|
|
|
// fetchUpstream retrieves one file through the mount covering the request
|
|
// path and stores it under root.
|
|
func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error {
|
|
mount, ok := cfg.C.MountFor(reqPath)
|
|
if !ok {
|
|
return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
|
|
}
|
|
rel := reqPath
|
|
if mount.Path != "/" {
|
|
rel = strings.TrimPrefix(reqPath, mount.Path)
|
|
}
|
|
if isDir {
|
|
rel = strings.TrimSuffix(rel, "/") + "/"
|
|
}
|
|
dst, err := serveLocal(root, reqPath, isDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Concurrent on-demand fetches and background crawls can target the
|
|
// same destination; fetch.File serializes them by path.
|
|
src := fetch.NewSource([]string{mount.Upstream})
|
|
_, err = fetch.File(ctx, src, rel, dst, nil, false, false)
|
|
return err
|
|
}
|
|
|
|
// writeFetchError reports an upstream failure to the client.
|
|
func writeFetchError(w http.ResponseWriter, err error) {
|
|
if errors.Is(err, fetch.ErrNotFound) {
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
|
|
}
|
|
|
|
// writeCachedFailure reports a remembered upstream failure.
|
|
func writeCachedFailure(w http.ResponseWriter, status int) {
|
|
if status == http.StatusNotFound {
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
|
|
}
|
|
|
|
// writeIndexesDisabled answers a directory request when directory indexes
|
|
// are turned off, confirming the mirror is alive without listing content.
|
|
func writeIndexesDisabled(w http.ResponseWriter) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
io.WriteString(w, "<!DOCTYPE html>\n<html>\n<head><title>repo-sync mirror</title></head>\n<body>\n"+
|
|
"<h1>repo-sync mirror</h1>\n<p>This mirror is running. Directory indexes are disabled.</p>\n"+
|
|
"</body>\n</html>\n")
|
|
}
|
|
|
|
// writeMountIndex answers the root directory request with a generated index
|
|
// of every mount and configured repository, giving clients a navigable
|
|
// entry point when no catch-all mount covers the root.
|
|
func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) {
|
|
seen := map[string]bool{}
|
|
var paths []string
|
|
add := func(p string) {
|
|
if p == "/" || seen[p] {
|
|
return
|
|
}
|
|
seen[p] = true
|
|
paths = append(paths, p)
|
|
}
|
|
for _, mount := range conf.Mounts {
|
|
add(mount.Path)
|
|
for _, repo := range mount.Repos {
|
|
add(repo.Path)
|
|
}
|
|
}
|
|
sort.Strings(paths)
|
|
|
|
var b strings.Builder
|
|
b.WriteString("<!DOCTYPE html>\n<html>\n<head><title>Index of /</title></head>\n<body>\n<h1>Index of /</h1>\n<ul>\n")
|
|
for _, p := range paths {
|
|
href := html.EscapeString(p + "/")
|
|
fmt.Fprintf(&b, "<li><a href=\"%s\">%s</a></li>\n", href, href)
|
|
}
|
|
b.WriteString("</ul>\n</body>\n</html>\n")
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
io.WriteString(w, b.String())
|
|
}
|
|
|
|
// serveMirror handles every mirror request: domain lookup, classification,
|
|
// and dispatch to the role-specific handler.
|
|
func serveMirror(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
domain, ok := cfg.C.Domain(r.Host)
|
|
if !ok {
|
|
http.Error(w, "unknown mirror domain", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/"))
|
|
isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/"
|
|
if internalFile(reqPath) {
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
// Directory requests are answered before role dispatch: a notice page
|
|
// when indexes are disabled, and a generated index of the mounts for a
|
|
// root that no catch-all mount covers.
|
|
if isDir {
|
|
if !cfg.C.HTTP.DirectoryIndexes {
|
|
writeIndexesDisabled(w)
|
|
return
|
|
}
|
|
if reqPath == "/" {
|
|
if _, ok := cfg.C.MountFor("/"); !ok {
|
|
writeMountIndex(w, cfg.C)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
res := classifyRequest(reqPath)
|
|
|
|
if domain.Role == cfg.RoleOnline {
|
|
handleOnline(w, r, domain, reqPath, isDir, res)
|
|
return
|
|
}
|
|
handleOffline(w, r, domain, reqPath, isDir, res)
|
|
}
|
|
|
|
// handleOnline serves from the writable cache, fetching and crawling on
|
|
// demand.
|
|
func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
|
|
local, err := serveLocal(domain.Root, reqPath, isDir)
|
|
if err != nil {
|
|
http.Error(w, "bad request path", http.StatusBadRequest)
|
|
return
|
|
}
|
|
now := time.Now()
|
|
|
|
// Directory listings are cached pass-through pages with no tracking.
|
|
if isDir {
|
|
if _, exists := regularFile(local); !exists {
|
|
if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil {
|
|
writeFetchError(w, err)
|
|
return
|
|
}
|
|
}
|
|
http.ServeFile(w, r, local)
|
|
return
|
|
}
|
|
|
|
if res.Kind == kindGeneric {
|
|
// Files under a registered repository keep it alive and are
|
|
// fetched on demand until its crawl completes.
|
|
members := state.S.TouchRepoMembers(reqPath, now)
|
|
if len(members) > 0 {
|
|
// The repository's crawl owns the file from here on; a generic
|
|
// entry left by a request that predates the registration would
|
|
// otherwise expire and evict the file from the verified tree.
|
|
state.S.Delete(res.Key)
|
|
if _, exists := regularFile(local); !exists {
|
|
var protectedKey string
|
|
var protectedEntry state.Entry
|
|
for key, entry := range members {
|
|
if protectedRepositoryKind(entry.Kind) && cfg.C.Crawler.SignatureMode != string(mirror.SignatureOff) {
|
|
protectedKey = key
|
|
protectedEntry = entry
|
|
break
|
|
}
|
|
}
|
|
var err error
|
|
if protectedKey != "" {
|
|
settings, settingsErr := loadSignatureSettings(cfg.C)
|
|
protectedRes := resource{
|
|
Kind: protectedEntry.Kind,
|
|
Key: protectedKey,
|
|
Path: protectedEntry.Path,
|
|
Root: protectedEntry.Root,
|
|
ReqPath: protectedEntry.Path,
|
|
}
|
|
err = crawlProtectedRepository(r.Context(), protectedRes, false, settings, settingsErr)
|
|
if err == nil {
|
|
_, exists = regularFile(local)
|
|
if !exists {
|
|
err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
|
|
}
|
|
}
|
|
} else {
|
|
err = fetchUpstream(r.Context(), domain.Root, reqPath, false)
|
|
}
|
|
if err != nil {
|
|
writeFetchError(w, err)
|
|
return
|
|
}
|
|
}
|
|
http.ServeFile(w, r, local)
|
|
return
|
|
}
|
|
handleGeneric(w, r, domain, reqPath, res, local, now)
|
|
return
|
|
}
|
|
|
|
// A repository entry point registers the repository and dispatches a
|
|
// background crawl on first sight so the whole repository fills in
|
|
// behind this response without holding it open. The requested file
|
|
// itself is fetched on demand so the client is answered immediately.
|
|
// Repositories already crawled only re-dispatch here to retry a failed
|
|
// crawl; missing optional files must not trigger a full crawl per
|
|
// request.
|
|
entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now)
|
|
_, exists := regularFile(local)
|
|
needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "")
|
|
conf := cfg.C
|
|
protected := protectedRepositoryKind(res.Kind) && conf.Crawler.SignatureMode != string(mirror.SignatureOff)
|
|
var signatureSettings signatureSettings
|
|
var signatureSettingsErr error
|
|
if protected {
|
|
signatureSettings, signatureSettingsErr = loadSignatureSettings(conf)
|
|
}
|
|
artifactsPresent := protected && signaturePolicyArtifactsPresent(conf, res, signatureSettings.mode)
|
|
needsSignatureCheck := protected && (signatureSettingsErr != nil || entry.SignaturePolicy != signatureSettings.policy || !artifactsPresent)
|
|
if !exists || needsSignatureCheck {
|
|
var err error
|
|
// A crawl that verified and published the repository proves the
|
|
// path is one, whether or not the requested entry point exists
|
|
// upstream.
|
|
verified := false
|
|
if protected {
|
|
// A protected entry point cannot be served until the crawl has
|
|
// verified and published its metadata generation.
|
|
err = crawlProtectedRepository(r.Context(), res, artifactsPresent, signatureSettings, signatureSettingsErr)
|
|
needCrawl = false
|
|
verified = err == nil
|
|
_, exists = regularFile(local)
|
|
if err == nil && !exists {
|
|
err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
|
|
}
|
|
} else {
|
|
err = fetchUpstream(r.Context(), domain.Root, reqPath, false)
|
|
}
|
|
if err != nil {
|
|
// A path whose first contact failed was never shown to be a
|
|
// repository, so the registration this request created is
|
|
// dropped and the scheduler never crawls it. Registrations an
|
|
// earlier request established are kept: every entry point of one
|
|
// repository shares a key, so a repository serving Release but
|
|
// not InRelease would otherwise deregister itself on the miss.
|
|
// A verified repository is kept for the same reason, or its
|
|
// member files would fall to the generic path unchecked.
|
|
if !tracked && !verified {
|
|
state.S.Delete(res.Key)
|
|
}
|
|
writeFetchError(w, err)
|
|
return
|
|
}
|
|
}
|
|
if needCrawl {
|
|
startCrawl(res)
|
|
}
|
|
http.ServeFile(w, r, local)
|
|
}
|
|
|
|
// handleGeneric serves a plain cached file, refreshing stale copies and
|
|
// remembering upstream failures.
|
|
func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) {
|
|
if status, hit := fetchFailures.Hit(res.Key, now); hit {
|
|
if _, exists := regularFile(local); !exists {
|
|
writeCachedFailure(w, status)
|
|
return
|
|
}
|
|
} else {
|
|
// GenericMaxAge bounds how often the upstream is revalidated,
|
|
// measured from the last upstream check recorded in state. The
|
|
// local file's modification time carries the upstream's
|
|
// Last-Modified, so it does not record when the mirror last checked.
|
|
entry, tracked := state.S.Entry(res.Key)
|
|
_, exists := regularFile(local)
|
|
if !exists || !tracked || now.Sub(entry.LastCrawled) > cfg.C.GenericMaxAge {
|
|
err := fetchUpstream(r.Context(), domain.Root, reqPath, false)
|
|
switch {
|
|
case errors.Is(err, fetch.ErrNotFound):
|
|
fetchFailures.Add(res.Key, http.StatusNotFound, now, cfg.C.NegativeCacheTTL)
|
|
state.S.Delete(res.Key)
|
|
if !exists {
|
|
writeFetchError(w, err)
|
|
return
|
|
}
|
|
log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.")
|
|
case err != nil:
|
|
fetchFailures.Add(res.Key, http.StatusBadGateway, now, cfg.C.TransientErrorTTL)
|
|
if !exists {
|
|
writeFetchError(w, err)
|
|
return
|
|
}
|
|
log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.")
|
|
default:
|
|
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
|
|
// Record the upstream check so GenericMaxAge gates the next
|
|
// request. The scheduler skips generic entries, so the retry
|
|
// time this also sets is unused.
|
|
state.S.MarkCrawled(res.Key, now, nil)
|
|
}
|
|
} else {
|
|
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
|
|
}
|
|
}
|
|
http.ServeFile(w, r, local)
|
|
}
|
|
|
|
// handleOffline serves the read-only published tree, reading through the
|
|
// online cache for anything it does not hold.
|
|
func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
|
|
local, err := serveLocal(domain.Root, reqPath, isDir)
|
|
if err != nil {
|
|
http.Error(w, "bad request path", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if _, exists := regularFile(local); exists {
|
|
http.ServeFile(w, r, local)
|
|
return
|
|
}
|
|
handleOnline(w, r, cfg.C.OnlineDomain(), reqPath, isDir, res)
|
|
}
|