// Package server runs the caching mirror: it answers requests from the
// local tree, fetches misses through the configured upstream mounts, and
// crawls the repositories it discovers in the background so they stay
// current without holding a request open.
package server
import (
"context"
"errors"
"fmt"
"html"
"io"
"net/http"
"os"
"path"
"sort"
"strings"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// Handler returns the mirror server's HTTP handler.
func Handler() http.Handler {
return http.HandlerFunc(serveMirror)
}
// regularFile stats a path, reporting only regular files as present.
func regularFile(name string) (os.FileInfo, bool) {
info, err := os.Stat(name)
if err != nil || !info.Mode().IsRegular() {
return nil, false
}
return info, true
}
// serveLocal resolves a request path under a domain root, mapping
// directory requests to a cached index.html.
func serveLocal(root, reqPath string, isDir bool) (string, error) {
p := reqPath
if isDir || p == "/" {
p = strings.TrimSuffix(p, "/") + "/index.html"
}
return fetch.LocalJoin(root, p)
}
// internalFile reports paths that belong to the mirror's own bookkeeping
// and must never be served.
func internalFile(reqPath string) bool {
base := path.Base(reqPath)
return base == fetch.LockFileName || base == fetch.PruneStateName ||
strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix)
}
// fetchUpstream retrieves one file through the mount covering the request
// path and stores it under root.
func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error {
mount, ok := cfg.C.MountFor(reqPath)
if !ok {
return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
}
rel := reqPath
if mount.Path != "/" {
rel = strings.TrimPrefix(reqPath, mount.Path)
}
if isDir {
rel = strings.TrimSuffix(rel, "/") + "/"
}
dst, err := serveLocal(root, reqPath, isDir)
if err != nil {
return err
}
// Concurrent on-demand fetches and background crawls can target the
// same destination; fetch.File serializes them by path.
src := fetch.NewSource([]string{mount.Upstream})
_, err = fetch.File(ctx, src, rel, dst, nil, false, false)
return err
}
// writeFetchError reports an upstream failure to the client.
func writeFetchError(w http.ResponseWriter, err error) {
if errors.Is(err, fetch.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeCachedFailure reports a remembered upstream failure.
func writeCachedFailure(w http.ResponseWriter, status int) {
if status == http.StatusNotFound {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeIndexesDisabled answers a directory request when directory indexes
// are turned off, confirming the mirror is alive without listing content.
func writeIndexesDisabled(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, "\n\nrepo-sync mirror\n\n"+
"repo-sync mirror
\nThis mirror is running. Directory indexes are disabled.
\n"+
"\n\n")
}
// writeMountIndex answers the root directory request with a generated index
// of every mount and configured repository, giving clients a navigable
// entry point when no catch-all mount covers the root.
func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) {
seen := map[string]bool{}
var paths []string
add := func(p string) {
if p == "/" || seen[p] {
return
}
seen[p] = true
paths = append(paths, p)
}
for _, mount := range conf.Mounts {
add(mount.Path)
for _, repo := range mount.Repos {
add(repo.Path)
}
}
sort.Strings(paths)
var b strings.Builder
b.WriteString("\n\nIndex of /\n\nIndex of /
\n\n")
for _, p := range paths {
href := html.EscapeString(p + "/")
fmt.Fprintf(&b, "- %s
\n", href, href)
}
b.WriteString("
\n\n\n")
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, b.String())
}
// serveMirror handles every mirror request: domain lookup, classification,
// and dispatch to the role-specific handler.
func serveMirror(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
domain, ok := cfg.C.Domain(r.Host)
if !ok {
http.Error(w, "unknown mirror domain", http.StatusNotFound)
return
}
reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/"))
isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/"
if internalFile(reqPath) {
http.Error(w, "not found", http.StatusNotFound)
return
}
// Directory requests are answered before role dispatch: a notice page
// when indexes are disabled, and a generated index of the mounts for a
// root that no catch-all mount covers.
if isDir {
if !cfg.C.HTTP.DirectoryIndexes {
writeIndexesDisabled(w)
return
}
if reqPath == "/" {
if _, ok := cfg.C.MountFor("/"); !ok {
writeMountIndex(w, cfg.C)
return
}
}
}
res := classifyRequest(reqPath)
if domain.Role == cfg.RoleOnline {
handleOnline(w, r, domain, reqPath, isDir, res)
return
}
handleOffline(w, r, domain, reqPath, isDir, res)
}
// handleOnline serves from the writable cache, fetching and crawling on
// demand.
func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
now := time.Now()
// Directory listings are cached pass-through pages with no tracking.
if isDir {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
if res.Kind == kindGeneric {
// Files under a registered repository keep it alive and are
// fetched on demand until its crawl completes.
if state.S.TouchRepoMembers(reqPath, now) {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
handleGeneric(w, r, domain, reqPath, res, local, now)
return
}
// A repository entry point registers the repository and dispatches a
// background crawl on first sight so the whole repository fills in
// behind this response without holding it open. The requested file
// itself is fetched on demand so the client is answered immediately.
// Repositories already crawled only re-dispatch here to retry a failed
// crawl; missing optional files must not trigger a full crawl per
// request.
entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now)
_, exists := regularFile(local)
needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "")
if !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
// A path whose first contact failed was never shown to be a
// repository, so the registration this request created is
// dropped and the scheduler never crawls it. Registrations an
// earlier request established are kept: every entry point of one
// repository shares a key, so a repository serving Release but
// not InRelease would otherwise deregister itself on the miss.
if !tracked {
state.S.Delete(res.Key)
}
writeFetchError(w, err)
return
}
}
if needCrawl {
startCrawl(res)
}
http.ServeFile(w, r, local)
}
// handleGeneric serves a plain cached file, refreshing stale copies and
// remembering upstream failures.
func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) {
if status, hit := fetchFailures.Hit(res.Key, now); hit {
if _, exists := regularFile(local); !exists {
writeCachedFailure(w, status)
return
}
} else {
// GenericMaxAge bounds how often the upstream is revalidated,
// measured from the last upstream check recorded in state. The
// local file's modification time carries the upstream's
// Last-Modified, so it does not record when the mirror last checked.
entry, tracked := state.S.Entry(res.Key)
_, exists := regularFile(local)
if !exists || !tracked || now.Sub(entry.LastCrawled) > cfg.C.GenericMaxAge {
err := fetchUpstream(r.Context(), domain.Root, reqPath, false)
switch {
case errors.Is(err, fetch.ErrNotFound):
fetchFailures.Add(res.Key, http.StatusNotFound, now, cfg.C.NegativeCacheTTL)
state.S.Delete(res.Key)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.")
case err != nil:
fetchFailures.Add(res.Key, http.StatusBadGateway, now, cfg.C.TransientErrorTTL)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.")
default:
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
// Record the upstream check so GenericMaxAge gates the next
// request. The scheduler skips generic entries, so the retry
// time this also sets is unused.
state.S.MarkCrawled(res.Key, now, nil)
}
} else {
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
}
}
http.ServeFile(w, r, local)
}
// handleOffline serves the read-only published tree, reading through the
// online cache for anything it does not hold.
func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
if _, exists := regularFile(local); exists {
http.ServeFile(w, r, local)
return
}
handleOnline(w, r, cfg.C.OnlineDomain(), reqPath, isDir, res)
}