repo-sync/fetch/missing.go

package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// MissingMode selects what a synchronization does with a file its
// repository metadata lists but the upstream does not serve. Published
// repositories are not always complete: an index can outlive the packages it
// references, and a whole repository is not worth abandoning over one of
// them.
type MissingMode string

const (
	// MissingFail stops a repository at the first missing file.
	MissingFail MissingMode = "fail"
	// MissingRetry keeps synchronizing the rest of the repository and fails
	// the run until the same file has been missing across enough consecutive
	// runs to be accepted as gone.
	MissingRetry MissingMode = "retry"
	// MissingIgnore skips missing files without ever failing the run.
	MissingIgnore MissingMode = "ignore"
)

// MissingModes lists every supported mode, in the order they escalate from
// tolerant to strict.
var MissingModes = []MissingMode{MissingIgnore, MissingRetry, MissingFail}

// MissingModeNames lists the supported mode names, for user facing messages.
func MissingModeNames() []string {
	names := make([]string, 0, len(MissingModes))
	for _, mode := range MissingModes {
		names = append(names, string(mode))
	}
	return names
}

// ParseMissingMode converts a configured mode name into a MissingMode. An
// empty name selects the strict mode, which is what a caller that configured
// none expects.
func ParseMissingMode(name string) (MissingMode, error) {
	name = strings.ToLower(strings.TrimSpace(name))
	if name == "" {
		return MissingFail, nil
	}
	mode := MissingMode(name)
	if !slices.Contains(MissingModes, mode) {
		return "", fmt.Errorf("unsupported missing file mode %q; expected one of %s",
			name, strings.Join(MissingModeNames(), ", "))
	}
	return mode, nil
}

// MissingStateName is the known-missing bookkeeping file kept in a
// synchronized repository.
const MissingStateName = ".repo-sync-missing.json"

// missingRecord is one file's absence: when it was first and last missed,
// how many consecutive runs have missed it, and whether it has been accepted
// as permanently gone.
type missingRecord struct {
	First time.Time `json:"first"`
	Last  time.Time `json:"last"`
	Runs  int       `json:"runs"`
	Known bool      `json:"known"`
}

// Missing tracks the files an upstream did not serve during one repository
// synchronization and decides whether those absences fail it. Records live
// in the repository itself so an absence that repeats across runs can be
// told apart from one that has only just appeared, which is the difference
// between a package that was withdrawn and an upstream having a bad day.
//
// A nil Missing is the strict mode: nothing is tracked and every absence
// fails the run.
type Missing struct {
	mode    MissingMode
	retries int
	root    string
	dryRun  bool

	// mu guards everything below; download workers report concurrently.
	mu      sync.Mutex
	records map[string]*missingRecord
	seen    map[string]bool
	// unresolved counts the files missed this run that are still inside
	// their retry budget, and first names one of them for the error.
	unresolved int
	first      string
}

// NewMissing starts tracking the files an upstream does not serve for a
// repository rooted at dir, loading what earlier runs recorded there. The
// strict mode tracks nothing, so it returns nil and every absence keeps
// failing the run.
func NewMissing(dir string, mode MissingMode, retries int, dryRun bool) *Missing {
	if mode != MissingRetry && mode != MissingIgnore {
		return nil
	}
	m := &Missing{
		mode:    mode,
		retries: retries,
		root:    dir,
		dryRun:  dryRun,
		records: map[string]*missingRecord{},
		seen:    map[string]bool{},
	}
	m.load()
	return m
}

// load reads the records an earlier run left in the repository. A missing or
// unreadable file starts fresh: the records only decide how much longer an
// absence is retried, so losing them costs retries rather than correctness.
func (m *Missing) load() {
	data, err := os.ReadFile(filepath.Join(m.root, MissingStateName))
	if err != nil {
		return
	}
	records := map[string]*missingRecord{}
	if err := json.Unmarshal(data, &records); err != nil {
		log.WithError(err).WithField("path", m.root).Warn("Failed to parse the missing file state; starting fresh.")
		return
	}
	// Drop null records a hand-edited file may carry, as every lookup
	// dereferences the stored pointer.
	for name, rec := range records {
		if rec != nil {
			m.records[name] = rec
		}
	}
}

// Tolerate records that the upstream did not serve reqPath and reports
// whether the synchronization may continue without it. A file inside its
// retry budget is still tolerated for this run, so the rest of the
// repository is mirrored either way; Finish is what reports it as a failure.
func (m *Missing) Tolerate(reqPath string) bool {
	if m == nil {
		return false
	}
	m.mu.Lock()
	defer m.mu.Unlock()

	// Count each file once per run: a repository may list the same file from
	// several indexes, and the budget counts runs rather than references.
	if m.seen[reqPath] {
		return true
	}
	m.seen[reqPath] = true

	now := time.Now()
	rec := m.records[reqPath]
	if rec == nil {
		rec = &missingRecord{First: now}
		m.records[reqPath] = rec
	}
	rec.Runs++
	rec.Last = now

	// A file missing for its whole retry budget is accepted as gone and
	// stops failing runs from here on. It is still requested every run, so
	// an upstream that publishes it again is picked up without intervention.
	if rec.Runs >= m.retries {
		rec.Known = true
	}
	if m.mode == MissingIgnore || rec.Known {
		log.WithField("path", reqPath).Debug("Skipped a file the upstream does not serve.")
		return true
	}

	m.unresolved++
	if m.first == "" {
		m.first = reqPath
	}
	log.WithFields(log.Fields{"path": reqPath, "runs": rec.Runs, "retries": m.retries}).
		Warn("Upstream does not serve a file its metadata lists.")
	return true
}

// Finish persists what this run observed and reports the absences that have
// not yet been accepted as permanent, which is what fails the repository
// while a missing file is still being retried.
func (m *Missing) Finish() error {
	if m == nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	m.save()
	if m.unresolved == 0 {
		return nil
	}
	return fmt.Errorf("%d file(s) listed by the metadata are missing upstream and will be retried, starting with %s",
		m.unresolved, m.first)
}

// save rewrites the records for the files missed this run, and removes the
// file once nothing is missing. Only files missed this run are kept:
// anything else is served again, or is no longer listed at all, and either
// way its retry budget starts over.
func (m *Missing) save() {
	if m.dryRun {
		return
	}
	name := filepath.Join(m.root, MissingStateName)
	if len(m.seen) == 0 {
		if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
			log.WithError(err).WithField("path", name).Warn("Failed to remove the missing file state.")
		}
		return
	}
	kept := make(map[string]*missingRecord, len(m.seen))
	for reqPath := range m.seen {
		kept[reqPath] = m.records[reqPath]
	}
	data, err := json.Marshal(kept)
	if err == nil {
		err = os.MkdirAll(m.root, 0755)
	}
	if err == nil {
		err = os.WriteFile(name, data, 0644)
	}
	if err != nil {
		log.WithError(err).WithField("path", name).Warn("Failed to save the missing file state.")
	}
}