repo-sync/mirror/sync_test.go

package mirror

import (
	"context"
	"encoding/json"
	"io/fs"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/grmrgecko/repo-sync/fetch"
	"github.com/grmrgecko/repo-sync/internal/testrepos"
)

// countFiles returns the number of regular files under root.
func countFiles(t *testing.T, root string) int {
	t.Helper()
	count := 0
	err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() {
			count++
		}
		return nil
	})
	if err != nil && !os.IsNotExist(err) {
		t.Fatal(err)
	}
	return count
}

// TestDryRun verifies a dry run leaves the destination unchanged: no
// packages downloaded, no metadata published, and no files pruned.
func TestDryRun(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
	srv := testrepos.ServeDir(t, www)

	// A dry run against an empty destination writes no files at all.
	dest := t.TempDir()
	opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, DryRun: true, Prune: true}
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	if n := countFiles(t, dest); n != 0 {
		t.Errorf("dry run left %d files in the destination", n)
	}

	// A real run works, and a later dry run with pruning keeps stale files.
	opts.DryRun = false
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	stale := filepath.Join(dest, "el9", "Packages", "stale.rpm")
	if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
		t.Fatal(err)
	}
	opts.DryRun = true
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(stale); err != nil {
		t.Error("dry run pruned a file:", err)
	}
}

// TestPruneGrace verifies stale files wait out the grace period across
// runs before being removed.
func TestPruneGrace(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
	srv := testrepos.ServeDir(t, www)

	dest := t.TempDir()
	opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, Prune: true, PruneGrace: time.Hour}
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}

	local := filepath.Join(dest, "el9")
	stale := filepath.Join(local, "Packages", "stale.rpm")
	if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
		t.Fatal(err)
	}

	// The first run only starts the grace period.
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(stale); err != nil {
		t.Fatal("file pruned before its grace period expired:", err)
	}
	statePath := filepath.Join(local, fetch.PruneStateName)
	if _, err := os.Stat(statePath); err != nil {
		t.Fatal("prune state not recorded:", err)
	}

	// A second run within the grace period still keeps the file.
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(stale); err != nil {
		t.Fatal("file pruned within its grace period:", err)
	}

	// Age the recorded timestamp past the grace period; the next run
	// prunes the file and clears the bookkeeping.
	data, err := os.ReadFile(statePath)
	if err != nil {
		t.Fatal(err)
	}
	state := map[string]time.Time{}
	if err := json.Unmarshal(data, &state); err != nil {
		t.Fatal(err)
	}
	for rel := range state {
		state[rel] = time.Now().Add(-2 * time.Hour)
	}
	data, err = json.Marshal(state)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(statePath, data, 0644); err != nil {
		t.Fatal(err)
	}
	if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(stale); err == nil {
		t.Error("stale file survived past its grace period")
	}
	if _, err := os.Stat(statePath); err == nil {
		t.Error("prune state lingered after all grace periods resolved")
	}
}

// TestLockDestination verifies a held destination lock blocks a second run,
// is released on release, and leaves no lock file behind in the tree.
func TestLockDestination(t *testing.T) {
	dest := t.TempDir()
	lock, err := lockDestination(dest)
	if err != nil {
		t.Fatal(err)
	}
	name := filepath.Join(dest, fetch.LockFileName)
	if _, err := os.Stat(name); err != nil {
		t.Error("no lock file while the lock is held:", err)
	}
	if _, err := lockDestination(dest); err == nil {
		t.Error("second lock unexpectedly succeeded")
	}

	lock.release()
	if _, err := os.Stat(name); !os.IsNotExist(err) {
		t.Error("lock file survived the run that took it")
	}
	lock, err = lockDestination(dest)
	if err != nil {
		t.Error("lock not released:", err)
	} else {
		lock.release()
	}
}

// TestLockDestinationReplaced verifies a lock taken on a file that is no
// longer the one at the path is not accepted: a run that released between
// the open and the lock takes its file with it, and the lock left over
// guards nothing the next run would consult.
func TestLockDestinationReplaced(t *testing.T) {
	dest := t.TempDir()
	name := filepath.Join(dest, fetch.LockFileName)

	// Stand in for the released run by replacing the lock file with a new
	// one, which is the state a racing run would find.
	stale, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		t.Fatal(err)
	}
	defer stale.Close()
	if err := os.Remove(name); err != nil {
		t.Fatal(err)
	}
	replacement, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		t.Fatal(err)
	}
	replacement.Close()

	lock, err := lockDestination(dest)
	if err != nil {
		t.Fatal(err)
	}
	defer lock.release()
	held, err := lock.file.Stat()
	if err != nil {
		t.Fatal(err)
	}
	current, err := os.Stat(name)
	if err != nil {
		t.Fatal(err)
	}
	if !os.SameFile(held, current) {
		t.Error("locked a file that is no longer the destination's lock file")
	}
}

// TestSyncSummary verifies a run reports what it changed: the first run
// fetches, a repeat run finds everything unchanged, and a removed upstream
// file is counted as pruned.
func TestSyncSummary(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
	srv := testrepos.ServeDir(t, www)

	dest := t.TempDir()
	opts := &Options{
		Type:        RepoRPM,
		URLs:        []string{srv.URL + "/el9"},
		Destination: dest,
		Workers:     2,
		Prune:       true,
	}

	// The first run populates the destination, so everything it fetched is a
	// change.
	summary, err := Sync(context.Background(), opts)
	if err != nil {
		t.Fatal(err)
	}
	if summary.Repositories != 1 || summary.Failed != 0 {
		t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed)
	}
	if summary.Fetched == 0 {
		t.Error("first run reported nothing fetched")
	}
	if summary.FetchedBytes == 0 {
		t.Error("first run reported no bytes fetched")
	}
	if !summary.Changed() {
		t.Error("first run did not report a change")
	}

	// Repeating the run transfers nothing, which is what a hook keys off to
	// stay idle.
	summary, err = Sync(context.Background(), opts)
	if err != nil {
		t.Fatal(err)
	}
	if summary.Fetched != 0 || summary.Pruned != 0 {
		t.Errorf("repeat run fetched %d and pruned %d, want 0 and 0", summary.Fetched, summary.Pruned)
	}
	if summary.Unchanged == 0 {
		t.Error("repeat run reported nothing unchanged")
	}
	if summary.Changed() {
		t.Error("repeat run reported a change")
	}

	// A local file the upstream never served is pruned, which counts as a
	// change even though nothing was fetched.
	stale := filepath.Join(dest, "el9", "Packages", "stale.rpm")
	if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
		t.Fatal(err)
	}
	summary, err = Sync(context.Background(), opts)
	if err != nil {
		t.Fatal(err)
	}
	if summary.Pruned != 1 {
		t.Errorf("pruned = %d, want 1", summary.Pruned)
	}
	if !summary.Changed() {
		t.Error("pruning run did not report a change")
	}
}

// TestSyncSummaryDryRun verifies a dry run reports its planned work without
// ever claiming the mirror changed.
func TestSyncSummaryDryRun(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
	srv := testrepos.ServeDir(t, www)

	opts := &Options{
		Type:        RepoRPM,
		URLs:        []string{srv.URL + "/el9"},
		Destination: t.TempDir(),
		Workers:     2,
		DryRun:      true,
	}
	summary, err := Sync(context.Background(), opts)
	if err != nil {
		t.Fatal(err)
	}
	if summary.Planned == 0 {
		t.Error("dry run planned nothing")
	}
	if summary.PlannedBytes == 0 {
		t.Error("dry run planned no bytes")
	}
	// A dry run still fetches metadata to plan against, but none of it
	// reaches the destination, so the run must never read as a change.
	if summary.Changed() {
		t.Error("dry run reported a change")
	}
}

// TestSyncSummaryFailure verifies a run that fails still reports the totals
// it accumulated, so a caller can tell a failed run apart from an idle one.
func TestSyncSummaryFailure(t *testing.T) {
	www := t.TempDir()
	srv := testrepos.ServeDir(t, www)

	opts := &Options{
		Type:        RepoRPM,
		URLs:        []string{srv.URL + "/missing"},
		Destination: t.TempDir(),
		Workers:     2,
	}
	summary, err := Sync(context.Background(), opts)
	if err == nil {
		t.Fatal("expected a failure syncing a repository that does not exist")
	}
	if summary == nil {
		t.Fatal("no summary returned for a failed run")
	}
	if summary.Repositories != 1 || summary.Failed != 1 {
		t.Errorf("repositories = %d, failed = %d, want 1 and 1", summary.Repositories, summary.Failed)
	}
	if summary.Changed() {
		t.Error("a run that transferred nothing reported a change")
	}
}

// TestSyncDiscoverMixed verifies one run covers an upstream publishing more
// than one repository format, skips the trees it was told to exclude, and
// mirrors the loose files it was told to include beside the repositories.
func TestSyncDiscoverMixed(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
	testrepos.BuildDebRepo(t, filepath.Join(www, "apt"))
	testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles15"))
	testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-test"), []byte("key"))
	testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("loose"))
	testrepos.WriteFile(t, filepath.Join(www, "yum", "notes.txt"), []byte("ignored"))
	srv := testrepos.ServeDir(t, www)

	dest := t.TempDir()
	opts := &Options{
		URLs:          []string{srv.URL},
		Destination:   dest,
		Discover:      true,
		DiscoverDepth: 4,
		Exclude:       []string{"sles*"},
		IncludeFiles:  []string{"*.rpm", "RPM-GPG-KEY-*"},
		Workers:       2,
	}
	summary, err := Sync(context.Background(), opts)
	if err != nil {
		t.Fatal(err)
	}
	if summary.Repositories != 2 || summary.Failed != 0 {
		t.Errorf("repositories = %d, failed = %d, want 2 and 0", summary.Repositories, summary.Failed)
	}

	// Both formats landed, each under its own upstream path.
	for _, rel := range []string{
		filepath.Join("yum", "el9", "repodata", "repomd.xml"),
		filepath.Join("apt", "dists", "test", "Release"),
		filepath.Join("apt", "pool", "main", "h", "hello", "hello_1.0_amd64.deb"),
		"RPM-GPG-KEY-test",
		filepath.Join("yum", "loose-1.0.rpm"),
	} {
		if _, err := os.Stat(filepath.Join(dest, rel)); err != nil {
			t.Errorf("missing %s: %v", rel, err)
		}
	}

	// The excluded tree was never crawled, and a file no pattern matched
	// was never fetched.
	for _, rel := range []string{
		filepath.Join("yum", "sles15"),
		filepath.Join("yum", "notes.txt"),
	} {
		if _, err := os.Stat(filepath.Join(dest, rel)); err == nil {
			t.Errorf("%s was synchronized despite not being asked for", rel)
		}
	}
}

// TestSyncDetectType verifies a repository URL given without a type is
// synchronized as whatever its metadata identifies it to be.
func TestSyncDetectType(t *testing.T) {
	www := t.TempDir()
	testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
	srv := testrepos.ServeDir(t, www)

	dest := t.TempDir()
	summary, err := Sync(context.Background(), &Options{
		URLs:        []string{srv.URL + "/el9"},
		Destination: dest,
		Workers:     2,
	})
	if err != nil {
		t.Fatal(err)
	}
	if summary.Repositories != 1 || summary.Failed != 0 {
		t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed)
	}
	if _, err := os.Stat(filepath.Join(dest, "el9", "repodata", "repomd.xml")); err != nil {
		t.Error("repository not synchronized:", err)
	}
}