package mirror import ( "context" "os" "path/filepath" "strconv" "strings" "testing" "time" "github.com/grmrgecko/repo-sync/fetch" "github.com/grmrgecko/repo-sync/internal/testrepos" ) // traceOptions builds sync options with tracing enabled and a fixed set of // mirror details, so tests assert on known values. func traceOptions(typ RepoType, dest string) *Options { return &Options{ Type: typ, Destination: dest, Workers: 2, Trace: true, TraceInfo: TraceInfo{ Host: "mirror.example.com", Maintainer: "Test Maintainer ", Sponsor: "Example Inc", Country: "US", Location: "Texas", Throughput: "10Gb", }, } } // traceField returns the value following a field prefix in a trace. func traceField(t *testing.T, trace, prefix string) string { t.Helper() for _, line := range strings.Split(trace, "\n") { if strings.HasPrefix(line, prefix) { return strings.TrimPrefix(line, prefix) } } t.Fatalf("trace has no %q field\ngot:\n%s", prefix, trace) return "" } // readTrace returns the trace file published under root. func readTrace(t *testing.T, root string) string { t.Helper() data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(TraceDir), "mirror.example.com")) if err != nil { t.Fatal("reading trace:", err) } return string(data) } // TestTraceName verifies a trace host is reduced to a single path element, // so a configured value cannot direct the file outside the repository. func TestTraceName(t *testing.T) { cases := []struct { host string want string ok bool }{ {"mirror.example.com", "mirror.example.com", true}, {" mirror.example.com ", "mirror.example.com", true}, {"../../etc/cron.d/evil", "evil", true}, {"/etc/passwd", "passwd", true}, {`..\..\windows`, "windows", true}, {"sub/dir/host", "host", true}, {"", "", false}, {"..", "", false}, {"/", "", false}, } for _, c := range cases { got, err := traceName(c.host) if c.ok && err != nil { t.Errorf("traceName(%q) unexpected error: %v", c.host, err) continue } if !c.ok { if err == nil { t.Errorf("traceName(%q) = %q, want error", c.host, got) } continue } if got != c.want { t.Errorf("traceName(%q) = %q, want %q", c.host, got, c.want) } } } // TestSyncTraceWritten verifies a trace lands in the repository directory // carrying the configured mirror details and the upstream actually used. func TestSyncTraceWritten(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) repoURL := srv.URL + "/alpine/v3.24/community/x86_64" if err := syncOne(context.Background(), repoURL, opts); err != nil { t.Fatal(err) } trace := readTrace(t, filepath.Join(dest, "alpine", "v3.24", "community", "x86_64")) for _, want := range []string{ "Date: ", "Date-Started: ", "Creator: repo-sync ", "Running on host: mirror.example.com", "Maintainer: Test Maintainer ", "Sponsor: Example Inc", "Country: US", "Location: Texas", "Throughput: 10Gb", "Repository type: apk", "Upstream-mirror: " + repoURL, "Total bytes received: ", "Total time spent syncing: ", } { if !strings.Contains(trace, want) { t.Errorf("trace missing %q\ngot:\n%s", want, trace) } } // The first line is the plain completion date the convention expects, // carrying no field name of its own and matching date(1)'s UTC output. first, _, _ := strings.Cut(trace, "\n") if strings.HasPrefix(first, "Date:") { t.Errorf("first line should be the bare date, got %q", first) } const dateLayout = "Mon Jan _2 15:04:05 UTC 2006" parsed, err := time.Parse(dateLayout, first) if err != nil { t.Fatalf("first line %q is not a date(1)-style UTC date: %v", first, err) } // Parsing alone is too lenient about spacing to catch a malformed day // field, so require the line to survive a round trip unchanged. if canonical := parsed.Format(dateLayout); canonical != first { t.Errorf("first line %q is not canonically formatted, want %q", first, canonical) } } // TestSyncTraceOmitsUnsetFields verifies fields with no configured value // are left out rather than written empty. func TestSyncTraceOmitsUnsetFields(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := &Options{ Type: RepoApk, Destination: dest, Workers: 2, Trace: true, TraceInfo: TraceInfo{Host: "mirror.example.com"}, } if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } trace := readTrace(t, filepath.Join(dest, "repo")) for _, absent := range []string{"Maintainer:", "Sponsor:", "Country:", "Location:", "Throughput:"} { if strings.Contains(trace, absent) { t.Errorf("trace contains unset field %q\ngot:\n%s", absent, trace) } } if !strings.Contains(trace, "Running on host: mirror.example.com") { t.Errorf("trace missing host\ngot:\n%s", trace) } } // TestSyncTraceDisabled verifies no trace directory appears when tracing // is off, which is the default. func TestSyncTraceDisabled(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dest, "repo", "project")); err == nil { t.Error("trace written while tracing is disabled") } } // TestSyncTraceSurvivesPrune verifies a trace is not treated as a stale // file. A flat layout puts the trace inside the pruned tree itself, which // is the case that would otherwise delete it on the following sync. func TestSyncTraceSurvivesPrune(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) opts.Flat = true opts.Prune = true // Two syncs: the first writes the trace, the second prunes with that // trace already on disk. for i := 0; i < 2; i++ { if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatalf("sync %d: %v", i+1, err) } if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { t.Fatalf("trace missing after sync %d: %v", i+1, err) } } // A stale file beside it is still pruned, confirming pruning ran. stale := filepath.Join(dest, "old-1.0-r0.apk") if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { t.Fatal(err) } if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if _, err := os.Stat(stale); err == nil { t.Error("stale package survived pruning, so pruning did not run") } if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { t.Error("trace pruned:", err) } } // TestSyncTraceDryRun verifies a dry run neither writes a trace nor // reports an existing one as a file it would prune. func TestSyncTraceDryRun(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) opts.Flat = true opts.DryRun = true opts.Prune = true if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dest, "project")); err == nil { t.Error("dry run wrote a trace") } // With a trace already on disk, a dry run must not count it as // prunable. The file is planted directly so this stands on its own // rather than depending on a preceding sync having written one. traceFile := filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com") if err := os.MkdirAll(filepath.Dir(traceFile), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(traceFile, []byte("previous trace\n"), 0644); err != nil { t.Fatal(err) } fetch.Stats.Reset() if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if n := fetch.Stats.Pruned.Load(); n != 0 { t.Errorf("dry run would prune %d files, want 0 (the trace must be kept)", n) } if _, err := os.Stat(traceFile); err != nil { t.Error("dry run removed the existing trace:", err) } } // TestSyncTraceDebAtArchiveRoot verifies an apt trace is published at the // archive root, where the convention places it, rather than in the suite. func TestSyncTraceDebAtArchiveRoot(t *testing.T) { www := t.TempDir() testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoDeb, dest) opts.Architectures = []string{"amd64", "source"} if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts); err != nil { t.Fatal(err) } root := filepath.Join(dest, "debian") trace := readTrace(t, root) if !strings.Contains(trace, "Repository type: deb") { t.Errorf("trace missing repository type\ngot:\n%s", trace) } if !strings.Contains(trace, "Architectures: amd64 source") { t.Errorf("trace missing architectures\ngot:\n%s", trace) } if _, err := os.Stat(filepath.Join(root, "dists", "test", "project")); err == nil { t.Error("trace written into the suite directory instead of the archive root") } } // TestSyncIntoWritesTrace verifies the mirror server's entry point // publishes a trace like the sync commands do, and that it still leaves the // caller's options untouched. func TestSyncIntoWritesTrace(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) src := fetch.NewSource([]string{srv.URL + "/repo"}) if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil { t.Fatal(err) } trace := readTrace(t, dest) if !strings.Contains(trace, "Running on host: mirror.example.com") { t.Errorf("server trace missing host\ngot:\n%s", trace) } if !opts.Trace { t.Error("SyncInto mutated the caller's options") } } // TestSyncIntoTraceDisabled verifies the server path honors tracing being // off, which is the default. func TestSyncIntoTraceDisabled(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} src := fetch.NewSource([]string{srv.URL + "/repo"}) if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dest, "project")); err == nil { t.Error("trace written while tracing is disabled") } } // TestSyncTraceReportsOwnTransfer verifies a trace reports the bytes its own // synchronization moved rather than whatever the shared collector holds. // That isolation is what makes a trace meaningful on the server's path, // where crawls run concurrently and all report into the same collector. func TestSyncTraceReportsOwnTransfer(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) // Stand in for a concurrent crawl's transfer, which must not be // attributed to this run. const foreign = int64(1) << 40 fetch.Stats.Reset() fetch.Stats.FetchedBytes.Add(foreign) dest := t.TempDir() opts := traceOptions(RepoApk, dest) if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } got := traceField(t, readTrace(t, filepath.Join(dest, "repo")), "Total bytes received: ") bytes, err := strconv.ParseInt(got, 10, 64) if err != nil { t.Fatalf("byte count %q: %v", got, err) } if bytes <= 0 { t.Errorf("trace reports %d bytes received, want the run's own transfer", bytes) } if bytes >= foreign { t.Errorf("trace reports %d bytes received, which includes another run's transfer", bytes) } } // TestSyncTraceMirrorsUpstream verifies upstream's own traces are copied // into this mirror's trace directory, so the chain back to the origin // archive is preserved, while the file naming this mirror stays the one // this run wrote. func TestSyncTraceMirrorsUpstream(t *testing.T) { www := t.TempDir() repo := filepath.Join(www, "repo") testrepos.BuildApkRepo(t, repo) upstreamTrace := filepath.Join(repo, filepath.FromSlash(TraceDir)) testrepos.WriteFile(t, filepath.Join(upstreamTrace, "origin.example.net"), []byte("origin trace\n")) testrepos.WriteFile(t, filepath.Join(upstreamTrace, "middle.example.net"), []byte("middle trace\n")) // Upstream also carries a trace under this mirror's own name, which // describes a different mirror and must not replace ours. testrepos.WriteFile(t, filepath.Join(upstreamTrace, "mirror.example.com"), []byte("someone else\n")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) opts.Prune = true if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } traceDir := filepath.Join(dest, "repo", filepath.FromSlash(TraceDir)) for name, want := range map[string]string{ "origin.example.net": "origin trace\n", "middle.example.net": "middle trace\n", } { data, err := os.ReadFile(filepath.Join(traceDir, name)) if err != nil { t.Errorf("upstream trace %s not mirrored: %v", name, err) continue } if string(data) != want { t.Errorf("upstream trace %s = %q, want %q", name, data, want) } } if trace := readTrace(t, filepath.Join(dest, "repo")); !strings.Contains(trace, "Running on host: mirror.example.com") { t.Errorf("upstream trace replaced this mirror's own\ngot:\n%s", trace) } // A second sync prunes with the mirrored traces already on disk; they // belong to the repository and must survive. if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(traceDir, "origin.example.net")); err != nil { t.Error("mirrored upstream trace pruned:", err) } } // TestSyncTraceUpstreamAbsent verifies an upstream that publishes no traces // is not a synchronization failure. func TestSyncTraceUpstreamAbsent(t *testing.T) { www := t.TempDir() testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) srv := testrepos.ServeDir(t, www) dest := t.TempDir() opts := traceOptions(RepoApk, dest) if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dest, "repo", filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { t.Error("trace missing:", err) } } // TestTraceArchitectures verifies architectures are only claimed for apt // runs that were actually restricted to them. func TestTraceArchitectures(t *testing.T) { if got := traceArchitectures(&Options{Type: RepoDeb, Architectures: []string{"source", "amd64"}}); got != "amd64 source" { t.Errorf("deb architectures = %q, want %q", got, "amd64 source") } if got := traceArchitectures(&Options{Type: RepoDeb}); got != "" { t.Errorf("unrestricted deb architectures = %q, want empty", got) } if got := traceArchitectures(&Options{Type: RepoRPM, Architectures: []string{"x86_64"}}); got != "" { t.Errorf("rpm architectures = %q, want empty", got) } }