374 lines
12 KiB
Go
374 lines
12 KiB
Go
package mirror
|
|
|
|
import (
|
|
"context"
|
|
"net/url"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/grmrgecko/repo-sync/internal/testrepos"
|
|
)
|
|
|
|
// TestChildEntry verifies anchor resolution against a directory URL.
|
|
func TestChildEntry(t *testing.T) {
|
|
base, _ := url.Parse("http://example.com/repos/")
|
|
cases := []struct {
|
|
href string
|
|
want string
|
|
isDir bool
|
|
}{
|
|
{"el9/", "el9", true},
|
|
{"file.rpm", "file.rpm", false},
|
|
{"/repos/el8/", "el8", true},
|
|
{"../", "", false},
|
|
{"?C=N;O=D", "", false},
|
|
{"http://other.example.com/repos/x/", "", false},
|
|
{"/outside/", "", false},
|
|
{"deep/nested/", "", false},
|
|
{"#anchor", "", false},
|
|
}
|
|
for _, c := range cases {
|
|
got, isDir := childEntry(base, c.href)
|
|
if got != c.want || isDir != c.isDir {
|
|
t.Errorf("childEntry(%q) = %q,%v, want %q,%v", c.href, got, isDir, c.want, c.isDir)
|
|
}
|
|
}
|
|
}
|
|
|
|
// discoverOpts builds the options a discovery test crawls with.
|
|
func discoverOpts(typ RepoType, depth int) *Options {
|
|
return &Options{Type: typ, DiscoverDepth: depth}
|
|
}
|
|
|
|
// repoURLs reduces a crawl's repositories to their URLs, for comparison
|
|
// against what a fixture tree publishes.
|
|
func repoURLs(found *discovery) []string {
|
|
urls := make([]string, 0, len(found.repos))
|
|
for _, repo := range found.repos {
|
|
urls = append(urls, repo.url)
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// checkURLs verifies a crawl found exactly the expected repositories, in
|
|
// order.
|
|
func checkURLs(t *testing.T, found *discovery, want []string) {
|
|
t.Helper()
|
|
got := repoURLs(found)
|
|
if len(got) != len(want) {
|
|
t.Fatalf("found %v, want %v", got, want)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Errorf("found[%d] = %q, want %q", i, got[i], want[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestDiscoverRepos verifies RPM repositories are found through directory
|
|
// listings and that the depth limit is honored.
|
|
func TestDiscoverRepos(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "a", "repo1"))
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "b", "nested", "repo2"))
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "deep", "x", "y", "z", "repo3"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 3))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/a/repo1", srv.URL + "/b/nested/repo2"})
|
|
|
|
// A deeper limit reaches the third repository.
|
|
found, err = discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 6))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found.repos) != 3 {
|
|
t.Errorf("depth 6 found %v, want 3 repositories", repoURLs(found))
|
|
}
|
|
}
|
|
|
|
// TestDiscoverArchRepos verifies pacman repositories are recognized by
|
|
// their database files.
|
|
func TestDiscoverArchRepos(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
|
|
testrepos.BuildArchRepo(t, filepath.Join(www, "extra", "os", "x86_64"), "extra")
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoArch, 4))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/core/os/x86_64", srv.URL + "/extra/os/x86_64"})
|
|
}
|
|
|
|
// TestDiscoverApkRepos verifies Alpine repositories are recognized by
|
|
// their index files, matching the arch directories under a release tree.
|
|
func TestDiscoverApkRepos(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "x86_64"))
|
|
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "aarch64"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
found, err := discoverRepos(context.Background(), srv.URL+"/v3.24/community", discoverOpts(RepoApk, 1))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/v3.24/community/aarch64", srv.URL + "/v3.24/community/x86_64"})
|
|
}
|
|
|
|
// TestDiscoverDebRepos verifies suite directories are recognized by their
|
|
// release files.
|
|
func TestDiscoverDebRepos(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
|
|
testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoDeb, 4))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/debian/dists/test", srv.URL + "/flat"})
|
|
}
|
|
|
|
// TestDiscoverAllTypes verifies a crawl that names no type finds every
|
|
// format under one URL, as a vendor archive publishing both yum and apt
|
|
// repositories requires.
|
|
func TestDiscoverAllTypes(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
|
|
testrepos.BuildDebRepo(t, filepath.Join(www, "apt"))
|
|
testrepos.BuildArchRepo(t, filepath.Join(www, "arch", "os", "x86_64"), "core")
|
|
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "x86_64"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
found, err := discoverRepos(context.Background(), srv.URL, &Options{DiscoverDepth: 4})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := map[string]RepoType{
|
|
srv.URL + "/alpine/v3.24/x86_64": RepoApk,
|
|
srv.URL + "/apt/dists/test": RepoDeb,
|
|
srv.URL + "/arch/os/x86_64": RepoArch,
|
|
srv.URL + "/yum/el9": RepoRPM,
|
|
}
|
|
if len(found.repos) != len(want) {
|
|
t.Fatalf("found %v, want %v", found.repos, want)
|
|
}
|
|
for _, repo := range found.repos {
|
|
if want[repo.url] != repo.typ {
|
|
t.Errorf("%s discovered as %q, want %q", repo.url, repo.typ, want[repo.url])
|
|
}
|
|
}
|
|
|
|
// Limiting the crawl to one format leaves the rest of the tree alone.
|
|
found, err = discoverRepos(context.Background(), srv.URL, &Options{Types: []RepoType{RepoRPM}, DiscoverDepth: 4})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/yum/el9"})
|
|
}
|
|
|
|
// TestDiscoverExclude verifies excluded directories are neither crawled nor
|
|
// reported, by name anywhere in the tree or by path below the crawled URL.
|
|
func TestDiscoverExclude(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el", "9"))
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles", "15"))
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "fc", "40"))
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "docker", "el", "9"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
opts := discoverOpts(RepoRPM, 4)
|
|
opts.Exclude = []string{"sles", "fc", "docker"}
|
|
found, err := discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/yum/el/9"})
|
|
|
|
// A pattern with a slash pins the exclusion to one place in the tree.
|
|
opts.Exclude = []string{"yum/*"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/docker/el/9"})
|
|
}
|
|
|
|
// TestDiscoverIncludeFiles verifies loose files outside repositories are
|
|
// matched by the include patterns, and that files inside a repository are
|
|
// left to that repository's own synchronization.
|
|
func TestDiscoverIncludeFiles(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-mysql-2023"), []byte("key"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "README.txt"), []byte("notes"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "yum", "el9", "extra.rpm"), []byte("rpm"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
opts := discoverOpts(RepoRPM, 4)
|
|
opts.IncludeFiles = []string{"*.rpm", "RPM-GPG-KEY-*"}
|
|
found, err := discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, []string{srv.URL + "/yum/el9"})
|
|
|
|
want := []fileTarget{
|
|
{dirURL: srv.URL, name: "RPM-GPG-KEY-mysql-2023"},
|
|
{dirURL: srv.URL + "/yum", name: "loose-1.0.rpm"},
|
|
}
|
|
if len(found.files) != len(want) {
|
|
t.Fatalf("matched %v, want %v", found.files, want)
|
|
}
|
|
for i := range want {
|
|
if found.files[i] != want[i] {
|
|
t.Errorf("files[%d] = %v, want %v", i, found.files[i], want[i])
|
|
}
|
|
}
|
|
|
|
// An exclusion wins over an include pattern.
|
|
opts.Exclude = []string{"RPM-GPG-KEY-*"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found.files) != 1 || found.files[0].name != "loose-1.0.rpm" {
|
|
t.Errorf("matched %v, want only the loose rpm", found.files)
|
|
}
|
|
}
|
|
|
|
// TestDiscoverAnchoredPatterns verifies a leading slash limits a pattern to
|
|
// the crawled directory itself, for both include and exclude patterns.
|
|
func TestDiscoverAnchoredPatterns(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "root-1.0.rpm"), []byte("rpm"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm"))
|
|
testrepos.WriteFile(t, filepath.Join(www, "yum", "deep", "buried-1.0.rpm"), []byte("rpm"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
// Anchored, only the root directory's file is mirrored.
|
|
opts := discoverOpts(RepoRPM, 4)
|
|
opts.IncludeFiles = []string{"/*.rpm"}
|
|
found, err := discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []fileTarget{{dirURL: srv.URL, name: "root-1.0.rpm"}}
|
|
if len(found.files) != len(want) || found.files[0] != want[0] {
|
|
t.Errorf("anchored include matched %v, want %v", found.files, want)
|
|
}
|
|
|
|
// The same pattern unanchored matches at every depth, which is what
|
|
// makes the anchor worth having.
|
|
opts.IncludeFiles = []string{"*.rpm"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found.files) != 3 {
|
|
t.Errorf("unanchored include matched %v, want 3 files", found.files)
|
|
}
|
|
|
|
// An anchored exclusion applies to the crawled directory only, leaving
|
|
// the same name deeper in the tree alone.
|
|
opts.Exclude = []string{"/root-1.0.rpm"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found.files) != 2 {
|
|
t.Errorf("anchored exclude left %v, want 2 files", found.files)
|
|
}
|
|
for _, file := range found.files {
|
|
if file.name == "root-1.0.rpm" {
|
|
t.Errorf("anchored exclude kept %v", file)
|
|
}
|
|
}
|
|
|
|
// An anchored directory exclusion keeps the crawl out of a top-level
|
|
// directory without matching that name elsewhere.
|
|
opts.Exclude = []string{"/deep"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found.files) != 3 {
|
|
t.Errorf("top-level exclude of an unrelated name left %v, want 3 files", found.files)
|
|
}
|
|
opts.Exclude = []string{"/yum"}
|
|
found, err = discoverRepos(context.Background(), srv.URL, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkURLs(t, found, nil)
|
|
if len(found.files) != 1 || found.files[0].name != "root-1.0.rpm" {
|
|
t.Errorf("anchored directory exclude left %v, want only the root rpm", found.files)
|
|
}
|
|
}
|
|
|
|
// TestMatchSubject verifies the pairing of a pattern with the string it is
|
|
// matched against, which decides whether a pattern is anchored, pinned by
|
|
// path, or matched by name anywhere in the tree.
|
|
func TestMatchSubject(t *testing.T) {
|
|
cases := []struct {
|
|
pattern string
|
|
wantPattern string
|
|
wantSubject string
|
|
}{
|
|
{"*.rpm", "*.rpm", "pkg.rpm"},
|
|
{"/*.rpm", "*.rpm", "yum/deep/pkg.rpm"},
|
|
{"yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"},
|
|
{"/yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"},
|
|
}
|
|
for _, c := range cases {
|
|
pattern, subject := matchSubject(c.pattern, "pkg.rpm", "yum/deep/pkg.rpm")
|
|
if pattern != c.wantPattern || subject != c.wantSubject {
|
|
t.Errorf("matchSubject(%q) = %q,%q, want %q,%q",
|
|
c.pattern, pattern, subject, c.wantPattern, c.wantSubject)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestDetectType verifies a repository URL given without a type is
|
|
// identified by its metadata, and that an unidentifiable URL is reported.
|
|
func TestDetectType(t *testing.T) {
|
|
www := t.TempDir()
|
|
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
|
|
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
|
|
testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
|
|
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine"))
|
|
srv := testrepos.ServeDir(t, www)
|
|
|
|
cases := []struct {
|
|
path string
|
|
want RepoType
|
|
}{
|
|
{"/el9", RepoRPM},
|
|
{"/debian/dists/test", RepoDeb},
|
|
{"/core/os/x86_64", RepoArch},
|
|
{"/alpine", RepoApk},
|
|
}
|
|
for _, c := range cases {
|
|
got, err := detectType(context.Background(), srv.URL+c.path, AllTypes)
|
|
if err != nil {
|
|
t.Errorf("detectType(%s): %v", c.path, err)
|
|
continue
|
|
}
|
|
if got != c.want {
|
|
t.Errorf("detectType(%s) = %q, want %q", c.path, got, c.want)
|
|
}
|
|
}
|
|
|
|
// A URL holding no repository metadata cannot be identified.
|
|
if _, err := detectType(context.Background(), srv.URL+"/debian", AllTypes); err == nil {
|
|
t.Error("expected an error detecting the type of an archive root")
|
|
}
|
|
}
|