package mirror import ( "context" "errors" "fmt" "net/url" "github.com/grmrgecko/repo-sync/fetch" log "github.com/sirupsen/logrus" ) // probePaths lists the metadata files that identify a repository of a type. // Types without fixed-name metadata return nothing. func probePaths(typ RepoType) []string { switch typ { case RepoRPM: return []string{"repodata/repomd.xml"} case RepoDeb: return []string{"InRelease", "Release"} case RepoApk: return []string{APKIndexName} } return nil } // hasQueryOrFragment reports whether a URL carries a query or fragment, // which makes it unusable as a base for joining repository paths onto. // Unparsable URLs are reported as plain so the caller still probes them. func hasQueryOrFragment(rawURL string) bool { u, err := url.Parse(rawURL) if err != nil { return false } return u.RawQuery != "" || u.Fragment != "" } // resolveSource decides whether repoURL is a repository root or a mirror // list and returns the source to fetch from, along with the URL whose path // reflects the repository layout: repoURL itself for a direct repository, // or the first mirror's URL when repoURL is a mirror list. RPM mirror lists // may be plain text or metalink documents. func resolveSource(ctx context.Context, repoURL string, typ RepoType) (*fetch.Source, string, error) { // Pacman databases are named after the repository, so there is no // fixed path to probe and pacman mirrorlists are config snippets // rather than URL lists; the URL is used directly. direct := fetch.NewSource([]string{repoURL}) probes := probePaths(typ) if len(probes) == 0 { return direct, repoURL, nil } // Probe for repository metadata directly under the URL. A URL carrying a // query or fragment cannot have a repository path joined onto it, as the // path would land inside the query, so skip the probe: such URLs are // mirrorlist or metalink endpoints and are read as one below. if !hasQueryOrFragment(repoURL) { for _, probe := range probes { resp, err := direct.Get(ctx, probe, fetch.GetOptions{}) if err == nil { resp.Body.Close() return direct, repoURL, nil } if !errors.Is(err, fetch.ErrNotFound) { return nil, "", err } } } // Fall back to reading the URL itself as a mirror list. body, err := fetch.ReadURL(ctx, repoURL) if err != nil { return nil, "", fmt.Errorf("read mirrorlist %s: %w", repoURL, err) } kind := "mirrorlist" var bases []string if typ == RepoRPM { if bases = fetch.ParseMetalink(body); len(bases) > 0 { kind = "metalink" } } if len(bases) == 0 { bases = fetch.ParseMirrorlist(body) } if len(bases) == 0 { return nil, "", fmt.Errorf("%s is neither a repository nor a mirrorlist", repoURL) } log.WithFields(log.Fields{"url": repoURL, "mirrors": len(bases), "kind": kind}).Info("Using mirror list for repository.") return fetch.NewSource(bases), bases[0], nil }