package mirror import ( "context" "fmt" "io" "net/url" "slices" "sort" "strings" "github.com/grmrgecko/repo-sync/fetch" log "github.com/sirupsen/logrus" "golang.org/x/net/html" ) // dirListing holds the entries parsed from an HTML directory index. type dirListing struct { dirs []string files map[string]bool } // discoverRepos crawls directory listings under baseURL looking for // repositories of the given type, descending at most maxDepth levels. func discoverRepos(ctx context.Context, baseURL string, typ RepoType, maxDepth int) ([]string, error) { type node struct { url string depth int } start := strings.TrimRight(baseURL, "/") + "/" queue := []node{{url: start}} visited := map[string]bool{start: true} var found []string for len(queue) > 0 { if err := ctx.Err(); err != nil { return nil, err } n := queue[0] queue = queue[1:] listing, err := listDir(ctx, n.url) if err != nil { log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.") continue } // A directory containing repository metadata is synchronized as a // whole; its children are not crawled further. switch typ { case RepoRPM: if slices.Contains(listing.dirs, "repodata") { found = append(found, strings.TrimRight(n.url, "/")) continue } case RepoDeb: if listing.files["InRelease"] || listing.files["Release"] { found = append(found, strings.TrimRight(n.url, "/")) continue } case RepoArch: if hasArchDB(listing.files) { found = append(found, strings.TrimRight(n.url, "/")) continue } case RepoApk: if listing.files[APKIndexName] { found = append(found, strings.TrimRight(n.url, "/")) continue } } if n.depth >= maxDepth { continue } for _, dir := range listing.dirs { child := n.url + url.PathEscape(dir) + "/" if visited[child] { continue } visited[child] = true queue = append(queue, node{url: child, depth: n.depth + 1}) } } sort.Strings(found) return found, nil } // hasArchDB reports whether a directory listing contains a pacman database. func hasArchDB(files map[string]bool) bool { for name := range files { if strings.HasSuffix(name, ".db") { return true } } return false } // listDir fetches an HTML directory index and extracts its immediate child // directories and files. func listDir(ctx context.Context, dirURL string) (*dirListing, error) { resp, err := fetch.Get(ctx, dirURL) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode) } base, err := url.Parse(dirURL) if err != nil { return nil, err } listing := &dirListing{files: map[string]bool{}} seenDirs := map[string]bool{} tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20)) for { tt := tok.Next() if tt == html.ErrorToken { break } if tt != html.StartTagToken && tt != html.SelfClosingTagToken { continue } name, hasAttr := tok.TagName() if string(name) != "a" || !hasAttr { continue } var href string for { key, val, more := tok.TagAttr() if string(key) == "href" { href = string(val) } if !more { break } } entry, isDir := childEntry(base, href) if entry == "" { continue } if isDir { if !seenDirs[entry] { seenDirs[entry] = true listing.dirs = append(listing.dirs, entry) } } else { listing.files[entry] = true } } return listing, nil } // childEntry resolves an anchor href against the directory URL, returning // the entry name when it is an immediate child of that directory. func childEntry(base *url.URL, href string) (string, bool) { if href == "" || strings.HasPrefix(href, "#") { return "", false } ref, err := url.Parse(href) if err != nil || ref.RawQuery != "" || ref.Fragment != "" { return "", false } resolved := base.ResolveReference(ref) if resolved.Scheme != base.Scheme || resolved.Host != base.Host { return "", false } rel := strings.TrimPrefix(resolved.Path, base.Path) if rel == "" || rel == resolved.Path { return "", false } isDir := strings.HasSuffix(rel, "/") rel = strings.TrimSuffix(rel, "/") if rel == "" || strings.Contains(rel, "/") { return "", false } return rel, isDir }