repo-sync/mirror/discover.go

package mirror

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"slices"
	"sort"
	"strings"

	"github.com/grmrgecko/repo-sync/fetch"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/html"
)

// dirListing holds the entries parsed from an HTML directory index.
type dirListing struct {
	dirs  []string
	files map[string]bool
}

// discoverRepos crawls directory listings under baseURL looking for
// repositories of the given type, descending at most maxDepth levels.
func discoverRepos(ctx context.Context, baseURL string, typ RepoType, maxDepth int) ([]string, error) {
	type node struct {
		url   string
		depth int
	}
	start := strings.TrimRight(baseURL, "/") + "/"
	queue := []node{{url: start}}
	visited := map[string]bool{start: true}
	var found []string

	for len(queue) > 0 {
		if err := ctx.Err(); err != nil {
			return nil, err
		}
		n := queue[0]
		queue = queue[1:]
		listing, err := listDir(ctx, n.url)
		if err != nil {
			log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.")
			continue
		}

		// A directory containing repository metadata is synchronized as a
		// whole; its children are not crawled further.
		switch typ {
		case RepoRPM:
			if slices.Contains(listing.dirs, "repodata") {
				found = append(found, strings.TrimRight(n.url, "/"))
				continue
			}
		case RepoDeb:
			if listing.files["InRelease"] || listing.files["Release"] {
				found = append(found, strings.TrimRight(n.url, "/"))
				continue
			}
		case RepoArch:
			if hasArchDB(listing.files) {
				found = append(found, strings.TrimRight(n.url, "/"))
				continue
			}
		case RepoApk:
			if listing.files[APKIndexName] {
				found = append(found, strings.TrimRight(n.url, "/"))
				continue
			}
		}
		if n.depth >= maxDepth {
			continue
		}
		for _, dir := range listing.dirs {
			child := n.url + url.PathEscape(dir) + "/"
			if visited[child] {
				continue
			}
			visited[child] = true
			queue = append(queue, node{url: child, depth: n.depth + 1})
		}
	}
	sort.Strings(found)
	return found, nil
}

// hasArchDB reports whether a directory listing contains a pacman database.
func hasArchDB(files map[string]bool) bool {
	for name := range files {
		if strings.HasSuffix(name, ".db") {
			return true
		}
	}
	return false
}

// listDir fetches an HTML directory index and extracts its immediate child
// directories and files.
func listDir(ctx context.Context, dirURL string) (*dirListing, error) {
	resp, err := fetch.Get(ctx, dirURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode)
	}

	base, err := url.Parse(dirURL)
	if err != nil {
		return nil, err
	}
	listing := &dirListing{files: map[string]bool{}}
	seenDirs := map[string]bool{}
	tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20))
	for {
		tt := tok.Next()
		if tt == html.ErrorToken {
			break
		}
		if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
			continue
		}
		name, hasAttr := tok.TagName()
		if string(name) != "a" || !hasAttr {
			continue
		}
		var href string
		for {
			key, val, more := tok.TagAttr()
			if string(key) == "href" {
				href = string(val)
			}
			if !more {
				break
			}
		}
		entry, isDir := childEntry(base, href)
		if entry == "" {
			continue
		}
		if isDir {
			if !seenDirs[entry] {
				seenDirs[entry] = true
				listing.dirs = append(listing.dirs, entry)
			}
		} else {
			listing.files[entry] = true
		}
	}
	return listing, nil
}

// childEntry resolves an anchor href against the directory URL, returning
// the entry name when it is an immediate child of that directory.
func childEntry(base *url.URL, href string) (string, bool) {
	if href == "" || strings.HasPrefix(href, "#") {
		return "", false
	}
	ref, err := url.Parse(href)
	if err != nil || ref.RawQuery != "" || ref.Fragment != "" {
		return "", false
	}
	resolved := base.ResolveReference(ref)
	if resolved.Scheme != base.Scheme || resolved.Host != base.Host {
		return "", false
	}
	rel := strings.TrimPrefix(resolved.Path, base.Path)
	if rel == "" || rel == resolved.Path {
		return "", false
	}
	isDir := strings.HasSuffix(rel, "/")
	rel = strings.TrimSuffix(rel, "/")
	if rel == "" || strings.Contains(rel, "/") {
		return "", false
	}
	return rel, isDir
}