commit 6782f33778648d58a1e83197c4811e96cd0d9f38 Author: James Coleman Date: Mon Jul 27 14:16:32 2026 -0500 first commit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9527d4f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,31 @@ +on: + release: + types: [created] + +permissions: + contents: write + packages: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - + name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: '~> v2' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_golang.yaml b/.github/workflows/test_golang.yaml new file mode 100644 index 0000000..01faee4 --- /dev/null +++ b/.github/workflows/test_golang.yaml @@ -0,0 +1,21 @@ +name: Go package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a7f429a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/repo-sync +/dist/ + +*.exe +*.test +*.out +*.prof diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d6d3422 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,61 @@ +# GoReleaser config for repo-sync. +# https://goreleaser.com +# +# CGO is disabled so the binary is fully static (no glibc dependency) and runs +# unmodified across modern Linux distributions. +version: 2 + +project_name: repo-sync + +before: + hooks: + - go mod tidy + - go test ./... + +builds: + - id: repo-sync + main: . + binary: repo-sync + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -X github.com/grmrgecko/repo-sync/config.Version={{ .Version }} + - -X github.com/grmrgecko/repo-sync/config.Commit={{ .ShortCommit }} + - -X github.com/grmrgecko/repo-sync/config.Date={{ .Date }} + - -X github.com/grmrgecko/repo-sync/config.Mode=release + goos: + - linux + goarch: + - "386" + - amd64 + - arm + - arm64 + - ppc64le + goarm: + - "6" + +archives: + - id: default + formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + files: + - README.md + - LICENSE + +checksum: + name_template: "checksums.txt" + +snapshot: + version_template: "{{ incpatch .Version }}-snapshot" + +changelog: + use: git + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..83d4a90 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2026 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5470b7c --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +BINARY := repo-sync +MODULE := github.com/grmrgecko/repo-sync/config +# VERSION is the single source of truth for the version string. COMMIT and DATE +# are derived from git and the build clock. +VERSION ?= $(shell cat VERSION 2>/dev/null || echo dev) +COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) +DATE := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ') +LDFLAGS := -s -w -X $(MODULE).Version=$(VERSION) -X $(MODULE).Commit=$(COMMIT) -X $(MODULE).Date=$(DATE) + +.PHONY: all build test vet fmt snapshot release clean + +all: build + +## build: native static binary into dist/ +build: + CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(BINARY) . + +## test: run the unit tests +test: + go test ./... + +vet: + go vet ./... + +fmt: + gofmt -w . + +## snapshot: local GoReleaser build without publishing (artifacts in dist/) +snapshot: + goreleaser release --snapshot --clean + +## release: full GoReleaser release (CI runs this on a tag) +release: + goreleaser release --clean + +clean: + rm -rf dist diff --git a/README.md b/README.md new file mode 100644 index 0000000..b3a2685 --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +# repo-sync + +A universal Linux package repository synchronization tool. It mirrors remote +repositories into a local directory tree, preserving the upstream layout so +the result can be served directly to package managers. + +Supported repository types: + +- **rpm** — yum/dnf repositories (`repodata/repomd.xml`), including + plain-text mirrorlist and metalink URLs with failover between mirrors. +- **deb** — apt repositories, both standard (`dists/` with a shared + `pool/`) and flat layouts, including `Acquire-By-Hash` population. +- **arch** — pacman repositories (`.db`), including detached package + signatures and companion metadata (`.files`, `.db.tar.gz`, `.links.tar.gz`). +- **apk** — Alpine Linux repositories (`APKINDEX.tar.gz` with packages + beside it). + +## Usage + +``` +repo-sync [flags] [ ...] +``` + +The last argument is always the destination directory; every argument before +it is a repository or mirrorlist URL, synchronized one after the other. + +A configuration file is optional: without one the built-in defaults apply. +When one is present its `crawler` section supplies the defaults for every +command, so `workers`, `prune_grace`, `request_timeout`, and `user_agent` +are shared by the sync commands and the server. The global flags +`--config-path`, `--log-level`, `--user-agent`, and `--request-timeout` +override the configuration for any command. + +```sh +# Mirror one repository. The URL path is copied below the destination, so +# this produces ./mirror/repos/CentOS/7/EA4/. +repo-sync rpm https://example.com/repos/CentOS/7/EA4/ ./mirror + +# Trim the first two path components: ./mirror/7/EA4/. +repo-sync rpm --trim 2 https://example.com/repos/CentOS/7/EA4/ ./mirror + +# No path copying at all: the repository lands directly in ./mirror. +repo-sync rpm --flat https://example.com/repos/CentOS/7/EA4/ ./mirror + +# Multiple repositories in one run. +repo-sync rpm https://example.com/repos/a/ https://example.com/repos/b/ ./mirror + +# Crawl directory listings for repositories, at most 4 levels deep. +repo-sync rpm --discover --depth 4 https://example.com/repos/ ./mirror + +# A pacman repository. The database name is discovered from the directory +# listing, or by probing URL path segments when listings are disabled. +repo-sync arch https://example.com/archlinux/core/os/x86_64/ ./mirror + +# An Alpine repository is one arch directory; discovery syncs every arch +# of a release/repo tree in one run. +repo-sync apk https://dl-cdn.alpinelinux.org/alpine/v3.24/community/x86_64/ ./mirror +repo-sync apk --discover --depth 1 https://dl-cdn.alpinelinux.org/alpine/v3.24/community/ ./mirror + +# A Fedora-style metalink URL; mirrors are used in preference order. The +# mirrors' paths differ, so --flat keeps the destination stable. +repo-sync rpm --flat 'https://mirrors.fedoraproject.org/metalink?repo=epel-9&arch=x86_64' ./mirror/epel9 + +# An apt suite. Pool files resolve against the archive root, producing +# ./mirror/debian/dists/bookworm/ and ./mirror/debian/pool/. +repo-sync deb https://deb.example.com/debian/dists/bookworm ./mirror + +# Limit an apt mirror to specific components and architectures. Include +# "source" as an architecture to keep source indexes. +repo-sync deb --component main --arch amd64,source https://deb.example.com/debian/dists/bookworm ./mirror +``` + +## Mirror server + +`repo-sync server` runs a caching mirror in front of upstream repositories. +Repositories are discovered from client requests and then kept synchronized +in the background, so package managers can be pointed straight at it. + +```sh +repo-sync server --config-path ./config.yaml +``` + +Without `--config-path` the config is read from `./config.yaml`, +`~/.config/repo-sync/config.yaml`, or `/etc/repo-sync/config.yaml`. See +[config.example.yaml](config.example.yaml) for a documented configuration +covering the listener, domains, mounts, and crawler tuning. The server +needs at least one domain and one mount; the sync commands do not. + +To run it as a systemd service, install `/etc/repo-sync/config.yaml` and: + +```sh +repo-sync service install +repo-sync service start +``` + +`service` also accepts `stop`, `restart`, `status`, and `uninstall`. The +installed unit runs `repo-sync server` as a notify service, restarts on +failure, and reloads its configuration on `systemctl reload repo-sync`. + +## Building + +```sh +make +``` + +The build stamps the binary with the contents of the `VERSION` file plus the +git commit and build date, shown by `repo-sync --version`. A plain +`go build` also works but reports the version as `dev`. + +## Testing + +```sh +make test +``` + +The suite is hermetic: fixture repositories for every supported format +(rpm, deb, arch, apk) are generated in temp directories and served over +local HTTP, so no network access is required. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..3c7361b --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,112 @@ +# Example repo-sync mirror server configuration. +# +# The server proxies and caches upstream package mirrors. Repositories are +# discovered from client requests: fetching a repository's entry-point +# metadata (repodata/repomd.xml, dists//InRelease, .db, +# APKINDEX.tar.gz) registers it for full synchronization and periodic +# refreshing. Repositories listed under a mount's `repos` are kept +# synchronized regardless of traffic. + +# state_path persists discovered repositories across restarts. When unset +# it defaults to state.yaml next to the config file (./config.yaml keeps +# state in ./state.yaml), or /etc/repo-sync/state.yaml without one. +state_path: /etc/repo-sync/state.yaml +state_flush_interval: 30s + +# repo_crawl_interval is how often known repositories are refreshed while +# they keep being requested. failure_retry_interval schedules the next +# attempt after a failed crawl. crawl_check_interval is the scheduler tick. +repo_crawl_interval: 4h +failure_retry_interval: 1h +crawl_check_interval: 30m + +# generic_max_age bounds how stale a plain cached file (anything outside a +# repository) may be before it is revalidated upstream. Upstream 404s are +# remembered for negative_cache_ttl and transient errors for +# transient_error_ttl so failures are not amplified per client. +generic_max_age: 6h +negative_cache_ttl: 1h +transient_error_ttl: 1m + +http: + bind_addr: "" + port: 8080 + read_header_timeout: 10s + read_timeout: 5m + write_timeout: 30m + idle_timeout: 2m + # directory_indexes enables directory listings: upstream index pages are + # passed through, and when no catch-all "/" mount exists the root serves + # a generated index of the mounts and their configured repositories. + # When false, directory requests answer with a static notice page. + directory_indexes: true + +log: + # Severity threshold: debug, info, warn, error. + level: info + # Formatter: console (text) or json. + type: console + # Outputs may include "console" (stderr) or explicit file paths. + outputs: + - console + +# domains map the request Host header to a filesystem tree. Exactly one +# domain has the online role: its tree is the writable cache that misses +# are fetched into and crawls write to. Offline domains are read-only +# published trees that fall back to the online tree for anything missing, +# and are promoted out-of-band (e.g. by snapshot tooling). +domains: + - domain: mirror.example.com + role: online + root: /var/lib/repo-sync/online + - domain: stage.mirror.example.com + role: offline + root: /var/lib/repo-sync/stage + +# mounts map base request paths onto upstream mirrors; the longest matching +# path wins, and "/" is allowed as a catch-all. A request for +# /almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml resolves through the +# /almalinux mount and registers that path as an RPM repository. Each mount +# may also pin repositories that stay synchronized without client traffic. +mounts: + - path: /almalinux + upstream: https://mirror.example.com/almalinux + repos: + - path: /almalinux/9/BaseOS/x86_64/os + type: rpm + - path: /centos + upstream: https://vault.centos.org + - path: /ubuntu + upstream: https://mirror.example.com/ubuntu + - path: /archlinux + upstream: https://mirror.example.com/archlinux + - path: /alpine + upstream: https://dl-cdn.alpinelinux.org/alpine + +# The crawler section also applies to the sync commands: they use these +# workers, prune_grace, request_timeout, and user_agent unless the +# equivalent flag overrides them. +crawler: + # workers is the number of download workers inside a single crawl; + # concurrent_crawls bounds how many repository crawls run at once. + workers: 4 + concurrent_crawls: 4 + # request_timeout bounds how long an upstream may take to start + # responding to a request (response header timeout). + request_timeout: 2m + # user_agent overrides the default repo-sync/. + user_agent: "" + # prune_grace keeps files that left a repository for this long before + # crawl-time pruning removes them. + prune_grace: 24h + # Backoff steps applied once a repository stops being requested. The + # first tier is repo_crawl_interval; each entry extends the refresh + # interval as the resource ages, and once the total budget elapses the + # resource is evicted from the online cache. + refresh_schedule: + - 12h + - 24h + - 48h + - 72h + - 96h + - 120h diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..86d0459 --- /dev/null +++ b/config/config.go @@ -0,0 +1,440 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "time" + + "github.com/go-playground/validator/v10" + "github.com/spf13/viper" +) + +// Domain roles: the online domain is the writable upstream cache, offline +// domains are read-only published trees that fall back to the online tree. +const ( + RoleOnline = "online" + RoleOffline = "offline" +) + +// HTTPConfig configures the mirror server listener. +type HTTPConfig struct { + BindAddr string `mapstructure:"bind_addr" yaml:"bind_addr"` + Port uint `mapstructure:"port" yaml:"port" validate:"gt=0,lte=65535"` + ReadHeaderTimeout time.Duration `mapstructure:"read_header_timeout" yaml:"read_header_timeout" validate:"gt=0"` + ReadTimeout time.Duration `mapstructure:"read_timeout" yaml:"read_timeout" validate:"gt=0"` + WriteTimeout time.Duration `mapstructure:"write_timeout" yaml:"write_timeout" validate:"gt=0"` + IdleTimeout time.Duration `mapstructure:"idle_timeout" yaml:"idle_timeout" validate:"gt=0"` + // DirectoryIndexes enables directory listings: upstream index pages are + // passed through, and the root serves a generated index of the mounts + // when no catch-all mount covers it. When disabled, every directory + // request answers with a static notice page instead. + DirectoryIndexes bool `mapstructure:"directory_indexes" yaml:"directory_indexes"` +} + +// ListenAddr renders the bind address and port for net.Listen, bracketing +// IPv6 addresses as required. +func (h HTTPConfig) ListenAddr() string { + return net.JoinHostPort(h.BindAddr, strconv.Itoa(int(h.Port))) +} + +// LogConfig configures the server's logging output. +type LogConfig struct { + Level string `mapstructure:"level" yaml:"level" validate:"oneof=debug info warn error"` + Type string `mapstructure:"type" yaml:"type" validate:"oneof=console json"` + Outputs []string `mapstructure:"outputs" yaml:"outputs" validate:"required,min=1,dive,required"` +} + +// DomainConfig maps a request Host header to a filesystem root and role. +type DomainConfig struct { + Domain string `mapstructure:"domain" yaml:"domain" validate:"required"` + Root string `mapstructure:"root" yaml:"root" validate:"required"` + Role string `mapstructure:"role" yaml:"role" validate:"required,oneof=online offline"` +} + +// MountRepoConfig is a repository kept synchronized regardless of client +// traffic, identified by its request path and format. +type MountRepoConfig struct { + Path string `mapstructure:"path" yaml:"path" validate:"required"` + Type string `mapstructure:"type" yaml:"type" validate:"required,oneof=rpm deb arch apk"` +} + +// MountConfig maps a base request path onto an upstream mirror URL. +type MountConfig struct { + Path string `mapstructure:"path" yaml:"path" validate:"required"` + Upstream string `mapstructure:"upstream" yaml:"upstream" validate:"required,url"` + Repos []MountRepoConfig `mapstructure:"repos" yaml:"repos" validate:"omitempty,dive"` +} + +// CrawlerConfig tunes the background synchronization behavior. +type CrawlerConfig struct { + Workers int `mapstructure:"workers" yaml:"workers" validate:"gt=0"` + ConcurrentCrawls int `mapstructure:"concurrent_crawls" yaml:"concurrent_crawls" validate:"gt=0"` + RequestTimeout time.Duration `mapstructure:"request_timeout" yaml:"request_timeout" validate:"gt=0"` + UserAgent string `mapstructure:"user_agent" yaml:"user_agent"` + PruneGrace time.Duration `mapstructure:"prune_grace" yaml:"prune_grace" validate:"gte=0"` + RefreshSchedule []time.Duration `mapstructure:"refresh_schedule" yaml:"refresh_schedule" validate:"required,min=1,dive,gt=0"` +} + +// Config is the mirror server configuration. +type Config struct { + HTTP HTTPConfig `mapstructure:"http" yaml:"http"` + Log LogConfig `mapstructure:"log" yaml:"log"` + StatePath string `mapstructure:"state_path" yaml:"state_path" validate:"required"` + StateFlushInterval time.Duration `mapstructure:"state_flush_interval" yaml:"state_flush_interval" validate:"gt=0"` + RepoCrawlInterval time.Duration `mapstructure:"repo_crawl_interval" yaml:"repo_crawl_interval" validate:"gt=0"` + FailureRetryInterval time.Duration `mapstructure:"failure_retry_interval" yaml:"failure_retry_interval" validate:"gt=0"` + CrawlCheckInterval time.Duration `mapstructure:"crawl_check_interval" yaml:"crawl_check_interval" validate:"gt=0"` + GenericMaxAge time.Duration `mapstructure:"generic_max_age" yaml:"generic_max_age" validate:"gt=0"` + NegativeCacheTTL time.Duration `mapstructure:"negative_cache_ttl" yaml:"negative_cache_ttl" validate:"gt=0"` + TransientErrorTTL time.Duration `mapstructure:"transient_error_ttl" yaml:"transient_error_ttl" validate:"gt=0"` + Domains []DomainConfig `mapstructure:"domains" yaml:"domains" validate:"omitempty,dive"` + Mounts []MountConfig `mapstructure:"mounts" yaml:"mounts" validate:"omitempty,dive"` + Crawler CrawlerConfig `mapstructure:"crawler" yaml:"crawler"` + + domainsByName map[string]int + onlineIndex int + mountOrder []int +} + +// C is the active server configuration. A reload builds and validates a +// replacement in full before repointing C at it, so readers always see a +// complete configuration; requests already in flight finish against the +// one they picked up. +var C *Config + +// C starts out holding the built-in defaults so it is never nil. Commands +// that run without a configuration file, and packages exercised by tests +// that never call Init, read those defaults instead of guarding for nil. +func init() { + C = defaults() +} + +// defaults builds a configuration from the registered defaults alone, +// consulting no file. Nothing here can fail: the values are static and the +// indexes start empty, so validation is left to Init. +func defaults() *Config { + v := viper.New() + setDefaults(v) + c := &Config{} + // Unmarshaling static defaults into their own struct cannot fail. + _ = v.Unmarshal(c) + c.StatePath = "/etc/repo-sync/state.yaml" + c.domainsByName = map[string]int{} + c.onlineIndex = -1 + return c +} + +// loadConfig reads and validates a configuration file. Init installs the +// result; callers work with the C singleton rather than loading their own. +func loadConfig(configPath string) (*Config, error) { + v := viper.New() + setDefaults(v) + + file := findConfig(configPath) + if file != "" { + v.SetConfigFile(file) + if err := v.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read config %s: %w", file, err) + } + } + + c := &Config{} + if err := v.Unmarshal(c); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + + // Default the state file next to the config file, so ./config.yaml + // keeps state in ./state.yaml, or to the system path without one. + if c.StatePath == "" { + if file != "" { + c.StatePath = filepath.Join(filepath.Dir(file), "state.yaml") + } else { + c.StatePath = "/etc/repo-sync/state.yaml" + } + } + + if err := c.finalize(); err != nil { + return nil, err + } + return c, nil +} + +// Init loads and validates the configuration, installs it as the active +// configuration, and applies its logging settings. A failed reload leaves +// the previous configuration in place. +func Init(configPath string) error { + c, err := loadConfig(configPath) + if err != nil { + return err + } + install(c) + return nil +} + +// InitServer is Init with the mirror server's additional requirements +// checked before the configuration is installed, so a bad reload leaves +// the running server on the configuration it is already serving. +func InitServer(configPath string) error { + c, err := loadConfig(configPath) + if err != nil { + return err + } + if err := c.RequireServer(); err != nil { + return err + } + install(c) + return nil +} + +// install publishes a configuration and applies the settings derived from +// it. +func install(c *Config) { + C = c + SetupLogging(c.Log) +} + +// findConfig resolves the configuration file path: the explicit flag, the +// working directory, the user config directory, then the system directory. +func findConfig(explicit string) string { + if explicit != "" { + return explicit + } + candidates := []string{"config.yaml"} + if home, err := os.UserHomeDir(); err == nil { + candidates = append(candidates, filepath.Join(home, ".config", "repo-sync", "config.yaml")) + } + candidates = append(candidates, "/etc/repo-sync/config.yaml") + for _, name := range candidates { + if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() { + return name + } + } + return "" +} + +// setDefaults registers every configuration default. +func setDefaults(v *viper.Viper) { + v.SetDefault("state_flush_interval", 30*time.Second) + v.SetDefault("repo_crawl_interval", 4*time.Hour) + v.SetDefault("failure_retry_interval", time.Hour) + v.SetDefault("crawl_check_interval", 30*time.Minute) + v.SetDefault("generic_max_age", 6*time.Hour) + v.SetDefault("negative_cache_ttl", time.Hour) + v.SetDefault("transient_error_ttl", time.Minute) + + v.SetDefault("http.port", 8080) + v.SetDefault("http.directory_indexes", true) + v.SetDefault("http.read_header_timeout", 10*time.Second) + v.SetDefault("http.read_timeout", 5*time.Minute) + v.SetDefault("http.write_timeout", 30*time.Minute) + v.SetDefault("http.idle_timeout", 2*time.Minute) + + v.SetDefault("log.level", "info") + v.SetDefault("log.type", "console") + v.SetDefault("log.outputs", []string{"console"}) + + v.SetDefault("crawler.workers", 4) + v.SetDefault("crawler.concurrent_crawls", 4) + v.SetDefault("crawler.request_timeout", 2*time.Minute) + v.SetDefault("crawler.user_agent", Name+"/"+Version) + v.SetDefault("crawler.prune_grace", 0) + v.SetDefault("crawler.refresh_schedule", []time.Duration{ + 12 * time.Hour, 24 * time.Hour, 48 * time.Hour, + 72 * time.Hour, 96 * time.Hour, 120 * time.Hour, + }) +} + +// finalize canonicalizes, validates, and indexes a parsed configuration. +func (c *Config) finalize() error { + c.canonicalize() + if err := validateConfig(c); err != nil { + return err + } + if err := c.indexDomains(); err != nil { + return err + } + return c.indexMounts() +} + +// canonicalize normalizes user-supplied values before validation. +func (c *Config) canonicalize() { + c.StatePath = expandHome(strings.TrimSpace(c.StatePath)) + for i := range c.Domains { + c.Domains[i].Domain = strings.ToLower(strings.TrimSpace(c.Domains[i].Domain)) + c.Domains[i].Role = strings.ToLower(strings.TrimSpace(c.Domains[i].Role)) + c.Domains[i].Root = expandHome(strings.TrimSpace(c.Domains[i].Root)) + } + for i := range c.Mounts { + c.Mounts[i].Path = cleanRequestPath(c.Mounts[i].Path) + c.Mounts[i].Upstream = strings.TrimRight(strings.TrimSpace(c.Mounts[i].Upstream), "/") + for j := range c.Mounts[i].Repos { + c.Mounts[i].Repos[j].Path = cleanRequestPath(c.Mounts[i].Repos[j].Path) + c.Mounts[i].Repos[j].Type = strings.ToLower(strings.TrimSpace(c.Mounts[i].Repos[j].Type)) + } + } +} + +// cleanRequestPath normalizes a request path to a single leading slash with +// no trailing slash. +func cleanRequestPath(p string) string { + return path.Clean("/" + strings.Trim(strings.TrimSpace(p), "/")) +} + +// expandHome resolves a leading ~ against the user's home directory. +func expandHome(p string) string { + if p == "~" || strings.HasPrefix(p, "~/") { + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, strings.TrimPrefix(p, "~")) + } + } + return p +} + +// validateConfig applies the declarative validation rules, reporting +// failures by their YAML key names. +func validateConfig(c *Config) error { + validate := validator.New() + validate.RegisterTagNameFunc(func(field reflect.StructField) string { + name := strings.SplitN(field.Tag.Get("mapstructure"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + err := validate.Struct(c) + if err == nil { + return nil + } + var verrs validator.ValidationErrors + if !errors.As(err, &verrs) { + return err + } + var fields []string + for _, verr := range verrs { + fields = append(fields, fmt.Sprintf("%s (%s)", strings.ToLower(verr.Namespace()), verr.Tag())) + } + return fmt.Errorf("invalid configuration: %s", strings.Join(fields, ", ")) +} + +// indexDomains builds the host lookup and enforces exactly one online +// domain. +func (c *Config) indexDomains() error { + c.domainsByName = map[string]int{} + c.onlineIndex = -1 + for i := range c.Domains { + d := c.Domains[i] + if _, ok := c.domainsByName[d.Domain]; ok { + return fmt.Errorf("domain %s is configured more than once", d.Domain) + } + c.domainsByName[d.Domain] = i + if d.Role == RoleOnline { + if c.onlineIndex != -1 { + return fmt.Errorf("only one online domain may be configured") + } + c.onlineIndex = i + } + } + return nil +} + +// RequireServer reports whether the configuration carries what the mirror +// server needs. Domains and mounts are only meaningful to the server, so +// they are checked here rather than at load; the sync commands run against +// a configuration that has neither. +func (c *Config) RequireServer() error { + if len(c.Domains) == 0 { + return errors.New("at least one domain must be configured") + } + if c.onlineIndex == -1 { + return errors.New("one online domain must be configured") + } + if len(c.Mounts) == 0 { + return errors.New("at least one mount must be configured") + } + return nil +} + +// indexMounts validates mount uniqueness and upstream URLs, orders mounts +// longest path first for prefix lookup, and checks configured repositories +// sit under their mount. +func (c *Config) indexMounts() error { + seen := map[string]bool{} + for i := range c.Mounts { + m := &c.Mounts[i] + if seen[m.Path] { + return fmt.Errorf("mount path %s is configured more than once", m.Path) + } + seen[m.Path] = true + u, err := url.Parse(m.Upstream) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("mount %s upstream must include scheme and host", m.Path) + } + // Request URLs are built by joining a relative path onto the + // upstream; a query or fragment would swallow that path, so reject + // them rather than compose a malformed URL later. + if u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("mount %s upstream must not contain a query or fragment", m.Path) + } + for _, repo := range m.Repos { + if !pathUnder(repo.Path, m.Path) { + return fmt.Errorf("repository %s is not under mount %s", repo.Path, m.Path) + } + } + } + c.mountOrder = make([]int, len(c.Mounts)) + for i := range c.mountOrder { + c.mountOrder[i] = i + } + sort.Slice(c.mountOrder, func(a, b int) bool { + return len(c.Mounts[c.mountOrder[a]].Path) > len(c.Mounts[c.mountOrder[b]].Path) + }) + return nil +} + +// pathUnder reports whether a request path equals or sits below a base +// path. +func pathUnder(p, base string) bool { + if base == "/" { + return true + } + return p == base || strings.HasPrefix(p, base+"/") +} + +// Domain looks up the configuration for a request Host header. The port is +// stripped and IPv6 literals are unbracketed before the lookup. +func (c *Config) Domain(host string) (DomainConfig, bool) { + host = strings.ToLower(strings.TrimSpace(host)) + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.Trim(host, "[]") + i, ok := c.domainsByName[host] + if !ok { + return DomainConfig{}, false + } + return c.Domains[i], true +} + +// OnlineDomain returns the single writable cache domain. +func (c *Config) OnlineDomain() DomainConfig { + return c.Domains[c.onlineIndex] +} + +// MountFor finds the longest mount matching a request path. +func (c *Config) MountFor(reqPath string) (MountConfig, bool) { + for _, i := range c.mountOrder { + if pathUnder(reqPath, c.Mounts[i].Path) { + return c.Mounts[i], true + } + } + return MountConfig{}, false +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..48bd831 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,142 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// writeConfig writes a temp config file and returns its path. +func writeConfig(t *testing.T, content string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(name, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return name +} + +// TestInitExample verifies the shipped example configuration is valid. +func TestInitExample(t *testing.T) { + if err := Init("../config.example.yaml"); err != nil { + t.Fatal(err) + } + if _, ok := C.Domain("mirror.example.com:443"); !ok { + t.Error("online domain not resolvable with a port") + } + if C.OnlineDomain().Domain != "mirror.example.com" { + t.Errorf("online domain = %q", C.OnlineDomain().Domain) + } + mount, ok := C.MountFor("/almalinux/9/BaseOS/x86_64/os") + if !ok || mount.Path != "/almalinux" { + t.Errorf("mount = %+v, %v", mount, ok) + } +} + +// TestStatePathDefault verifies an unset state path lands next to the +// config file. +func TestStatePathDefault(t *testing.T) { + path := writeConfig(t, ` +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://example.com"} +`) + if err := Init(path); err != nil { + t.Fatal(err) + } + want := filepath.Join(filepath.Dir(path), "state.yaml") + if C.StatePath != want { + t.Errorf("StatePath = %q, want %q", C.StatePath, want) + } +} + +// TestMountLongestPrefix verifies the most specific mount wins. +func TestMountLongestPrefix(t *testing.T) { + path := writeConfig(t, ` +state_path: /tmp/state.yaml +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://fallback.example.com"} + - {path: /centos, upstream: "https://centos.example.com"} + - {path: /centos/vault, upstream: "https://vault.example.com"} +`) + if err := Init(path); err != nil { + t.Fatal(err) + } + cases := map[string]string{ + "/centos/vault/7/os": "https://vault.example.com", + "/centos/8/BaseOS": "https://centos.example.com", + "/debian/dists/testy": "https://fallback.example.com", + } + for reqPath, want := range cases { + mount, ok := C.MountFor(reqPath) + if !ok || mount.Upstream != want { + t.Errorf("MountFor(%q) = %q, want %q", reqPath, mount.Upstream, want) + } + } +} + +// TestInitServerRejections verifies invalid server configurations fail +// with clear errors. +func TestInitServerRejections(t *testing.T) { + cases := map[string]string{ + "two online domains": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} + - {domain: b.test, role: online, root: /tmp/b} +mounts: + - {path: /, upstream: "https://example.com"} +`, + "no online domain": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: offline, root: /tmp/a} +mounts: + - {path: /, upstream: "https://example.com"} +`, + "repo outside mount": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} +mounts: + - path: /centos + upstream: "https://example.com" + repos: + - {path: /debian/dists/x, type: deb} +`, + "bad repo type": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} +mounts: + - path: /centos + upstream: "https://example.com" + repos: + - {path: /centos/7/os, type: gentoo} +`, + } + for name, content := range cases { + if err := InitServer(writeConfig(t, content)); err == nil { + t.Errorf("%s: configuration unexpectedly accepted", name) + } + } +} + +// TestInitWithoutServerSections verifies a configuration carrying no +// domains or mounts loads for the sync commands but is refused for the +// server. +func TestInitWithoutServerSections(t *testing.T) { + path := writeConfig(t, "state_path: /tmp/state.yaml\ncrawler:\n workers: 9\n") + if err := Init(path); err != nil { + t.Fatalf("Init rejected a sync-only configuration: %v", err) + } + if C.Crawler.Workers != 9 { + t.Errorf("Crawler.Workers = %d, want 9", C.Crawler.Workers) + } + if err := InitServer(path); err == nil { + t.Error("InitServer accepted a configuration with no domains") + } +} diff --git a/config/info.go b/config/info.go new file mode 100644 index 0000000..b1aed1c --- /dev/null +++ b/config/info.go @@ -0,0 +1,19 @@ +// Package config loads and validates the mirror server configuration, +// publishes it for reload-aware readers, configures logging, and carries +// the application identity and build identifiers. +package config + +// Build identifiers populated at build time via -ldflags. +var ( + Version = "dev" + Commit = "" + Date = "" + Mode = "dev" +) + +// Application identifiers used by the CLI and the upstream user agent. +const ( + Name = "repo-sync" + DisplayName = "Repository Sync" + Description = "Synchronize Linux package repositories into a local directory tree." +) diff --git a/config/log.go b/config/log.go new file mode 100644 index 0000000..008adfa --- /dev/null +++ b/config/log.go @@ -0,0 +1,56 @@ +package config + +import ( + "io" + "os" + + log "github.com/sirupsen/logrus" +) + +// logFiles holds the files opened by the previous SetupLogging call so a +// reload can close them once the new outputs are installed. +var logFiles []*os.File + +// SetupLogging applies the configured log level, formatter, and outputs to +// the global logger. Unknown output paths are treated as files opened for +// appending; failures fall back to stderr so the server always logs. +func SetupLogging(lc LogConfig) { + level, err := log.ParseLevel(lc.Level) + if err != nil { + level = log.InfoLevel + } + log.SetLevel(level) + + if lc.Type == "json" { + log.SetFormatter(&log.JSONFormatter{}) + } else { + log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + } + + var writers []io.Writer + var files []*os.File + for _, output := range lc.Outputs { + if output == "console" { + writers = append(writers, os.Stderr) + continue + } + f, err := os.OpenFile(expandHome(output), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + log.WithError(err).WithField("output", output).Warn("Unable to open log output; skipping.") + continue + } + writers = append(writers, f) + files = append(files, f) + } + if len(writers) == 0 { + writers = append(writers, os.Stderr) + } + log.SetOutput(io.MultiWriter(writers...)) + + // Close the previous reload's files only after the new writer is + // installed; the logger serializes writes, so none remain in flight. + for _, f := range logFiles { + f.Close() + } + logFiles = files +} diff --git a/fetch/fetch.go b/fetch/fetch.go new file mode 100644 index 0000000..37c37aa --- /dev/null +++ b/fetch/fetch.go @@ -0,0 +1,784 @@ +// Package fetch downloads repository files over HTTP with mirror failover, +// verifying published sizes and checksums, staging updates so a live tree +// is never served a partially updated repository, and pruning files the +// upstream no longer lists. +package fetch + +import ( + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "hash/fnv" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/klauspost/compress/zstd" + log "github.com/sirupsen/logrus" + "github.com/ulikunitz/xz" +) + +// client is the shared HTTP client used for all repository fetches. It is +// atomic because a SIGHUP reload swaps it while requests are in flight. +var client atomic.Pointer[http.Client] + +func init() { + // Build a client immediately so the package is usable without an + // explicit Reload. The config package is initialized first, so its + // defaults are already installed. + Reload() +} + +// Reload rebuilds the shared HTTP client from the active crawler +// configuration. A header timeout is used instead of a whole-request +// timeout so large package downloads are not cut short. +func Reload() { + old := client.Load() + client.Store(&http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: cfg.C.Crawler.RequestTimeout, + }, + }) + // Release the replaced transport's idle pool; in-flight requests on it + // finish undisturbed. + if old != nil { + old.CloseIdleConnections() + } +} + +// ErrNotFound marks an upstream file that does not exist, letting callers +// treat missing optional files differently from transport failures. +var ErrNotFound = errors.New("upstream file not found") + +// ErrNotModified marks a conditional request the upstream answered with +// 304, meaning the local copy is already current. +var ErrNotModified = errors.New("upstream file not modified") + +// StagedSuffix is appended to staged downloads until they are promoted. +const StagedSuffix = ".staged" + +// PartialSuffix is appended to in-progress downloads. Partials of files +// with published checksums survive failures so the next run can resume. +const PartialSuffix = ".partial" + +// LockFileName is the flock target guarding a destination directory. +const LockFileName = ".repo-sync.lock" + +// PruneStateName is the prune grace bookkeeping file kept at the root of a +// pruned tree. +const PruneStateName = ".repo-sync-prune.json" + +// Expect describes the size and checksums a fetched file must match. A size +// of -1 means the size is unknown, and Sums maps lowercase algorithm names +// to hex digests. +type Expect struct { + Size int64 + Sums map[string]string +} + +// MakeExpect builds an Expect from a size and a single checksum, treating +// non-positive sizes and empty checksums as unknown. +func MakeExpect(size int64, algo, digest string) *Expect { + e := &Expect{Size: size, Sums: map[string]string{}} + if size <= 0 { + e.Size = -1 + } + if algo != "" && digest != "" { + e.Sums[strings.ToLower(algo)] = strings.ToLower(digest) + } + return e +} + +// hashOrder lists checksum algorithms from most to least preferred. +var hashOrder = []string{"sha512", "sha384", "sha256", "sha224", "sha1", "sha", "md5", "md5sum"} + +// newHasher returns a hash implementation for a checksum algorithm name as +// found in repository metadata, or nil when the algorithm is unknown. +func newHasher(algo string) hash.Hash { + switch strings.ToLower(algo) { + case "md5", "md5sum": + return md5.New() + case "sha", "sha1": + return sha1.New() + case "sha224": + return sha256.New224() + case "sha256": + return sha256.New() + case "sha384": + return sha512.New384() + case "sha512": + return sha512.New() + } + return nil +} + +// bestSum picks the strongest checksum available in the expectation. +func (e *Expect) bestSum() (string, string) { + for _, algo := range hashOrder { + if digest, ok := e.Sums[algo]; ok && digest != "" { + return algo, digest + } + } + return "", "" +} + +// FileState reports what File did with a file. It is only meaningful when +// the accompanying error is nil. +type FileState int + +const ( + // FileMissing marks an optional file the upstream does not serve. + FileMissing FileState = iota + // FileUnchanged marks an existing local file that matched expectations. + FileUnchanged + // FileFetched marks a file downloaded into its final location. + FileFetched + // FileStaged marks a file downloaded next to its final location for a + // later promote. + FileStaged +) + +// verifyLocal reports whether an existing regular file already matches the +// expected size and, when checkHash is set, the expected checksum. +func verifyLocal(name string, want *Expect, checkHash bool) bool { + info, err := os.Stat(name) + if err != nil || !info.Mode().IsRegular() { + return false + } + if want.Size >= 0 && info.Size() != want.Size { + return false + } + if !checkHash { + return true + } + algo, digest := want.bestSum() + if digest == "" { + return true + } + sum, err := hashFile(name, algo) + if err != nil { + return false + } + return sum == digest +} + +// hashFile computes the named checksum of a file, returned as lowercase hex. +func hashFile(name, algo string) (string, error) { + h := newHasher(algo) + if h == nil { + return "", fmt.Errorf("unsupported checksum algorithm %q", algo) + } + f, err := os.Open(name) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// fileStripes bounds the destination lock table. Downloads to the same +// destination are serialized by stripe; a collision between unrelated +// destinations only delays a download briefly. +const fileStripes = 512 + +// fileLocks serializes downloads by destination path. The mirror server's +// on-demand fetches and background crawls can target the same file +// concurrently, and their partial downloads must not interleave. +var fileLocks [fileStripes]sync.Mutex + +// lockFile holds the lock for a destination path and returns its unlock +// function. +func lockFile(dst string) func() { + h := fnv.New32a() + _, _ = io.WriteString(h, dst) + m := &fileLocks[h.Sum32()%fileStripes] + m.Lock() + return m.Unlock +} + +// File downloads reqPath from the source into dst, verifying size and +// checksum when known. Existing files that already match are left alone. +// With stage set the download is written next to dst for a later promote so +// readers of a live tree never see a partially updated repository. +func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting bool) (FileState, error) { + // Serialize writers to this destination so concurrent callers, such + // as the mirror server's on-demand fetches and background crawls, + // cannot interleave partial downloads of the same file. + unlock := lockFile(dst) + defer unlock() + + // Clear any staged leftover from an interrupted run so a later promote + // can never publish stale content. + if stage { + _ = os.Remove(dst + StagedSuffix) + } + + // Skip files that already match the expected content. + if want != nil && verifyLocal(dst, want, verifyExisting) { + Stats.Unchanged.Add(1) + return FileUnchanged, nil + } + + // Checksummed files with a known size can resume a failed download's + // partial file with a ranged request. + var resumeFrom int64 + if want != nil && want.Size > 0 { + if _, digest := want.bestSum(); digest != "" { + if info, err := os.Stat(dst + PartialSuffix); err == nil && info.Size() > 0 && info.Size() < want.Size { + resumeFrom = info.Size() + } + } + } + + for { + state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom) + if err != nil && resumeFrom > 0 && !errors.Is(err, ErrNotFound) && ctx.Err() == nil { + // The partial may not be a prefix of the current upstream file; + // retry once from scratch. + log.WithError(err).WithField("path", reqPath).Debug("Resumed download failed; retrying in full.") + _ = os.Remove(dst + PartialSuffix) + resumeFrom = 0 + continue + } + return state, err + } +} + +// download performs one transfer attempt for File, appending to the +// partial file when resuming from a prior failure. +func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64) (FileState, error) { + // Files without published checksums cannot be verified locally, so an + // existing copy is revalidated with a conditional request instead of + // being re-downloaded every run. + var modifiedSince time.Time + if want == nil { + if info, err := os.Stat(dst); err == nil && info.Mode().IsRegular() { + modifiedSince = info.ModTime() + } + } + + // Request the file, failing over between mirrors as needed. + resp, err := src.Get(ctx, reqPath, GetOptions{ModifiedSince: modifiedSince, RangeFrom: resumeFrom}) + if errors.Is(err, ErrNotModified) { + Stats.Unchanged.Add(1) + log.WithField("path", reqPath).Debug("Kept unchanged repository file.") + return FileUnchanged, nil + } + if err != nil { + return FileMissing, err + } + defer resp.Body.Close() + + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return FileMissing, err + } + + var hasher hash.Hash + var algo, digest string + if want != nil { + algo, digest = want.bestSum() + if digest != "" { + hasher = newHasher(algo) + } + } + // Partials are only kept for files whose content the next attempt can + // verify by checksum. + keepPartial := digest != "" && want.Size > 0 + + // A 206 continues the partial file, whose existing bytes must feed the + // hash first; any other success replaces it from the start. + partial := dst + PartialSuffix + resumed := resumeFrom > 0 && resp.StatusCode == http.StatusPartialContent + openFlags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC + if resumed { + openFlags = os.O_CREATE | os.O_WRONLY | os.O_APPEND + if hasher != nil { + pf, err := os.Open(partial) + if err != nil { + return FileMissing, err + } + _, err = io.Copy(hasher, pf) + pf.Close() + if err != nil { + return FileMissing, err + } + } + log.WithFields(log.Fields{"path": reqPath, "offset": resumeFrom}).Debug("Resuming partial download.") + } + + // Stream the body into the partial file so a crash never leaves an + // incomplete file at the final location. + f, err := os.OpenFile(partial, openFlags, 0644) + if err != nil { + return FileMissing, err + } + w := io.Writer(f) + if hasher != nil { + w = io.MultiWriter(f, hasher) + } + written, copyErr := io.Copy(w, resp.Body) + closeErr := f.Close() + if copyErr != nil { + if !keepPartial { + _ = os.Remove(partial) + } + return FileMissing, fmt.Errorf("download %s: %w", reqPath, copyErr) + } + if closeErr != nil { + if !keepPartial { + _ = os.Remove(partial) + } + return FileMissing, closeErr + } + + // Verify the transfer against the published size and checksum. A + // mismatch invalidates the partial file entirely. + total := written + if resumed { + total += resumeFrom + } + if want != nil && want.Size >= 0 && total != want.Size { + _ = os.Remove(partial) + return FileMissing, fmt.Errorf("download %s: size %d does not match expected %d", reqPath, total, want.Size) + } + if hasher != nil { + if sum := hex.EncodeToString(hasher.Sum(nil)); sum != digest { + _ = os.Remove(partial) + return FileMissing, fmt.Errorf("download %s: %s checksum %s does not match expected %s", reqPath, algo, sum, digest) + } + } + + // Move the download into place, or next to it when staging. + final := dst + state := FileFetched + if stage { + final = dst + StagedSuffix + state = FileStaged + } + if err := os.Rename(partial, final); err != nil { + _ = os.Remove(partial) + return FileMissing, err + } + + // Carry the upstream modification time so later runs can revalidate + // with conditional requests. Promotion preserves it. + if lm, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil { + _ = os.Chtimes(final, time.Now(), lm) + } + Stats.Fetched.Add(1) + Stats.FetchedBytes.Add(written) + log.WithField("path", reqPath).Debug("Fetched repository file.") + return state, nil +} + +// PlanJobs records what a real run would download without transferring +// anything, still marking every destination as part of the repository so a +// dry-run prune report stays accurate. Optional files the upstream may not +// serve are counted, so the result is an upper bound. +func PlanJobs(jobs []Job, verifyExisting bool, keep *KeepSet) { + for _, job := range jobs { + keep.Add(job.Dst) + if job.Want != nil && verifyLocal(job.Dst, job.Want, verifyExisting) { + Stats.Unchanged.Add(1) + continue + } + Stats.Planned.Add(1) + if job.Want != nil && job.Want.Size > 0 { + Stats.PlannedBytes.Add(job.Want.Size) + } + log.WithField("path", job.ReqPath).Debug("Would fetch repository file.") + } +} + +// RemoveStale deletes a local file the upstream no longer serves, so an +// outdated copy is never served beside refreshed metadata. Missing files +// are not an error. +func RemoveStale(name string) { + if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", name).Warn("Failed to remove stale file.") + } +} + +// StagedOrFinal returns the staged copy of dst when one exists, falling +// back to the promoted file for runs where the upstream was unchanged. +func StagedOrFinal(dst string) string { + staged := dst + StagedSuffix + if _, err := os.Stat(staged); err == nil { + return staged + } + return dst +} + +// PromoteStaged moves a staged download into its final location. Missing +// staged files are not an error, as unchanged files are never staged. +func PromoteStaged(dst string) error { + staged := dst + StagedSuffix + if _, err := os.Stat(staged); errors.Is(err, fs.ErrNotExist) { + return nil + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + return os.Rename(staged, dst) +} + +// PromoteOrDiscard finishes a staged file: promoted normally, or removed in +// a dry run so no published repository state changes. +func PromoteOrDiscard(dst string, dryRun bool) error { + if dryRun { + _ = os.Remove(dst + StagedSuffix) + return nil + } + return PromoteStaged(dst) +} + +// Job describes one file for Many to download. +type Job struct { + ReqPath string + Dst string + Want *Expect + Optional bool + Stage bool +} + +// Many downloads jobs with a fixed worker pool, recording each job's +// resulting state. The first hard failure aborts the remaining work. +func Many(ctx context.Context, src *Source, jobs []Job, workers int, verifyExisting bool, keep *KeepSet) ([]FileState, error) { + states := make([]FileState, len(jobs)) + indexes := make(chan int) + errs := make(chan error, 1) + var wg sync.WaitGroup + + if workers < 1 { + workers = 1 + } + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for idx := range indexes { + job := jobs[idx] + state, err := File(ctx, src, job.ReqPath, job.Dst, job.Want, job.Stage, verifyExisting) + if errors.Is(err, ErrNotFound) && job.Optional { + log.WithField("path", job.ReqPath).Debug("Skipped file the upstream does not serve.") + continue + } + if err != nil { + select { + case errs <- err: + default: + } + return + } + states[idx] = state + keep.Add(job.Dst) + } + }() + } + + for idx := range jobs { + select { + case err := <-errs: + close(indexes) + wg.Wait() + return states, err + case <-ctx.Done(): + close(indexes) + wg.Wait() + return states, ctx.Err() + case indexes <- idx: + } + } + close(indexes) + wg.Wait() + + select { + case err := <-errs: + return states, err + default: + return states, nil + } +} + +// KeepSet tracks the files that belong to the current sync so pruning can +// remove everything else. A nil KeepSet ignores all operations. +type KeepSet struct { + mu sync.Mutex + paths map[string]bool +} + +// NewKeepSet returns a tracking set when pruning is enabled, or nil so all +// tracking becomes a no-op. +func NewKeepSet(enabled bool) *KeepSet { + if !enabled { + return nil + } + return &KeepSet{paths: map[string]bool{}} +} + +// Add records a local file as part of the repository. +func (k *KeepSet) Add(name string) { + if k == nil { + return + } + abs, err := filepath.Abs(name) + if err != nil { + return + } + k.mu.Lock() + k.paths[abs] = true + k.mu.Unlock() +} + +// has reports whether a local file is part of the repository. +func (k *KeepSet) has(name string) bool { + if k == nil { + return false + } + abs, err := filepath.Abs(name) + if err != nil { + return false + } + k.mu.Lock() + defer k.mu.Unlock() + return k.paths[abs] +} + +// loadPruneState reads the grace bookkeeping for a pruned tree, mapping +// root-relative paths to when they were first seen unreferenced. +func loadPruneState(root string) map[string]time.Time { + state := map[string]time.Time{} + data, err := os.ReadFile(filepath.Join(root, PruneStateName)) + if err != nil { + return state + } + if err := json.Unmarshal(data, &state); err != nil { + log.WithError(err).WithField("path", root).Warn("Failed to parse prune state; starting fresh.") + return map[string]time.Time{} + } + return state +} + +// savePruneState persists the grace bookkeeping, or removes it when no +// files are waiting out their grace period. +func savePruneState(root string, state map[string]time.Time) { + name := filepath.Join(root, PruneStateName) + if len(state) == 0 { + _ = os.Remove(name) + return + } + data, err := json.Marshal(state) + if err == nil { + err = os.WriteFile(name, data, 0644) + } + if err != nil { + log.WithError(err).WithField("path", name).Warn("Failed to save prune state.") + } +} + +// PruneTree removes files under root that are not part of the current sync, +// then clears out any directories left empty. A grace period defers +// deletion until a file has been unreferenced across runs for that long, +// so clients holding earlier metadata keep working. Failures are logged +// and do not fail the sync. +func PruneTree(root string, keep *KeepSet, grace time.Duration, dryRun bool) { + if keep == nil { + return + } + var state map[string]time.Time + if grace > 0 { + state = loadPruneState(root) + } + now := time.Now() + + var dirs []string + err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if name != root { + dirs = append(dirs, name) + } + return nil + } + base := filepath.Base(name) + if base == PruneStateName || base == LockFileName { + return nil + } + rel, relErr := filepath.Rel(root, name) + if relErr != nil { + return nil + } + if keep.has(name) { + delete(state, rel) + return nil + } + + // Internal work files are removed regardless of grace; anything + // else waits out the grace period across runs first. + internal := strings.HasSuffix(base, StagedSuffix) || strings.HasSuffix(base, PartialSuffix) + if grace > 0 && !internal { + first, ok := state[rel] + if !ok { + // A dry run records nothing, so report the pending grace + // entry instead of staying silent about the file. + if dryRun { + log.WithField("path", name).Info("Stale file would enter the prune grace period.") + return nil + } + state[rel] = now + log.WithField("path", name).Debug("Stale file entered the prune grace period.") + return nil + } + if now.Sub(first) < grace { + return nil + } + } + if dryRun { + Stats.Pruned.Add(1) + log.WithField("path", name).Info("Would prune stale file.") + return nil + } + if err := os.Remove(name); err != nil { + log.WithError(err).WithField("path", name).Warn("Failed to prune stale file.") + return nil + } + delete(state, rel) + Stats.Pruned.Add(1) + log.WithField("path", name).Debug("Pruned stale file.") + return nil + }) + if err != nil { + log.WithError(err).WithField("path", root).Warn("Failed to walk repository for pruning.") + } + + // Drop bookkeeping for files that no longer exist, then persist it. + if grace > 0 && !dryRun { + for rel := range state { + if _, err := os.Stat(filepath.Join(root, rel)); errors.Is(err, fs.ErrNotExist) { + delete(state, rel) + } + } + savePruneState(root, state) + } + if dryRun { + return + } + + // Remove now-empty directories deepest first. Separator count is the + // true depth, so a parent is always removed after its children even + // when a sibling's child has a longer name. + sep := string(os.PathSeparator) + sort.Slice(dirs, func(i, j int) bool { + return strings.Count(dirs[i], sep) > strings.Count(dirs[j], sep) + }) + for _, dir := range dirs { + _ = os.Remove(dir) + } +} + +// Decompressor wraps a reader with the decompressor matching the file name +// extension, returning an optional close function for the wrapper. Staged +// suffixes are ignored so staged indexes can be parsed before promotion. +func Decompressor(r io.Reader, filename string) (io.Reader, func(), error) { + filename = strings.TrimSuffix(filename, StagedSuffix) + switch { + case strings.HasSuffix(filename, ".gz"): + gz, err := gzip.NewReader(r) + if err != nil { + return nil, nil, err + } + return gz, func() { _ = gz.Close() }, nil + case strings.HasSuffix(filename, ".bz2"): + return bzip2.NewReader(r), nil, nil + case strings.HasSuffix(filename, ".xz"): + xr, err := xz.NewReader(r) + if err != nil { + return nil, nil, err + } + return xr, nil, nil + case strings.HasSuffix(filename, ".zst"): + zr, err := zstd.NewReader(r) + if err != nil { + return nil, nil, err + } + return zr, zr.Close, nil + default: + return r, nil, nil + } +} + +// SniffDecompressor wraps a reader with the decompressor matching its +// leading magic bytes, for formats whose file names carry no extension. +func SniffDecompressor(r io.Reader) (io.Reader, func(), error) { + br := bufio.NewReader(r) + magic, err := br.Peek(6) + if err != nil && !errors.Is(err, io.EOF) { + return nil, nil, err + } + switch { + case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: + gz, err := gzip.NewReader(br) + if err != nil { + return nil, nil, err + } + return gz, func() { _ = gz.Close() }, nil + case len(magic) >= 3 && magic[0] == 'B' && magic[1] == 'Z' && magic[2] == 'h': + return bzip2.NewReader(br), nil, nil + case len(magic) >= 6 && bytes.Equal(magic[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): + xr, err := xz.NewReader(br) + if err != nil { + return nil, nil, err + } + return xr, nil, nil + case len(magic) >= 4 && bytes.Equal(magic[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}): + zr, err := zstd.NewReader(br) + if err != nil { + return nil, nil, err + } + return zr, zr.Close, nil + default: + return br, nil, nil + } +} + +// LocalJoin resolves a repository-relative path under root, rejecting paths +// that would escape the destination tree. +func LocalJoin(root, rel string) (string, error) { + clean := path.Clean("/" + strings.ReplaceAll(rel, "\\", "/")) + dst := filepath.Join(root, filepath.FromSlash(clean)) + rootClean := filepath.Clean(root) + prefix := rootClean + string(os.PathSeparator) + if rootClean == string(os.PathSeparator) { + prefix = rootClean + } + if dst != rootClean && !strings.HasPrefix(dst, prefix) { + return "", fmt.Errorf("path %q escapes destination %q", rel, root) + } + return dst, nil +} diff --git a/fetch/fetch_test.go b/fetch/fetch_test.go new file mode 100644 index 0000000..b9b3f03 --- /dev/null +++ b/fetch/fetch_test.go @@ -0,0 +1,212 @@ +package fetch + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// resumeFixture serves a distinct-content file and records the Range +// headers of incoming requests. +func resumeFixture(t *testing.T, stripRange bool) (*Source, []byte, *[]string) { + t.Helper() + content := make([]byte, 3000) + for i := range content { + content[i] = byte(i % 251) + } + www := t.TempDir() + testrepos.WriteFile(t, filepath.Join(www, "big.bin"), content) + + var mu sync.Mutex + ranges := &[]string{} + fs := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + *ranges = append(*ranges, r.Header.Get("Range")) + mu.Unlock() + if stripRange { + r.Header.Del("Range") + } + fs.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return NewSource([]string{srv.URL}), content, ranges +} + +// TestFetchFileResume verifies a valid partial file is completed with a +// ranged request. +func TestFetchFileResume(t *testing.T) { + src, content, ranges := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, content[:1000]) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Error("resumed content mismatch") + } + if len(*ranges) != 1 || (*ranges)[0] != "bytes=1000-" { + t.Errorf("ranges = %v, want one bytes=1000- request", *ranges) + } +} + +// TestFetchFileResumeIgnored verifies a server that ignores ranges still +// produces a correct full download. +func TestFetchFileResumeIgnored(t *testing.T) { + src, content, _ := resumeFixture(t, true) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, content[:1000]) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, _ := os.ReadFile(dst) + if !bytes.Equal(got, content) { + t.Error("content mismatch after ignored range") + } +} + +// TestFetchFileResumeBadPartial verifies a partial that is not a prefix of +// the upstream file falls back to a full download. +func TestFetchFileResumeBadPartial(t *testing.T) { + src, content, ranges := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, bytes.Repeat([]byte("X"), 1000)) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, _ := os.ReadFile(dst) + if !bytes.Equal(got, content) { + t.Error("content mismatch after bad partial") + } + if len(*ranges) != 2 || (*ranges)[1] != "" { + t.Errorf("ranges = %v, want a ranged then a full request", *ranges) + } +} + +// TestFetchFileConditional verifies files without published checksums are +// revalidated with conditional requests: unchanged upstreams are kept, and +// changed upstreams are fetched again. +func TestFetchFileConditional(t *testing.T) { + www := t.TempDir() + upstream := filepath.Join(www, "meta.tar.gz") + testrepos.WriteFile(t, upstream, []byte("metadata")) + srv := testrepos.ServeDir(t, www) + src := NewSource([]string{srv.URL}) + + dst := filepath.Join(t.TempDir(), "meta.tar.gz") + state, err := File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileStaged { + t.Fatalf("first fetch = %v, %v", state, err) + } + if err := PromoteStaged(dst); err != nil { + t.Fatal(err) + } + + // An unchanged upstream answers the conditional request with 304. + state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileUnchanged { + t.Errorf("unchanged fetch = %v, %v", state, err) + } + + // A newer upstream file must be fetched again. The modification time + // is pushed forward because Last-Modified has one-second resolution. + testrepos.WriteFile(t, upstream, []byte("metadata-v2")) + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(upstream, future, future); err != nil { + t.Fatal(err) + } + state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileStaged { + t.Fatalf("changed fetch = %v, %v", state, err) + } + got, err := os.ReadFile(dst + StagedSuffix) + if err != nil { + t.Fatal(err) + } + if string(got) != "metadata-v2" { + t.Errorf("staged content = %q", got) + } +} + +// TestFetchFileConcurrent verifies concurrent downloads to the same +// destination are serialized: no caller's partial download can interleave +// with another's. +func TestFetchFileConcurrent(t *testing.T) { + src, content, _ := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + var wg sync.WaitGroup + errs := make(chan error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := File(context.Background(), src, "big.bin", dst, want, false, true); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Error("content mismatch after concurrent downloads") + } +} + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + Reload() + os.Exit(m.Run()) +} + +// TestLocalJoin verifies traversal attempts cannot escape the destination. +func TestLocalJoin(t *testing.T) { + root := t.TempDir() + got, err := LocalJoin(root, "../../etc/passwd") + if err != nil { + t.Fatal(err) + } + want := filepath.Join(root, "etc", "passwd") + if got != want { + t.Errorf("got %q, want %q", got, want) + } + if _, err := LocalJoin(root, "a/../b/c.rpm"); err != nil { + t.Errorf("clean relative path rejected: %v", err) + } + got, err = LocalJoin("/", "etc/passwd") + if err != nil { + t.Fatalf("filesystem root destination rejected: %v", err) + } + if got != "/etc/passwd" { + t.Errorf("got %q, want %q", got, "/etc/passwd") + } +} diff --git a/fetch/source.go b/fetch/source.go new file mode 100644 index 0000000..31793f2 --- /dev/null +++ b/fetch/source.go @@ -0,0 +1,248 @@ +package fetch + +import ( + "bufio" + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + log "github.com/sirupsen/logrus" +) + +// Source is the set of base URLs a repository can be fetched from. Direct +// repositories have a single base; mirrorlists provide several with +// failover between them. +type Source struct { + // mu guards bases and active. + mu sync.Mutex + bases []string + active int +} + +// NewSource builds a Source from base URLs, normalizing trailing slashes. +func NewSource(bases []string) *Source { + s := &Source{} + for _, base := range bases { + s.bases = append(s.bases, strings.TrimRight(base, "/")) + } + return s +} + +// Rebase trims a common relative suffix off every base URL so later fetches +// resolve against the parent directory. Bases without the suffix are kept +// unchanged. +func (s *Source) Rebase(suffix string) { + suffix = "/" + strings.Trim(suffix, "/") + s.mu.Lock() + defer s.mu.Unlock() + for i, base := range s.bases { + if strings.HasSuffix(base, suffix) { + s.bases[i] = strings.TrimSuffix(base, suffix) + } else { + log.WithField("mirror", base).Warn("Mirror URL does not share the expected path suffix; using it unchanged.") + } + } +} + +// GetOptions adjusts a single Source request. A non-zero ModifiedSince +// adds a conditional header, surfacing a 304 as ErrNotModified. A positive +// RangeFrom requests the remainder of a partially downloaded file. +type GetOptions struct { + ModifiedSince time.Time + RangeFrom int64 +} + +// Get requests a repository-relative path, failing over to the next mirror +// on transport or server errors. A 404 from the responding mirror is +// treated as authoritative so optional files do not trigger failover. +func (s *Source) Get(ctx context.Context, reqPath string, o GetOptions) (*http.Response, error) { + // Snapshot the bases so a concurrent Rebase cannot race the loop. + s.mu.Lock() + start := s.active + bases := append([]string(nil), s.bases...) + s.mu.Unlock() + if len(bases) == 0 { + return nil, errors.New("source has no mirror URLs") + } + + var lastErr error + for i := 0; i < len(bases); i++ { + idx := (start + i) % len(bases) + target := bases[idx] + "/" + strings.TrimPrefix(reqPath, "/") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + if !o.ModifiedSince.IsZero() { + req.Header.Set("If-Modified-Since", o.ModifiedSince.UTC().Format(http.TimeFormat)) + } + if o.RangeFrom > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", o.RangeFrom)) + } + resp, err := client.Load().Do(req) + if err != nil { + lastErr = fmt.Errorf("fetch %s: %w", target, err) + log.WithError(err).WithField("url", target).Debug("Mirror request failed.") + continue + } + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + resp.Body.Close() + return nil, fmt.Errorf("fetch %s: %w", target, ErrNotFound) + } + if resp.StatusCode == http.StatusNotModified && !o.ModifiedSince.IsZero() { + resp.Body.Close() + return nil, fmt.Errorf("fetch %s: %w", target, ErrNotModified) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + lastErr = fmt.Errorf("fetch %s: status %d", target, resp.StatusCode) + log.WithField("url", target).WithField("status", resp.StatusCode).Debug("Mirror request failed.") + continue + } + if idx != start { + s.mu.Lock() + s.active = idx + s.mu.Unlock() + } + return resp, nil + } + return nil, lastErr +} + +// ReadURL retrieves a raw URL fully into memory, limited to 4 MiB. It is +// used for probing mirrorlists, whose URLs may carry query strings that a +// Source cannot join paths onto. +func ReadURL(ctx context.Context, rawURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + resp, err := client.Load().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + return nil, fmt.Errorf("fetch %s: %w", rawURL, ErrNotFound) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("fetch %s: status %d", rawURL, resp.StatusCode) + } + // Read one byte past the cap so an oversized document can be detected + // and reported rather than silently truncated into a bad parse. + const maxSize = 4 << 20 + data, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxSize { + log.WithField("url", rawURL).Warn("Response exceeds the read limit and was truncated.") + data = data[:maxSize] + } + return data, nil +} + +// ParseMirrorlist extracts mirror base URLs from a mirrorlist body, +// skipping comments and lines that are not absolute HTTP URLs. +func ParseMirrorlist(body []byte) []string { + seen := map[string]bool{} + var bases []string + scanner := bufio.NewScanner(bytes.NewReader(body)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + u, err := url.Parse(line) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + continue + } + base := strings.TrimRight(line, "/") + if seen[base] { + continue + } + seen[base] = true + bases = append(bases, base) + } + return bases +} + +// ParseMetalink extracts mirror base URLs from a metalink document's +// repomd.xml entries, keeping the document's preference order. +func ParseMetalink(body []byte) []string { + const repomdSuffix = "/repodata/repomd.xml" + var ml struct { + Files []struct { + Name string `xml:"name,attr"` + URLs []struct { + Value string `xml:",chardata"` + } `xml:"resources>url"` + } `xml:"files>file"` + } + if xml.Unmarshal(body, &ml) != nil { + return nil + } + seen := map[string]bool{} + var bases []string + for _, file := range ml.Files { + if file.Name != "repomd.xml" { + continue + } + for _, u := range file.URLs { + v := strings.TrimSpace(u.Value) + if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") { + continue + } + base := strings.TrimSuffix(v, repomdSuffix) + if base == v { + continue + } + base = strings.TrimRight(base, "/") + if seen[base] { + continue + } + seen[base] = true + bases = append(bases, base) + } + } + return bases +} + +// agent returns the user agent sent with every upstream request, falling +// back to the build identity when the configuration clears it. +func agent() string { + if ua := cfg.C.Crawler.UserAgent; ua != "" { + return ua + } + return cfg.Name + "/" + cfg.Version +} + +// Get performs a plain GET of a raw URL with the mirror user agent, used +// for directory listings and other non-repository requests. +func Get(ctx context.Context, rawURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + return client.Load().Do(req) +} + +// Bases returns a copy of the mirror base URLs, primarily for tests and +// diagnostics. +func (s *Source) Bases() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.bases...) +} diff --git a/fetch/source_test.go b/fetch/source_test.go new file mode 100644 index 0000000..01ea993 --- /dev/null +++ b/fetch/source_test.go @@ -0,0 +1,97 @@ +package fetch + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseMirrorlist verifies URL extraction, comment skipping, and +// deduplication. +func TestParseMirrorlist(t *testing.T) { + body := "# comment\n\nhttp://mirror1.example.com/repo/\nhttps://mirror2.example.com/repo\nhttp://mirror1.example.com/repo\n/bare/path\nnot a url\n" + bases := ParseMirrorlist([]byte(body)) + want := []string{"http://mirror1.example.com/repo", "https://mirror2.example.com/repo"} + if len(bases) != len(want) { + t.Fatalf("got %v, want %v", bases, want) + } + for i := range want { + if bases[i] != want[i] { + t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i]) + } + } +} + +// TestSourceFailover verifies a dead first mirror fails over to a working +// one and that missing files surface the not-found sentinel. +func TestSourceFailover(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + src := NewSource([]string{"http://127.0.0.1:1/el9", srv.URL + "/el9"}) + resp, err := src.Get(context.Background(), "repodata/repomd.xml", GetOptions{}) + if err != nil { + t.Fatal("failover did not reach the working mirror:", err) + } + resp.Body.Close() + if src.active != 1 { + t.Errorf("active mirror = %d, want 1", src.active) + } + + if _, err := src.Get(context.Background(), "no/such/file", GetOptions{}); !errors.Is(err, ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } +} + +// TestParseMetalink verifies base URL extraction from metalink documents. +func TestParseMetalink(t *testing.T) { + body := `` + + `` + + `https://mirror1.example.com/fedora/repodata/repomd.xml` + + `rsync://mirror2.example.com/fedora/repodata/repomd.xml` + + `http://mirror3.example.com/fedora/repodata/repomd.xml` + + `https://mirror1.example.com/fedora/repodata/repomd.xml` + + `` + bases := ParseMetalink([]byte(body)) + want := []string{"https://mirror1.example.com/fedora", "http://mirror3.example.com/fedora"} + if len(bases) != len(want) { + t.Fatalf("got %v, want %v", bases, want) + } + for i := range want { + if bases[i] != want[i] { + t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i]) + } + } + if ParseMetalink([]byte("not xml at all")) != nil { + t.Error("non-XML input produced bases") + } +} + +// TestSourceNoBases verifies a Source with no mirrors reports an error and +// no response. +func TestSourceNoBases(t *testing.T) { + src := NewSource(nil) + resp, err := src.Get(context.Background(), "anything", GetOptions{}) + if err == nil { + t.Fatal("expected an error for a source without bases") + } + if resp != nil { + t.Error("response must be nil on error") + } +} + +// TestSourceRebase verifies suffix trimming for suite-relative sources. +func TestSourceRebase(t *testing.T) { + src := NewSource([]string{"http://example.com/debian/dists/test", "http://other.example.com/mirror"}) + src.Rebase("dists/test") + if src.bases[0] != "http://example.com/debian" { + t.Errorf("bases[0] = %q", src.bases[0]) + } + if src.bases[1] != "http://other.example.com/mirror" { + t.Errorf("bases[1] changed unexpectedly: %q", src.bases[1]) + } +} diff --git a/fetch/stats.go b/fetch/stats.go new file mode 100644 index 0000000..239856b --- /dev/null +++ b/fetch/stats.go @@ -0,0 +1,51 @@ +package fetch + +import ( + "fmt" + "sync/atomic" +) + +// counters accumulates transfer activity for one repository's summary log +// line. Counters are atomic because download workers run concurrently. +type counters struct { + Fetched atomic.Int64 + FetchedBytes atomic.Int64 + Unchanged atomic.Int64 + Pruned atomic.Int64 + Planned atomic.Int64 + PlannedBytes atomic.Int64 +} + +// Stats collects transfer activity for the repository currently +// synchronizing. The CLI synchronizes repositories sequentially, so one +// package-level collector suffices there. The server's concurrent crawls +// interleave into these counters, so their values are only meaningful in +// the sequential path; the server must not report them. +var Stats = &counters{} + +// Reset clears the collector for the next repository. +func (s *counters) Reset() { + s.Fetched.Store(0) + s.FetchedBytes.Store(0) + s.Unchanged.Store(0) + s.Pruned.Store(0) + s.Planned.Store(0) + s.PlannedBytes.Store(0) +} + +// FormatBytes renders a byte count in a human readable unit. +func FormatBytes(n int64) string { + const unit = 1024 + const prefixes = "KMGTPE" + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + // Cap the exponent at the last available prefix so an unexpectedly + // large count can never index past the prefix table. + for m := n / unit; m >= unit && exp < len(prefixes)-1; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), prefixes[exp]) +} diff --git a/flags.go b/flags.go new file mode 100644 index 0000000..739d20d --- /dev/null +++ b/flags.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "runtime/debug" + "strings" + "time" + + "github.com/alecthomas/kong" + cfg "github.com/grmrgecko/repo-sync/config" +) + +// VersionFlag prints build information and exits. +type VersionFlag bool + +// Decode satisfies kong.MapperValue. The flag is treated as a boolean toggle. +func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil } + +// IsBool reports the flag as a boolean for kong's parser. +func (v VersionFlag) IsBool() bool { return true } + +// BeforeApply emits version information then exits before the rest of the +// command is executed. +func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error { + fmt.Printf("%s: %s (%s)\n", cfg.Name, cfg.Version, cfg.Mode) + if cfg.Commit != "" { + fmt.Printf(" commit: %s\n", cfg.Commit) + } + if cfg.Date != "" { + fmt.Printf(" built: %s\n", cfg.Date) + } + if cfg.Commit == "" { + if bi, ok := debug.ReadBuildInfo(); ok { + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + fmt.Printf(" commit: %s\n", s.Value) + case "vcs.time": + fmt.Printf(" built: %s\n", s.Value) + } + } + } + } + app.Exit(0) + return nil +} + +// Flags is the root command line configuration. The global flags override +// the configuration file for every command, so a sync run and the server +// honor the same settings; each is empty when unset so the configured +// value stands. +type Flags struct { + ConfigPath string `help:"The path to the config file." optional:"" type:"existingfile"` + LogLevel string `help:"Override the configured log level." enum:",debug,info,warn,error" default:""` + UserAgent string `help:"Override the configured user agent sent to upstreams." optional:""` + RequestTimeout time.Duration `help:"Override the configured upstream response timeout (e.g. 2m)." optional:""` + Version VersionFlag `name:"version" help:"Print version information and quit"` + + RPM RPMCmd `cmd:"" name:"rpm" help:"Synchronize RPM (yum/dnf) repositories."` + Deb DebCmd `cmd:"" name:"deb" help:"Synchronize DEB (apt) repositories."` + Arch ArchCmd `cmd:"" name:"arch" help:"Synchronize Arch Linux (pacman) repositories."` + Apk ApkCmd `cmd:"" name:"apk" help:"Synchronize Alpine Linux (apk) repositories."` + Server ServerCmd `cmd:"" name:"server" help:"Run the caching mirror server."` + Service ServiceCmd `cmd:"" name:"service" help:"Manage the mirror server system service."` +} + +// flags holds the parsed command line configuration. +var flags Flags + +// ParseFlags parses the command line, exiting with usage information on +// error. +func ParseFlags() *kong.Context { + return kong.Parse(&flags, + kong.Name(cfg.Name), + kong.Description(cfg.Description), + kong.UsageOnError(), + kong.ConfigureHelp(kong.HelpOptions{ + Compact: true, + }), + kong.Vars{ + "serviceActions": strings.Join(ServiceAction, ","), + }, + ) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5f9996a --- /dev/null +++ b/go.mod @@ -0,0 +1,36 @@ +module github.com/grmrgecko/repo-sync + +go 1.26.4 + +require ( + github.com/alecthomas/kong v1.16.0 + github.com/coreos/go-systemd/v22 v22.7.0 + github.com/go-playground/validator/v10 v10.30.3 + github.com/kardianos/service v1.3.0 + github.com/klauspost/compress v1.19.1 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/viper v1.21.0 + github.com/ulikunitz/xz v0.5.16 + golang.org/x/net v0.57.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..890d18f --- /dev/null +++ b/go.sum @@ -0,0 +1,81 @@ +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4= +github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= +github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI= +github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= +github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/testrepos/testrepos.go b/internal/testrepos/testrepos.go new file mode 100644 index 0000000..7a9630f --- /dev/null +++ b/internal/testrepos/testrepos.go @@ -0,0 +1,258 @@ +// Package testrepos builds small fixture repositories for tests across +// the fetch, mirror, and server packages. +package testrepos + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// WriteFile creates a file with parent directories under a fixture tree. +func WriteFile(t *testing.T, name string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, data, 0644); err != nil { + t.Fatal(err) + } +} + +// GzipBytes compresses data with gzip for index fixtures. +func GzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := gz.Write(data); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// SHA256Hex returns the hex SHA-256 of data. +func SHA256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// MD5Hex returns the hex MD5 of data. +func MD5Hex(data []byte) string { + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]) +} + +// ServeDir serves a fixture tree over HTTP with directory listings, closing +// the server when the test finishes. +func ServeDir(t *testing.T, dir string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.FileServer(http.Dir(dir))) + t.Cleanup(srv.Close) + return srv +} + +// BuildRPMRepo writes a minimal yum repository into dir and returns the +// package contents by file name. +func BuildRPMRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "foo-1.0-1.x86_64.rpm": bytes.Repeat([]byte("foo"), 500), + "bar-2.0-1.x86_64.rpm": bytes.Repeat([]byte("bar"), 700), + } + var pkgXML bytes.Buffer + for name, data := range pkgs { + WriteFile(t, filepath.Join(dir, "Packages", name), data) + fmt.Fprintf(&pkgXML, + `%s%s`, + name, SHA256Hex(data), len(data), name) + } + primary := []byte(`` + pkgXML.String() + ``) + primaryGz := GzipBytes(t, primary) + primaryName := SHA256Hex(primaryGz) + "-primary.xml.gz" + WriteFile(t, filepath.Join(dir, "repodata", primaryName), primaryGz) + + // A prestodelta index references one delta package under drpms/. + drpm := RPMDelta() + WriteFile(t, filepath.Join(dir, "drpms", "foo-0.9_1.0-1.x86_64.drpm"), drpm) + presto := []byte(fmt.Sprintf( + `drpms/foo-0.9_1.0-1.x86_64.drpm%d%s`, + len(drpm), SHA256Hex(drpm))) + prestoGz := GzipBytes(t, presto) + prestoName := SHA256Hex(prestoGz) + "-prestodelta.xml.gz" + WriteFile(t, filepath.Join(dir, "repodata", prestoName), prestoGz) + + repomd := fmt.Sprintf( + `%s%d%s%d`, + SHA256Hex(primaryGz), primaryName, len(primaryGz), + SHA256Hex(prestoGz), prestoName, len(prestoGz)) + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml"), []byte(repomd)) + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.asc"), []byte("fake signature")) + return pkgs +} + +// RPMDelta is the delta package content used by BuildRPMRepo. +func RPMDelta() []byte { + return bytes.Repeat([]byte("drpm"), 100) +} + +// debPackagesStanza renders one binary package stanza for a Packages index. +func debPackagesStanza(name, version, poolPath string, data []byte) string { + return fmt.Sprintf( + "Package: %s\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSize: %d\nMD5sum: %s\nSHA256: %s\nDescription: Test package\n", + name, version, poolPath, len(data), MD5Hex(data), SHA256Hex(data)) +} + +// BuildDebRepo writes a minimal structured apt repository with main and +// universe components into dir and returns the pool file contents by +// archive-relative path. +func BuildDebRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pool := map[string][]byte{ + "pool/main/h/hello/hello_1.0_amd64.deb": bytes.Repeat([]byte("hello"), 300), + "pool/universe/e/extra/extra_2.0_amd64.deb": bytes.Repeat([]byte("extra"), 400), + } + for p, data := range pool { + WriteFile(t, filepath.Join(dir, filepath.FromSlash(p)), data) + } + + indexes := map[string][]byte{ + "main/binary-amd64/Packages": []byte(debPackagesStanza( + "hello", "1.0", "pool/main/h/hello/hello_1.0_amd64.deb", pool["pool/main/h/hello/hello_1.0_amd64.deb"])), + "universe/binary-amd64/Packages": []byte(debPackagesStanza( + "extra", "2.0", "pool/universe/e/extra/extra_2.0_amd64.deb", pool["pool/universe/e/extra/extra_2.0_amd64.deb"])), + } + + // The Release file lists both the uncompressed and gzip variants, but + // only the gzip variant is served, matching real archives. + release := "Origin: Test\nSuite: test\nCodename: test\nArchitectures: amd64\nComponents: main universe\nAcquire-By-Hash: yes\n" + listed := map[string][]byte{} + for p, data := range indexes { + gz := GzipBytes(t, data) + WriteFile(t, filepath.Join(dir, "dists", "test", filepath.FromSlash(p)+".gz"), gz) + listed[p] = data + listed[p+".gz"] = gz + } + for field, hasher := range map[string]func([]byte) string{"MD5Sum": MD5Hex, "SHA256": SHA256Hex} { + release += field + ":\n" + for p, data := range listed { + release += fmt.Sprintf(" %s %d %s\n", hasher(data), len(data), p) + } + } + WriteFile(t, filepath.Join(dir, "dists", "test", "Release"), []byte(release)) + return pool +} + +// BuildArchRepo writes a minimal pacman repository named name into dir and +// returns the package contents by file name. +func BuildArchRepo(t *testing.T, dir, name string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "zlib-1.3-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("zlib"), 300), + "jq-1.7-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("jq"), 400), + } + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for fname, data := range pkgs { + WriteFile(t, filepath.Join(dir, fname), data) + WriteFile(t, filepath.Join(dir, fname+".sig"), []byte("signature")) + desc := fmt.Sprintf("%%FILENAME%%\n%s\n\n%%NAME%%\n%s\n\n%%CSIZE%%\n%d\n\n%%SHA256SUM%%\n%s\n", + fname, strings.SplitN(fname, "-", 2)[0], len(data), SHA256Hex(data)) + hdr := &tar.Header{ + Name: strings.TrimSuffix(fname, "-x86_64.pkg.tar.zst") + "/desc", + Mode: 0644, + Size: int64(len(desc)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(desc)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + WriteFile(t, filepath.Join(dir, name+".db"), buf.Bytes()) + WriteFile(t, filepath.Join(dir, name+".files"), buf.Bytes()) + return pkgs +} + +// BuildApkRepo writes a minimal Alpine repository into dir and returns the +// package contents by file name. The index replicates apk's real layout: a +// signature tar segment with its end-of-archive blocks cut off and the +// index tar as separate gzip streams, concatenated. +func BuildApkRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "musl-1.2.5-r1.apk": bytes.Repeat([]byte("musl"), 300), + "zlib-1.3-r2.apk": bytes.Repeat([]byte("zlib"), 200), + } + index := "" + for fname, data := range pkgs { + name := strings.SplitN(fname, "-", 2)[0] + version := strings.TrimSuffix(strings.TrimPrefix(fname, name+"-"), ".apk") + index += fmt.Sprintf("C:Q1notarealpullchecksum=\nP:%s\nV:%s\nA:x86_64\nS:%d\n\n", name, version, len(data)) + WriteFile(t, filepath.Join(dir, fname), data) + } + + // tarBytes renders entries into an uncompressed tar archive. + tarBytes := func(entries map[string]string) []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for name, content := range entries { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() + } + + // The signature segment loses its two zero end-of-archive blocks so + // the tar stream continues into the index segment. + sig := tarBytes(map[string]string{".SIGN.RSA.test.rsa.pub": "signature"}) + sig = sig[:len(sig)-1024] + idx := tarBytes(map[string]string{"DESCRIPTION": "test repo", "APKINDEX": index}) + combined := append(GzipBytes(t, sig), GzipBytes(t, idx)...) + WriteFile(t, filepath.Join(dir, "APKINDEX.tar.gz"), combined) + return pkgs +} + +// BuildFlatDebRepo writes a minimal flat apt repository into dir and +// returns the package contents by file name. +func BuildFlatDebRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + data := bytes.Repeat([]byte("flat"), 250) + WriteFile(t, filepath.Join(dir, "hello_3.0_amd64.deb"), data) + packages := []byte(debPackagesStanza("hello", "3.0", "hello_3.0_amd64.deb", data)) + packagesGz := GzipBytes(t, packages) + WriteFile(t, filepath.Join(dir, "Packages.gz"), packagesGz) + release := "Architectures: amd64\nSHA256:\n" + for p, d := range map[string][]byte{"Packages": packages, "Packages.gz": packagesGz} { + release += fmt.Sprintf(" %s %d %s\n", SHA256Hex(d), len(d), p) + } + WriteFile(t, filepath.Join(dir, "Release"), []byte(release)) + return map[string][]byte{"hello_3.0_amd64.deb": data} +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..b072272 --- /dev/null +++ b/main.go @@ -0,0 +1,36 @@ +package main + +import ( + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// main parses the command line, installs the configuration every command +// shares, and dispatches to the selected command. +func main() { + ctx := ParseFlags() + if err := cfg.Init(flags.ConfigPath); err != nil { + log.Fatal(err) + } + applyGlobalFlags() + err := ctx.Run() + ctx.FatalIfErrorf(err) +} + +// applyGlobalFlags overrides the loaded configuration with the global +// command line flags, then reapplies the settings derived from it. It also +// runs after a SIGHUP reload so the flags keep winning over the file. +func applyGlobalFlags() { + if flags.LogLevel != "" { + cfg.C.Log.Level = flags.LogLevel + } + if flags.UserAgent != "" { + cfg.C.Crawler.UserAgent = flags.UserAgent + } + if flags.RequestTimeout > 0 { + cfg.C.Crawler.RequestTimeout = flags.RequestTimeout + } + cfg.SetupLogging(cfg.C.Log) + fetch.Reload() +} diff --git a/mirror/apk.go b/mirror/apk.go new file mode 100644 index 0000000..4680189 --- /dev/null +++ b/mirror/apk.go @@ -0,0 +1,169 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// APKIndexName is the fixed name of an Alpine repository index. +const APKIndexName = "APKINDEX.tar.gz" + +// apkPackage is one package entry parsed from an APKINDEX. Alpine's C: +// pull checksum only covers a package's control segment, so mirroring can +// verify downloads by size alone. +type apkPackage struct { + name string + version string + size int64 +} + +// filename returns the package file name as served beside the index. +func (p *apkPackage) filename() string { + return p.name + "-" + p.version + ".apk" +} + +// expect converts the package's metadata into a download expectation. +func (p *apkPackage) expect() *fetch.Expect { + e := &fetch.Expect{Size: p.size} + if p.size <= 0 { + e.Size = -1 + } + return e +} + +// syncApk synchronizes one Alpine repository from src into destDir. +func syncApk(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + + // Stage the index so a live tree keeps a consistent view until the + // packages it references are in place. + idxDst, err := fetch.LocalJoin(destDir, APKIndexName) + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, APKIndexName, idxDst, nil, true, false); err != nil { + return fmt.Errorf("fetch %s: %w", APKIndexName, err) + } + pkgs, err := readApkIndex(fetch.StagedOrFinal(idxDst)) + if err != nil { + return err + } + + // Download the packages, which live beside the index. + var jobs []fetch.Job + for _, pkg := range pkgs { + dst, err := fetch.LocalJoin(destDir, pkg.filename()) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename(), Dst: dst, Want: pkg.expect()}) + } + log.WithField("packages", len(pkgs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // Promote the index last so the published metadata chain is complete. + if err := fetch.PromoteOrDiscard(idxDst, opts.DryRun); err != nil { + return err + } + keep.Add(idxDst) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + return nil +} + +// readApkIndex parses an APKINDEX.tar.gz archive. The file is a signature +// segment and an index segment as concatenated gzip streams, which the +// multistream gzip reader and tar reader walk through transparently. +func readApkIndex(filename string) ([]apkPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + if hdr.Name != "APKINDEX" { + continue + } + pkgs, err := parseApkIndex(tr) + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + return pkgs, nil + } + return nil, fmt.Errorf("no APKINDEX entry in %s", filename) +} + +// parseApkIndex reads APKINDEX stanzas, collecting each package's name, +// version, and file size from the P:, V:, and S: fields. +func parseApkIndex(r io.Reader) ([]apkPackage, error) { + var pkgs []apkPackage + var cur apkPackage + + // flush records the current stanza when it is complete. + flush := func() { + if cur.name != "" && cur.version != "" { + pkgs = append(pkgs, cur) + } + cur = apkPackage{} + } + + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 256*1024), 256*1024) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + flush() + continue + } + field, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + switch field { + case "P": + cur.name = value + case "V": + cur.version = value + case "S": + cur.size, _ = strconv.ParseInt(value, 10, 64) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + flush() + return pkgs, nil +} diff --git a/mirror/apk_test.go b/mirror/apk_test.go new file mode 100644 index 0000000..d1cc51e --- /dev/null +++ b/mirror/apk_test.go @@ -0,0 +1,100 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseApkIndex verifies stanza splitting and field extraction. +func TestParseApkIndex(t *testing.T) { + index := "C:Q1abc=\nP:musl\nV:1.2.5-r1\nA:x86_64\nS:1200\n\nC:Q1def=\nP:zlib\nV:1.3-r2\nS:800\n" + pkgs, err := parseApkIndex(strings.NewReader(index)) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Fatalf("got %d packages, want 2", len(pkgs)) + } + if pkgs[0].filename() != "musl-1.2.5-r1.apk" || pkgs[0].size != 1200 { + t.Errorf("first package = %+v", pkgs[0]) + } + if pkgs[1].filename() != "zlib-1.3-r2.apk" || pkgs[1].size != 800 { + t.Errorf("second package = %+v", pkgs[1]) + } +} + +// TestReadApkIndex verifies parsing through the signature segment of a +// multistream index archive. +func TestReadApkIndex(t *testing.T) { + dir := t.TempDir() + testrepos.BuildApkRepo(t, dir) + pkgs, err := readApkIndex(filepath.Join(dir, APKIndexName)) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Errorf("got %d packages, want 2", len(pkgs)) + } +} + +// TestSyncApk verifies a full synchronization mirrors packages and the +// index with no staged leftovers. +func TestSyncApk(t *testing.T) { + www := t.TempDir() + pkgs := testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/alpine/v3.24/community/x86_64", opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "alpine", "v3.24", "community", "x86_64") + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(local, APKIndexName)); err != nil { + t.Error("index not promoted:", err) + } + staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncApkPrune verifies stale packages are removed when pruning. +func TestSyncApkPrune(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "repo", "old-1.0-r0.apk") + if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale package survived pruning") + } +} diff --git a/mirror/arch.go b/mirror/arch.go new file mode 100644 index 0000000..59ce1e3 --- /dev/null +++ b/mirror/arch.go @@ -0,0 +1,258 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path" + "sort" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// archPackage is one package entry parsed from a pacman database. +type archPackage struct { + filename string + size int64 + sums map[string]string +} + +// expect converts the package's metadata into a download expectation. +func (p *archPackage) expect() *fetch.Expect { + e := &fetch.Expect{Size: p.size, Sums: p.sums} + if p.size <= 0 { + e.Size = -1 + } + return e +} + +// archExtras lists the companion metadata files mirrored beside the +// database, as suffixes appended to the repository name. All are optional; +// which ones exist varies by repository. +var archExtras = []string{ + ".files", + ".db.tar.gz", + ".files.tar.gz", + ".links.tar.gz", + ".db.sig", + ".db.tar.gz.sig", + ".files.sig", + ".files.tar.gz.sig", + ".links.tar.gz.sig", +} + +// syncArch synchronizes one pacman repository from src into destDir. +func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + + // Determine the database name, which matches the repository rather + // than any fixed path. + name, err := archDBName(ctx, src, repoURL) + if err != nil { + return err + } + log.WithField("database", name+".db").Debug("Resolved pacman database.") + + // Stage the database so a live tree keeps a consistent view until the + // packages it references are in place. + dbDst, err := fetch.LocalJoin(destDir, name+".db") + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, name+".db", dbDst, nil, true, false); err != nil { + return fmt.Errorf("fetch %s.db: %w", name, err) + } + pkgs, err := readArchDB(fetch.StagedOrFinal(dbDst)) + if err != nil { + return err + } + + // Download packages with their detached signatures. Signatures have no + // published checksums, so an existing file is trusted as-is. + var jobs []fetch.Job + for _, pkg := range pkgs { + dst, err := fetch.LocalJoin(destDir, pkg.filename) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect()}) + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: &fetch.Expect{Size: -1}, Optional: true}) + } + log.WithField("packages", len(pkgs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // Stage the companion metadata files that exist upstream. A dry run + // only counts them, as none are needed for planning. + var extraJobs []fetch.Job + for _, suffix := range archExtras { + dst, err := fetch.LocalJoin(destDir, name+suffix) + if err != nil { + return err + } + extraJobs = append(extraJobs, fetch.Job{ReqPath: name + suffix, Dst: dst, Optional: true, Stage: true}) + } + if opts.DryRun { + fetch.PlanJobs(extraJobs, opts.Verify, keep) + } else { + states, err := fetch.Many(ctx, src, extraJobs, opts.Workers, opts.Verify, keep) + if err != nil { + return fmt.Errorf("fetch repository metadata: %w", err) + } + for i, job := range extraJobs { + if states[i] == fetch.FileMissing { + // The upstream dropped the file; drop the local copy so a + // stale companion or signature is never served. + fetch.RemoveStale(job.Dst) + continue + } + if err := fetch.PromoteStaged(job.Dst); err != nil { + return err + } + } + } + + // Promote the database last so the published metadata chain is + // complete. + if err := fetch.PromoteOrDiscard(dbDst, opts.DryRun); err != nil { + return err + } + keep.Add(dbDst) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + return nil +} + +// archDBName finds the pacman database name for a repository, preferring +// the directory listing and falling back to probing names derived from the +// URL path, deepest segment first. +func archDBName(ctx context.Context, src *fetch.Source, repoURL string) (string, error) { + if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil { + var names []string + for f := range listing.files { + if strings.HasSuffix(f, ".db") { + names = append(names, strings.TrimSuffix(f, ".db")) + } + } + sort.Strings(names) + if len(names) > 1 { + log.WithField("databases", names).Warn("Multiple pacman databases found; using the first.") + } + if len(names) > 0 { + return names[0], nil + } + } + + // Listings can be disabled; a repository at .../core/os/x86_64 is + // probed as x86_64.db, os.db, then core.db. + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + var segs []string + for _, seg := range strings.Split(u.Path, "/") { + if seg != "" { + segs = append(segs, seg) + } + } + for i := len(segs) - 1; i >= 0; i-- { + resp, err := src.Get(ctx, segs[i]+".db", fetch.GetOptions{}) + if err == nil { + resp.Body.Close() + return segs[i], nil + } + if !errors.Is(err, fetch.ErrNotFound) { + return "", err + } + } + return "", fmt.Errorf("no pacman database found at %s", repoURL) +} + +// readArchDB parses a pacman database archive and collects each package's +// file name, size, and checksums from its desc entry. +func readArchDB(filename string) ([]archPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + // The database carries no compression extension, so the format is + // sniffed from its leading bytes. + r, closeFn, err := fetch.SniffDecompressor(f) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + tr := tar.NewReader(r) + var pkgs []archPackage + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse pacman database %s: %w", filename, err) + } + if hdr.FileInfo().IsDir() || path.Base(hdr.Name) != "desc" { + continue + } + data, err := io.ReadAll(io.LimitReader(tr, 1<<20)) + if err != nil { + return nil, fmt.Errorf("read desc entry %s: %w", hdr.Name, err) + } + fields := parseDesc(data) + if fields["FILENAME"] == "" { + continue + } + size, _ := strconv.ParseInt(fields["CSIZE"], 10, 64) + sums := map[string]string{} + if v := fields["SHA256SUM"]; v != "" { + sums["sha256"] = strings.ToLower(v) + } + if v := fields["MD5SUM"]; v != "" { + sums["md5"] = strings.ToLower(v) + } + pkgs = append(pkgs, archPackage{filename: fields["FILENAME"], size: size, sums: sums}) + } + return pkgs, nil +} + +// parseDesc extracts the first value of each %FIELD% block from a pacman +// desc entry. +func parseDesc(data []byte) map[string]string { + fields := map[string]string{} + field := "" + scanner := bufio.NewScanner(strings.NewReader(string(data))) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + switch { + case line == "": + field = "" + case strings.HasPrefix(line, "%") && strings.HasSuffix(line, "%"): + field = strings.Trim(line, "%") + case field != "": + if _, ok := fields[field]; !ok { + fields[field] = line + } + } + } + return fields +} diff --git a/mirror/arch_test.go b/mirror/arch_test.go new file mode 100644 index 0000000..6b12994 --- /dev/null +++ b/mirror/arch_test.go @@ -0,0 +1,166 @@ +package mirror + +import ( + "archive/tar" + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseDesc verifies field extraction from a pacman desc entry. +func TestParseDesc(t *testing.T) { + desc := "%FILENAME%\nzlib-1.3-1-x86_64.pkg.tar.zst\n\n%CSIZE%\n1200\n\n%SHA256SUM%\nabc123\n\n%DESC%\nFirst line\nSecond line\n" + fields := parseDesc([]byte(desc)) + if fields["FILENAME"] != "zlib-1.3-1-x86_64.pkg.tar.zst" { + t.Errorf("FILENAME = %q", fields["FILENAME"]) + } + if fields["CSIZE"] != "1200" { + t.Errorf("CSIZE = %q", fields["CSIZE"]) + } + if fields["SHA256SUM"] != "abc123" { + t.Errorf("SHA256SUM = %q", fields["SHA256SUM"]) + } + if fields["DESC"] != "First line" { + t.Errorf("DESC = %q, want first value only", fields["DESC"]) + } +} + +// TestReadArchDB verifies database parsing for both compressed and plain +// tar archives. +func TestReadArchDB(t *testing.T) { + dir := t.TempDir() + testrepos.BuildArchRepo(t, dir, "core") + + pkgs, err := readArchDB(filepath.Join(dir, "core.db")) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Fatalf("got %d packages, want 2", len(pkgs)) + } + for _, pkg := range pkgs { + if pkg.filename == "" || pkg.size <= 0 || pkg.sums["sha256"] == "" { + t.Errorf("incomplete package entry: %+v", pkg) + } + } + + // A plain uncompressed tar database must parse through the sniffer. + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + desc := "%FILENAME%\nplain-1.0-1-any.pkg.tar.zst\n\n%CSIZE%\n10\n\n%SHA256SUM%\nabc\n" + if err := tw.WriteHeader(&tar.Header{Name: "plain-1.0-1/desc", Mode: 0644, Size: int64(len(desc))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(desc)); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + plain := filepath.Join(dir, "plain.db") + if err := os.WriteFile(plain, buf.Bytes(), 0644); err != nil { + t.Fatal(err) + } + pkgs, err = readArchDB(plain) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 1 || pkgs[0].filename != "plain-1.0-1-any.pkg.tar.zst" { + t.Errorf("plain tar parse = %+v", pkgs) + } +} + +// TestSyncArch verifies a full synchronization mirrors packages, +// signatures, and metadata with no staged leftovers. +func TestSyncArch(t *testing.T) { + www := t.TempDir() + pkgs := testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core") + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoArch, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/core/os/x86_64", opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "core", "os", "x86_64") + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + if _, err := os.Stat(filepath.Join(local, name+".sig")); err != nil { + t.Errorf("signature for %s missing: %v", name, err) + } + } + for _, name := range []string{"core.db", "core.files"} { + if _, err := os.Stat(filepath.Join(local, name)); err != nil { + t.Errorf("%s not promoted: %v", name, err) + } + } + staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncArchPrune verifies stale files are removed when pruning. +func TestSyncArchPrune(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "core"), "core") + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoArch, Destination: dest, Workers: 2, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/core", opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "core", "old-0.9-1-x86_64.pkg.tar.zst") + if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/core", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale package survived pruning") + } +} + +// TestArchDBNameFallback verifies the database name is probed from URL +// path segments when directory listings are unavailable. +func TestArchDBNameFallback(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "custom", "os", "x86_64"), "custom") + + // Serve files but refuse directory listings to force the fallback. + fs := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + http.NotFound(w, r) + return + } + fs.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + src := fetch.NewSource([]string{srv.URL + "/custom/os/x86_64"}) + name, err := archDBName(context.Background(), src, srv.URL+"/custom/os/x86_64") + if err != nil { + t.Fatal(err) + } + if name != "custom" { + t.Errorf("name = %q, want custom", name) + } +} diff --git a/mirror/control.go b/mirror/control.go new file mode 100644 index 0000000..d4416b2 --- /dev/null +++ b/mirror/control.go @@ -0,0 +1,100 @@ +package mirror + +import ( + "bufio" + "io" + "strings" +) + +// stanza is one RFC 822 style field block from an apt control file. +type stanza map[string]string + +// Get returns the first matching field, trying each name in order to cope +// with casing differences between repositories. +func (s stanza) Get(names ...string) string { + for _, name := range names { + if v, ok := s[name]; ok { + return v + } + } + return "" +} + +// controlReader reads stanzas from an apt control file. +type controlReader struct { + br *bufio.Reader +} + +// newControlReader wraps a reader for stanza-by-stanza parsing. +func newControlReader(r io.Reader) *controlReader { + return &controlReader{br: bufio.NewReaderSize(r, 64*1024)} +} + +// readStanza parses the next stanza, returning io.EOF when the input is +// exhausted. Continuation lines are joined with newlines so multi-line +// checksum fields keep one entry per line. +func (r *controlReader) readStanza() (stanza, error) { + st := stanza{} + var last string + for { + line, err := r.br.ReadString('\n') + if err != nil && err != io.EOF { + return nil, err + } + done := err == io.EOF && line == "" + line = strings.TrimRight(line, "\r\n") + switch { + case done || strings.TrimSpace(line) == "": + if len(st) > 0 { + return st, nil + } + if done { + return nil, io.EOF + } + case line[0] == ' ' || line[0] == '\t': + if last != "" { + st[last] += "\n" + strings.TrimSpace(line) + } + default: + name, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + last = name + st[name] = strings.TrimSpace(value) + } + } +} + +// stripClearsign extracts the signed payload from a clearsigned InRelease +// document, returning the input unchanged when it is not clearsigned. +func stripClearsign(data []byte) []byte { + const begin = "-----BEGIN PGP SIGNED MESSAGE-----" + const sig = "-----BEGIN PGP SIGNATURE-----" + text := strings.ReplaceAll(string(data), "\r\n", "\n") + idx := strings.Index(text, begin) + if idx < 0 { + return data + } + text = text[idx+len(begin):] + + // Skip the armor headers up to the first blank line. Without one the + // document is malformed, so treat it as not clearsigned rather than + // returning the armor headers as payload. + i := strings.Index(text, "\n\n") + if i < 0 { + return data + } + text = text[i+2:] + if i := strings.Index(text, sig); i >= 0 { + text = text[:i] + } + + // Reverse the dash escaping applied by the signer. + var b strings.Builder + for _, line := range strings.Split(text, "\n") { + b.WriteString(strings.TrimPrefix(line, "- ")) + b.WriteString("\n") + } + return []byte(b.String()) +} diff --git a/mirror/control_test.go b/mirror/control_test.go new file mode 100644 index 0000000..0b847d1 --- /dev/null +++ b/mirror/control_test.go @@ -0,0 +1,70 @@ +package mirror + +import ( + "errors" + "io" + "strings" + "testing" +) + +// TestReadStanza verifies stanza splitting, continuation joining, and EOF +// handling without a trailing newline. +func TestReadStanza(t *testing.T) { + input := "Package: hello\nSHA256:\n abc 10 path/one\n def 20 path/two\n\nPackage: world\nSize: 42" + cr := newControlReader(strings.NewReader(input)) + + st, err := cr.readStanza() + if err != nil { + t.Fatal(err) + } + if st.Get("Package") != "hello" { + t.Errorf("Package = %q, want hello", st.Get("Package")) + } + if want := "\nabc 10 path/one\ndef 20 path/two"; st.Get("SHA256") != want { + t.Errorf("SHA256 = %q, want %q", st.Get("SHA256"), want) + } + + st, err = cr.readStanza() + if err != nil { + t.Fatal(err) + } + if st.Get("Package") != "world" || st.Get("Size") != "42" { + t.Errorf("second stanza = %v", st) + } + + if _, err = cr.readStanza(); !errors.Is(err, io.EOF) { + t.Errorf("expected EOF, got %v", err) + } +} + +// TestStanzaGet verifies fallback field names cover casing differences. +func TestStanzaGet(t *testing.T) { + st := stanza{"MD5Sum": "abc"} + if got := st.Get("MD5sum", "MD5Sum"); got != "abc" { + t.Errorf("got %q, want abc", got) + } + if got := st.Get("Missing"); got != "" { + t.Errorf("got %q, want empty", got) + } +} + +// TestStripClearsign verifies payload extraction and dash unescaping from a +// clearsigned document, and passthrough of unsigned input. +func TestStripClearsign(t *testing.T) { + signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nOrigin: Test\n- Dashed: line\n-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n" + got := string(stripClearsign([]byte(signed))) + if !strings.Contains(got, "Origin: Test\n") { + t.Errorf("payload missing Origin: %q", got) + } + if !strings.Contains(got, "Dashed: line") || strings.Contains(got, "- Dashed") { + t.Errorf("dash escaping not reversed: %q", got) + } + if strings.Contains(got, "PGP") { + t.Errorf("armor not removed: %q", got) + } + + plain := "Origin: Test\n" + if string(stripClearsign([]byte(plain))) != plain { + t.Error("unsigned input was modified") + } +} diff --git a/mirror/deb.go b/mirror/deb.go new file mode 100644 index 0000000..effa101 --- /dev/null +++ b/mirror/deb.go @@ -0,0 +1,526 @@ +package mirror + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// debFile is one file referenced by repository metadata with its expected +// size and checksums, keyed by lowercase algorithm name. +type debFile struct { + path string + size int64 + sums map[string]string +} + +// expect converts the file's metadata into a download expectation. +func (f *debFile) expect() *fetch.Expect { + e := &fetch.Expect{Size: f.size, Sums: f.sums} + if f.size <= 0 { + e.Size = -1 + } + return e +} + +// debRelease is the parsed subset of an apt Release file needed to mirror +// a suite. +type debRelease struct { + components []string + architectures []string + acquireByHash bool + files map[string]*debFile +} + +// releaseSumFields maps Release checksum block names to algorithm names, +// including the MD5sum casing some repositories use. +var releaseSumFields = map[string]string{ + "MD5Sum": "md5sum", + "MD5sum": "md5sum", + "SHA1": "sha1", + "SHA256": "sha256", + "SHA512": "sha512", +} + +// byHashDirs maps checksum algorithm names to the by-hash directory names +// apt uses. +var byHashDirs = map[string]string{ + "md5sum": "MD5Sum", + "sha1": "SHA1", + "sha256": "SHA256", + "sha512": "SHA512", +} + +// parseRelease extracts the fields the sync needs from a Release or +// InRelease document. +func parseRelease(data []byte) (*debRelease, error) { + st, err := newControlReader(bytes.NewReader(stripClearsign(data))).readStanza() + if err != nil { + return nil, fmt.Errorf("parse release file: %w", err) + } + rel := &debRelease{files: map[string]*debFile{}} + rel.components = strings.Fields(st.Get("Components")) + rel.architectures = strings.Fields(st.Get("Architectures")) + rel.acquireByHash = strings.EqualFold(st.Get("Acquire-By-Hash"), "yes") + + // Merge each checksum block into one record per file path. + for field, algo := range releaseSumFields { + for _, line := range strings.Split(st.Get(field), "\n") { + parts := strings.Fields(line) + if len(parts) != 3 { + continue + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + continue + } + f := rel.files[parts[2]] + if f == nil { + f = &debFile{path: parts[2], sums: map[string]string{}} + rel.files[parts[2]] = f + } + f.size = size + f.sums[algo] = strings.ToLower(parts[0]) + } + } + return rel, nil +} + +// syncDeb synchronizes one apt suite or flat repository from repoURL. +func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + + // Split the URL into the archive root and the suite path. Repositories + // without a dists directory are flat, with everything relative to the + // repository URL itself. + u, err := url.Parse(repoURL) + if err != nil { + return err + } + suiteRel := "" + rootURL := repoURL + if idx := strings.Index(u.Path, "/dists/"); idx >= 0 { + suiteRel = strings.Trim(u.Path[idx+1:], "/") + root := *u + root.Path = u.Path[:idx] + rootURL = root.String() + src.Rebase(suiteRel) + } + destRoot, err := opts.repoDest(rootURL) + if err != nil { + return err + } + destSuite := destRoot + suitePrefix := "" + if suiteRel != "" { + if destSuite, err = fetch.LocalJoin(destRoot, suiteRel); err != nil { + return err + } + suitePrefix = suiteRel + "/" + } + + // Stage the release files; they are promoted last so a live tree keeps + // a consistent metadata chain. + releaseNames := []string{"InRelease", "Release", "Release.gpg"} + releaseStates := make([]fetch.FileState, len(releaseNames)) + for i, name := range releaseNames { + dst, err := fetch.LocalJoin(destSuite, name) + if err != nil { + return err + } + state, err := fetch.File(ctx, src, suitePrefix+name, dst, nil, true, false) + if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + // The upstream dropped this release file; drop the local + // copy so outdated metadata is never served. + if !opts.DryRun { + fetch.RemoveStale(dst) + } + continue + } + return fmt.Errorf("fetch %s: %w", name, err) + } + releaseStates[i] = state + } + + // Parse the strongest release document available, which may be the + // promoted copy when the upstream reported it unchanged. + var relData []byte + switch { + case releaseStates[0] != fetch.FileMissing: + relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "InRelease"))) + case releaseStates[1] != fetch.FileMissing: + relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "Release"))) + default: + return errors.New("repository serves neither InRelease nor Release") + } + if err != nil { + return err + } + rel, err := parseRelease(relData) + if err != nil { + return err + } + + // Stage every index file the release lists, applying component and + // architecture filters. Missing variants are normal; repositories list + // checksums for files they do not serve. A dry run only fetches the + // package and source indexes needed to plan the pool. + var indexJobs, plannedJobs []fetch.Job + for _, f := range sortedFiles(rel.files) { + // Releases commonly list their own release files; those are + // already staged above and must not be fetched twice. + if slices.Contains(releaseNames, f.path) { + continue + } + if !includeIndexFile(f.path, opts.Components, opts.Architectures) { + continue + } + dst, err := fetch.LocalJoin(destSuite, f.path) + if err != nil { + return err + } + job := fetch.Job{ + ReqPath: suitePrefix + f.path, + Dst: dst, + Want: f.expect(), + Optional: true, + Stage: true, + } + if opts.DryRun { + base, _ := splitCompressExt(f.path) + if name := path.Base(base); name != "Packages" && name != "Sources" { + plannedJobs = append(plannedJobs, job) + continue + } + } + indexJobs = append(indexJobs, job) + } + fetch.PlanJobs(plannedJobs, opts.Verify, keep) + states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, opts.Verify, keep) + if err != nil { + return fmt.Errorf("fetch suite indexes: %w", err) + } + + // Pick one local compression variant of each package and source index + // to parse for pool file references. + variants := map[string]string{} + variantExt := map[string]string{} + prefOrder := map[string]int{".gz": 0, ".xz": 1, ".zst": 2, ".bz2": 3, "": 4} + for i, job := range indexJobs { + if states[i] == fetch.FileMissing { + continue + } + relPath := strings.TrimPrefix(job.ReqPath, suitePrefix) + base, ext := splitCompressExt(relPath) + if name := path.Base(base); name != "Packages" && name != "Sources" { + continue + } + local := job.Dst + if states[i] == fetch.FileStaged { + local = job.Dst + fetch.StagedSuffix + } + if cur, ok := variantExt[base]; !ok || prefOrder[ext] < prefOrder[cur] { + variantExt[base] = ext + variants[base] = local + } + } + + // Collect the pool files referenced by the chosen indexes. + pool := map[string]*debFile{} + for _, base := range sortedKeys(variants) { + files, err := parseIndexFile(variants[base], path.Base(base)) + if err != nil { + return fmt.Errorf("parse %s: %w", base, err) + } + for _, f := range files { + if _, ok := pool[f.path]; !ok { + pool[f.path] = f + } + } + } + + // Download pool files relative to the archive root. + poolJobs := make([]fetch.Job, 0, len(pool)) + for _, f := range sortedFiles(pool) { + dst, err := fetch.LocalJoin(destRoot, f.path) + if err != nil { + return err + } + poolJobs = append(poolJobs, fetch.Job{ReqPath: f.path, Dst: dst, Want: f.expect()}) + } + log.WithField("files", len(poolJobs)).Info("Synchronizing pool files.") + if opts.DryRun { + fetch.PlanJobs(poolJobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, opts.Verify, keep); err != nil { + return fmt.Errorf("fetch pool files: %w", err) + } + + // Promote the staged indexes now the pool files they reference exist. + for i, job := range indexJobs { + if states[i] == fetch.FileMissing { + continue + } + if err := fetch.PromoteOrDiscard(job.Dst, opts.DryRun); err != nil { + return err + } + if rel.acquireByHash && !opts.DryRun { + addByHash(job.Dst, strings.TrimPrefix(job.ReqPath, suitePrefix), rel, keep) + } + } + + // Promote the release files last to complete the metadata chain. + for _, name := range []string{"Release.gpg", "Release", "InRelease"} { + dst, err := fetch.LocalJoin(destSuite, name) + if err != nil { + return err + } + if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { + return err + } + if _, err := os.Stat(dst); err == nil { + keep.Add(dst) + } + } + + // Prune only the suite directory for structured repositories; the pool + // can be shared with other suites of the same archive. + if opts.Prune { + if suiteRel == "" { + fetch.PruneTree(destRoot, keep, opts.PruneGrace, opts.DryRun) + } else { + log.Info("Pruning is limited to the suite directory; shared pool files are kept.") + fetch.PruneTree(destSuite, keep, opts.PruneGrace, opts.DryRun) + } + } + return nil +} + +// addByHash links an index file into its directory's by-hash tree for each +// checksum the release lists, matching apt's Acquire-By-Hash layout. +// Failures are logged and do not fail the sync. +func addByHash(dst, relPath string, rel *debRelease, keep *fetch.KeepSet) { + f := rel.files[relPath] + if f == nil || !strings.Contains(relPath, "/") { + return + } + dir := filepath.Dir(dst) + for algo, digest := range f.sums { + sub, ok := byHashDirs[algo] + if !ok { + continue + } + link := filepath.Join(dir, "by-hash", sub, digest) + if err := os.MkdirAll(filepath.Dir(link), 0755); err != nil { + log.WithError(err).WithField("path", link).Warn("Failed to create by-hash directory.") + return + } + if _, err := os.Stat(link); err == nil { + keep.Add(link) + continue + } + if err := os.Link(dst, link); err != nil { + // Fall back to copying when hard links are not supported. + if err := copyFile(dst, link); err != nil { + log.WithError(err).WithField("path", link).Warn("Failed to create by-hash entry.") + continue + } + } + keep.Add(link) + } +} + +// copyFile duplicates src to dst. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + _ = os.Remove(dst) + return err + } + return out.Close() +} + +// parseIndexFile reads a Packages or Sources index and returns the pool +// files it references, with paths relative to the archive root. +func parseIndexFile(filename, kind string) ([]*debFile, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + cr := newControlReader(r) + var out []*debFile + for { + st, err := cr.readStanza() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + if kind == "Sources" { + out = append(out, sourceFiles(st)...) + continue + } + name := st.Get("Filename") + if name == "" { + continue + } + size, _ := strconv.ParseInt(st.Get("Size"), 10, 64) + sums := map[string]string{} + if v := st.Get("MD5sum", "MD5Sum"); v != "" { + sums["md5sum"] = strings.ToLower(v) + } + for _, algo := range []string{"SHA1", "SHA256", "SHA512"} { + if v := st.Get(algo); v != "" { + sums[strings.ToLower(algo)] = strings.ToLower(v) + } + } + out = append(out, &debFile{path: path.Clean(name), size: size, sums: sums}) + } + return out, nil +} + +// sourceSumFields maps source stanza checksum list fields to algorithm +// names; Files is the historical MD5 list. +var sourceSumFields = map[string]string{ + "Files": "md5sum", + "Checksums-Sha1": "sha1", + "Checksums-Sha256": "sha256", + "Checksums-Sha512": "sha512", +} + +// sourceFiles extracts the files of one source package stanza, merging the +// per-algorithm checksum lists by file name. +func sourceFiles(st stanza) []*debFile { + dir := st.Get("Directory") + byName := map[string]*debFile{} + var order []string + for field, algo := range sourceSumFields { + for _, line := range strings.Split(st.Get(field), "\n") { + parts := strings.Fields(line) + if len(parts) != 3 { + continue + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + continue + } + name := parts[2] + f := byName[name] + if f == nil { + f = &debFile{path: path.Join(dir, name), sums: map[string]string{}} + byName[name] = f + order = append(order, name) + } + f.size = size + f.sums[algo] = strings.ToLower(parts[0]) + } + } + out := make([]*debFile, 0, len(byName)) + for _, name := range order { + out = append(out, byName[name]) + } + return out +} + +// includeIndexFile applies the component and architecture filters to a +// release-listed file path. +func includeIndexFile(p string, comps, archs []string) bool { + if len(comps) > 0 { + if segs := strings.Split(p, "/"); len(segs) > 1 && !slices.Contains(comps, segs[0]) { + return false + } + } + if len(archs) > 0 { + if arch := indexArch(p); arch != "" && !slices.Contains(archs, arch) { + return false + } + } + return true +} + +// indexArch extracts the architecture a release-listed file applies to, or +// an empty string when the file is architecture independent. +func indexArch(p string) string { + for _, seg := range strings.Split(p, "/") { + if arch, ok := strings.CutPrefix(seg, "binary-"); ok { + return arch + } + if arch, ok := strings.CutPrefix(seg, "installer-"); ok { + return arch + } + if seg == "source" { + return "source" + } + } + base, _ := splitCompressExt(path.Base(p)) + if arch, ok := strings.CutPrefix(base, "Contents-udeb-"); ok { + return arch + } + if arch, ok := strings.CutPrefix(base, "Contents-"); ok { + return arch + } + return "" +} + +// splitCompressExt splits a known compression extension off a path. +func splitCompressExt(p string) (string, string) { + for _, ext := range []string{".gz", ".xz", ".bz2", ".zst"} { + if strings.HasSuffix(p, ext) { + return strings.TrimSuffix(p, ext), ext + } + } + return p, "" +} + +// sortedFiles returns metadata file records in a stable path order. +func sortedFiles(files map[string]*debFile) []*debFile { + out := make([]*debFile, 0, len(files)) + for _, f := range files { + out = append(out, f) + } + sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path }) + return out +} + +// sortedKeys returns a map's keys in a stable order. +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/mirror/deb_test.go b/mirror/deb_test.go new file mode 100644 index 0000000..91c905d --- /dev/null +++ b/mirror/deb_test.go @@ -0,0 +1,203 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseRelease verifies field extraction and checksum block merging. +func TestParseRelease(t *testing.T) { + release := "Origin: Test\nArchitectures: amd64 arm64\nComponents: main universe\nAcquire-By-Hash: yes\n" + + "MD5Sum:\n aaa 10 main/binary-amd64/Packages\n" + + "SHA256:\n bbb 10 main/binary-amd64/Packages\n ccc 20 main/binary-amd64/Packages.gz\n" + rel, err := parseRelease([]byte(release)) + if err != nil { + t.Fatal(err) + } + if len(rel.components) != 2 || rel.components[0] != "main" { + t.Errorf("components = %v", rel.components) + } + if len(rel.architectures) != 2 { + t.Errorf("architectures = %v", rel.architectures) + } + if !rel.acquireByHash { + t.Error("acquireByHash not detected") + } + f := rel.files["main/binary-amd64/Packages"] + if f == nil || f.size != 10 || f.sums["md5sum"] != "aaa" || f.sums["sha256"] != "bbb" { + t.Errorf("merged file = %+v", f) + } + if rel.files["main/binary-amd64/Packages.gz"] == nil { + t.Error("gz variant missing") + } +} + +// TestIndexArch verifies architecture extraction from release file paths. +func TestIndexArch(t *testing.T) { + cases := map[string]string{ + "main/binary-amd64/Packages.gz": "amd64", + "main/debian-installer/binary-i386/Packages": "i386", + "main/installer-armhf/current/images/SHA256SUMS": "armhf", + "main/source/Sources.gz": "source", + "Contents-s390x.gz": "s390x", + "main/Contents-udeb-riscv64.gz": "riscv64", + "main/i18n/Translation-en.gz": "", + "main/dep11/icons-64x64.tar.gz": "", + "Release": "", + } + for p, want := range cases { + if got := indexArch(p); got != want { + t.Errorf("indexArch(%q) = %q, want %q", p, got, want) + } + } +} + +// TestIncludeIndexFile verifies component and architecture filtering. +func TestIncludeIndexFile(t *testing.T) { + cases := []struct { + path string + comps []string + archs []string + want bool + }{ + {"main/binary-amd64/Packages.gz", nil, nil, true}, + {"universe/binary-amd64/Packages.gz", []string{"main"}, nil, false}, + {"main/binary-i386/Packages.gz", []string{"main"}, []string{"amd64"}, false}, + {"main/binary-amd64/Packages.gz", []string{"main"}, []string{"amd64"}, true}, + {"main/i18n/Translation-en.gz", []string{"main"}, []string{"amd64"}, true}, + {"Contents-arm64.gz", []string{"main"}, []string{"amd64"}, false}, + {"main/source/Sources.gz", nil, []string{"amd64"}, false}, + {"main/source/Sources.gz", nil, []string{"amd64", "source"}, true}, + } + for _, c := range cases { + if got := includeIndexFile(c.path, c.comps, c.archs); got != c.want { + t.Errorf("includeIndexFile(%q, %v, %v) = %v, want %v", c.path, c.comps, c.archs, got, c.want) + } + } +} + +// TestSourceFiles verifies checksum list merging and directory joining for +// source package stanzas. +func TestSourceFiles(t *testing.T) { + st := stanza{ + "Directory": "pool/main/h/hello", + "Files": "\naaa 100 hello_1.0.dsc\nbbb 200 hello_1.0.tar.gz", + "Checksums-Sha256": "\nccc 100 hello_1.0.dsc\nddd 200 hello_1.0.tar.gz", + } + files := sourceFiles(st) + if len(files) != 2 { + t.Fatalf("got %d files, want 2", len(files)) + } + byPath := map[string]*debFile{} + for _, f := range files { + byPath[f.path] = f + } + dsc := byPath["pool/main/h/hello/hello_1.0.dsc"] + if dsc == nil || dsc.size != 100 || dsc.sums["md5sum"] != "aaa" || dsc.sums["sha256"] != "ccc" { + t.Errorf("dsc = %+v", dsc) + } +} + +// TestSyncDebStructured verifies a dists-layout synchronization: pool files +// at the archive root, promoted indexes, by-hash entries, and no staged +// leftovers. +func TestSyncDebStructured(t *testing.T) { + www := t.TempDir() + pool := testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "debian") + for p, data := range pool { + got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(p))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("pool file %s content mismatch", p) + } + } + suite := filepath.Join(root, "dists", "test") + if _, err := os.Stat(filepath.Join(suite, "Release")); err != nil { + t.Error("Release not promoted:", err) + } + if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages.gz")); err != nil { + t.Error("Packages.gz missing:", err) + } + // The uncompressed variant is listed in the Release file but not + // served, so it must be skipped without failing the sync. + if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages")); err == nil { + t.Error("unserved uncompressed Packages unexpectedly present") + } + byHash, err := filepath.Glob(filepath.Join(suite, "main", "binary-amd64", "by-hash", "SHA256", "*")) + if err != nil || len(byHash) == 0 { + t.Error("by-hash entries missing") + } + staged, _ := filepath.Glob(filepath.Join(suite, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncDebComponentFilter verifies filtered components are not mirrored. +func TestSyncDebComponentFilter(t *testing.T) { + www := t.TempDir() + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2, Components: []string{"main"}} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "debian") + if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil { + t.Error("main pool file missing:", err) + } + if _, err := os.Stat(filepath.Join(root, "pool", "universe")); err == nil { + t.Error("filtered universe pool files present") + } + if _, err := os.Stat(filepath.Join(root, "dists", "test", "universe")); err == nil { + t.Error("filtered universe indexes present") + } +} + +// TestSyncDebFlat verifies a repository without a dists directory mirrors +// relative to the repository URL itself. +func TestSyncDebFlat(t *testing.T) { + www := t.TempDir() + pool := testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/flat", opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "flat") + for name, data := range pool { + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(root, "Release")); err != nil { + t.Error("Release not promoted:", err) + } +} diff --git a/mirror/discover.go b/mirror/discover.go new file mode 100644 index 0000000..7aa6cb0 --- /dev/null +++ b/mirror/discover.go @@ -0,0 +1,178 @@ +package mirror + +import ( + "context" + "fmt" + "io" + "net/url" + "slices" + "sort" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" + "golang.org/x/net/html" +) + +// dirListing holds the entries parsed from an HTML directory index. +type dirListing struct { + dirs []string + files map[string]bool +} + +// discoverRepos crawls directory listings under baseURL looking for +// repositories of the given type, descending at most maxDepth levels. +func discoverRepos(ctx context.Context, baseURL string, typ RepoType, maxDepth int) ([]string, error) { + type node struct { + url string + depth int + } + start := strings.TrimRight(baseURL, "/") + "/" + queue := []node{{url: start}} + visited := map[string]bool{start: true} + var found []string + + for len(queue) > 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + n := queue[0] + queue = queue[1:] + listing, err := listDir(ctx, n.url) + if err != nil { + log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.") + continue + } + + // A directory containing repository metadata is synchronized as a + // whole; its children are not crawled further. + switch typ { + case RepoRPM: + if slices.Contains(listing.dirs, "repodata") { + found = append(found, strings.TrimRight(n.url, "/")) + continue + } + case RepoDeb: + if listing.files["InRelease"] || listing.files["Release"] { + found = append(found, strings.TrimRight(n.url, "/")) + continue + } + case RepoArch: + if hasArchDB(listing.files) { + found = append(found, strings.TrimRight(n.url, "/")) + continue + } + case RepoApk: + if listing.files[APKIndexName] { + found = append(found, strings.TrimRight(n.url, "/")) + continue + } + } + if n.depth >= maxDepth { + continue + } + for _, dir := range listing.dirs { + child := n.url + url.PathEscape(dir) + "/" + if visited[child] { + continue + } + visited[child] = true + queue = append(queue, node{url: child, depth: n.depth + 1}) + } + } + sort.Strings(found) + return found, nil +} + +// hasArchDB reports whether a directory listing contains a pacman database. +func hasArchDB(files map[string]bool) bool { + for name := range files { + if strings.HasSuffix(name, ".db") { + return true + } + } + return false +} + +// listDir fetches an HTML directory index and extracts its immediate child +// directories and files. +func listDir(ctx context.Context, dirURL string) (*dirListing, error) { + resp, err := fetch.Get(ctx, dirURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode) + } + + base, err := url.Parse(dirURL) + if err != nil { + return nil, err + } + listing := &dirListing{files: map[string]bool{}} + seenDirs := map[string]bool{} + tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20)) + for { + tt := tok.Next() + if tt == html.ErrorToken { + break + } + if tt != html.StartTagToken && tt != html.SelfClosingTagToken { + continue + } + name, hasAttr := tok.TagName() + if string(name) != "a" || !hasAttr { + continue + } + var href string + for { + key, val, more := tok.TagAttr() + if string(key) == "href" { + href = string(val) + } + if !more { + break + } + } + entry, isDir := childEntry(base, href) + if entry == "" { + continue + } + if isDir { + if !seenDirs[entry] { + seenDirs[entry] = true + listing.dirs = append(listing.dirs, entry) + } + } else { + listing.files[entry] = true + } + } + return listing, nil +} + +// childEntry resolves an anchor href against the directory URL, returning +// the entry name when it is an immediate child of that directory. +func childEntry(base *url.URL, href string) (string, bool) { + if href == "" || strings.HasPrefix(href, "#") { + return "", false + } + ref, err := url.Parse(href) + if err != nil || ref.RawQuery != "" || ref.Fragment != "" { + return "", false + } + resolved := base.ResolveReference(ref) + if resolved.Scheme != base.Scheme || resolved.Host != base.Host { + return "", false + } + rel := strings.TrimPrefix(resolved.Path, base.Path) + if rel == "" || rel == resolved.Path { + return "", false + } + isDir := strings.HasSuffix(rel, "/") + rel = strings.TrimSuffix(rel, "/") + if rel == "" || strings.Contains(rel, "/") { + return "", false + } + return rel, isDir +} diff --git a/mirror/discover_test.go b/mirror/discover_test.go new file mode 100644 index 0000000..48cf531 --- /dev/null +++ b/mirror/discover_test.go @@ -0,0 +1,138 @@ +package mirror + +import ( + "context" + "net/url" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestChildEntry verifies anchor resolution against a directory URL. +func TestChildEntry(t *testing.T) { + base, _ := url.Parse("http://example.com/repos/") + cases := []struct { + href string + want string + isDir bool + }{ + {"el9/", "el9", true}, + {"file.rpm", "file.rpm", false}, + {"/repos/el8/", "el8", true}, + {"../", "", false}, + {"?C=N;O=D", "", false}, + {"http://other.example.com/repos/x/", "", false}, + {"/outside/", "", false}, + {"deep/nested/", "", false}, + {"#anchor", "", false}, + } + for _, c := range cases { + got, isDir := childEntry(base, c.href) + if got != c.want || isDir != c.isDir { + t.Errorf("childEntry(%q) = %q,%v, want %q,%v", c.href, got, isDir, c.want, c.isDir) + } + } +} + +// TestDiscoverRepos verifies RPM repositories are found through directory +// listings and that the depth limit is honored. +func TestDiscoverRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "a", "repo1")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "b", "nested", "repo2")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "deep", "x", "y", "z", "repo3")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, RepoRPM, 3) + if err != nil { + t.Fatal(err) + } + want := []string{srv.URL + "/a/repo1", srv.URL + "/b/nested/repo2"} + if len(found) != len(want) { + t.Fatalf("found %v, want %v", found, want) + } + for i := range want { + if found[i] != want[i] { + t.Errorf("found[%d] = %q, want %q", i, found[i], want[i]) + } + } + + // A deeper limit reaches the third repository. + found, err = discoverRepos(context.Background(), srv.URL, RepoRPM, 6) + if err != nil { + t.Fatal(err) + } + if len(found) != 3 { + t.Errorf("depth 6 found %v, want 3 repositories", found) + } +} + +// TestDiscoverArchRepos verifies pacman repositories are recognized by +// their database files. +func TestDiscoverArchRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core") + testrepos.BuildArchRepo(t, filepath.Join(www, "extra", "os", "x86_64"), "extra") + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, RepoArch, 4) + if err != nil { + t.Fatal(err) + } + want := []string{srv.URL + "/core/os/x86_64", srv.URL + "/extra/os/x86_64"} + if len(found) != len(want) { + t.Fatalf("found %v, want %v", found, want) + } + for i := range want { + if found[i] != want[i] { + t.Errorf("found[%d] = %q, want %q", i, found[i], want[i]) + } + } +} + +// TestDiscoverApkRepos verifies Alpine repositories are recognized by +// their index files, matching the arch directories under a release tree. +func TestDiscoverApkRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "x86_64")) + testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "aarch64")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL+"/v3.24/community", RepoApk, 1) + if err != nil { + t.Fatal(err) + } + want := []string{srv.URL + "/v3.24/community/aarch64", srv.URL + "/v3.24/community/x86_64"} + if len(found) != len(want) { + t.Fatalf("found %v, want %v", found, want) + } + for i := range want { + if found[i] != want[i] { + t.Errorf("found[%d] = %q, want %q", i, found[i], want[i]) + } + } +} + +// TestDiscoverDebRepos verifies suite directories are recognized by their +// release files. +func TestDiscoverDebRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, RepoDeb, 4) + if err != nil { + t.Fatal(err) + } + want := []string{srv.URL + "/debian/dists/test", srv.URL + "/flat"} + if len(found) != len(want) { + t.Fatalf("found %v, want %v", found, want) + } + for i := range want { + if found[i] != want[i] { + t.Errorf("found[%d] = %q, want %q", i, found[i], want[i]) + } + } +} diff --git a/mirror/mirror_test.go b/mirror/mirror_test.go new file mode 100644 index 0000000..6b5aa44 --- /dev/null +++ b/mirror/mirror_test.go @@ -0,0 +1,14 @@ +package mirror + +import ( + "os" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + fetch.Reload() + os.Exit(m.Run()) +} diff --git a/mirror/options.go b/mirror/options.go new file mode 100644 index 0000000..a789378 --- /dev/null +++ b/mirror/options.go @@ -0,0 +1,91 @@ +// Package mirror synchronizes remote package repositories into a local +// directory tree, preserving the upstream layout so the result can be +// served directly to package managers. +package mirror + +import ( + "net/url" + "path" + "strings" + "time" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// RepoType identifies the repository format being synchronized. +type RepoType string + +const ( + // RepoRPM synchronizes yum/dnf style repositories. + RepoRPM RepoType = "rpm" + // RepoDeb synchronizes apt style repositories. + RepoDeb RepoType = "deb" + // RepoArch synchronizes pacman style repositories. + RepoArch RepoType = "arch" + // RepoApk synchronizes Alpine apk style repositories. + RepoApk RepoType = "apk" +) + +// Options configures a synchronization run. +type Options struct { + // Type selects the repository format. + Type RepoType + // URLs lists the repository or mirrorlist URLs to synchronize. + URLs []string + // Destination is the local directory repositories are placed under. + Destination string + // Trim drops this many leading URL path components when building the + // destination path. + Trim int + // Flat places repositories directly in Destination without copying the + // URL path. + Flat bool + // Discover crawls each URL's directory listings for repositories. + Discover bool + // DiscoverDepth limits how deep discovery crawls below each URL. + DiscoverDepth int + // Workers is the number of concurrent download workers. + Workers int + // Verify re-verifies checksums of existing local files. + Verify bool + // Prune deletes local files no longer part of the repository. + Prune bool + // PruneGrace defers pruning a file until it has been unreferenced for + // this long across runs. + PruneGrace time.Duration + // DryRun reports what would change without downloading packages or + // altering the published repository. + DryRun bool + // destBase, when set by the mirror server, pins the repository root + // directory directly instead of deriving it from the URL path. + destBase string + // Components limits deb synchronization to these components. + Components []string + // Architectures limits deb synchronization to these architectures. + Architectures []string +} + +// repoDest maps a repository URL onto the destination directory according +// to the configured trimming rules. +func (o *Options) repoDest(repoURL string) (string, error) { + if o.destBase != "" { + return o.destBase, nil + } + if o.Flat { + return o.Destination, nil + } + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + var segs []string + for _, seg := range strings.Split(u.Path, "/") { + if seg != "" { + segs = append(segs, seg) + } + } + if o.Trim >= len(segs) { + return o.Destination, nil + } + return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...)) +} diff --git a/mirror/options_test.go b/mirror/options_test.go new file mode 100644 index 0000000..84004de --- /dev/null +++ b/mirror/options_test.go @@ -0,0 +1,31 @@ +package mirror + +import ( + "path/filepath" + "testing" +) + +// TestRepoDest verifies URL path copying, trimming, and flat placement. +func TestRepoDest(t *testing.T) { + dest := t.TempDir() + repoURL := "https://example.com/repos/CentOS/7/EA4/" + cases := []struct { + name string + opts Options + want string + }{ + {"default", Options{Destination: dest}, filepath.Join(dest, "repos", "CentOS", "7", "EA4")}, + {"trim2", Options{Destination: dest, Trim: 2}, filepath.Join(dest, "7", "EA4")}, + {"trim-all", Options{Destination: dest, Trim: 10}, dest}, + {"flat", Options{Destination: dest, Flat: true}, dest}, + } + for _, c := range cases { + got, err := c.opts.repoDest(repoURL) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got != c.want { + t.Errorf("%s: got %q, want %q", c.name, got, c.want) + } + } +} diff --git a/mirror/resolve.go b/mirror/resolve.go new file mode 100644 index 0000000..a024061 --- /dev/null +++ b/mirror/resolve.go @@ -0,0 +1,90 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// probePaths lists the metadata files that identify a repository of a type. +// Types without fixed-name metadata return nothing. +func probePaths(typ RepoType) []string { + switch typ { + case RepoRPM: + return []string{"repodata/repomd.xml"} + case RepoDeb: + return []string{"InRelease", "Release"} + case RepoApk: + return []string{APKIndexName} + } + return nil +} + +// hasQueryOrFragment reports whether a URL carries a query or fragment, +// which makes it unusable as a base for joining repository paths onto. +// Unparsable URLs are reported as plain so the caller still probes them. +func hasQueryOrFragment(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + return u.RawQuery != "" || u.Fragment != "" +} + +// resolveSource decides whether repoURL is a repository root or a mirror +// list and returns the source to fetch from, along with the URL whose path +// reflects the repository layout: repoURL itself for a direct repository, +// or the first mirror's URL when repoURL is a mirror list. RPM mirror lists +// may be plain text or metalink documents. +func resolveSource(ctx context.Context, repoURL string, typ RepoType) (*fetch.Source, string, error) { + // Pacman databases are named after the repository, so there is no + // fixed path to probe and pacman mirrorlists are config snippets + // rather than URL lists; the URL is used directly. + direct := fetch.NewSource([]string{repoURL}) + probes := probePaths(typ) + if len(probes) == 0 { + return direct, repoURL, nil + } + + // Probe for repository metadata directly under the URL. A URL carrying a + // query or fragment cannot have a repository path joined onto it, as the + // path would land inside the query, so skip the probe: such URLs are + // mirrorlist or metalink endpoints and are read as one below. + if !hasQueryOrFragment(repoURL) { + for _, probe := range probes { + resp, err := direct.Get(ctx, probe, fetch.GetOptions{}) + if err == nil { + resp.Body.Close() + return direct, repoURL, nil + } + if !errors.Is(err, fetch.ErrNotFound) { + return nil, "", err + } + } + } + + // Fall back to reading the URL itself as a mirror list. + body, err := fetch.ReadURL(ctx, repoURL) + if err != nil { + return nil, "", fmt.Errorf("read mirrorlist %s: %w", repoURL, err) + } + kind := "mirrorlist" + var bases []string + if typ == RepoRPM { + if bases = fetch.ParseMetalink(body); len(bases) > 0 { + kind = "metalink" + } + } + if len(bases) == 0 { + bases = fetch.ParseMirrorlist(body) + } + if len(bases) == 0 { + return nil, "", fmt.Errorf("%s is neither a repository nor a mirrorlist", repoURL) + } + log.WithFields(log.Fields{"url": repoURL, "mirrors": len(bases), "kind": kind}).Info("Using mirror list for repository.") + return fetch.NewSource(bases), bases[0], nil +} diff --git a/mirror/resolve_test.go b/mirror/resolve_test.go new file mode 100644 index 0000000..a3e5157 --- /dev/null +++ b/mirror/resolve_test.go @@ -0,0 +1,123 @@ +package mirror + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestResolveSource verifies direct repositories are used as-is and +// mirrorlist bodies expand into multi-mirror sources. +func TestResolveSource(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + // The mirrorlist needs the real server URL, which is only known now + // that the listener is up; files are read per-request. + testrepos.WriteFile(t, filepath.Join(www, "ml.txt"), + []byte("http://127.0.0.1:1/el9\n"+srv.URL+"/el9\n")) + + direct, layoutURL, err := resolveSource(context.Background(), srv.URL+"/el9", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(direct.Bases()) != 1 { + t.Errorf("direct bases = %v", direct.Bases()) + } + if layoutURL != srv.URL+"/el9" { + t.Errorf("direct layout URL = %q", layoutURL) + } + + ml, layoutURL, err := resolveSource(context.Background(), srv.URL+"/ml.txt", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(ml.Bases()) != 2 { + t.Errorf("mirrorlist bases = %v", ml.Bases()) + } + if layoutURL != "http://127.0.0.1:1/el9" { + t.Errorf("mirrorlist layout URL = %q", layoutURL) + } + + if _, _, err := resolveSource(context.Background(), srv.URL+"/el9/Packages", RepoRPM); err == nil { + t.Error("expected error for a non-repository URL") + } +} + +// TestResolveSourceQueryMirrorlist verifies a mirrorlist URL carrying a +// query string is read as a mirrorlist rather than probed as a repository. +// A repository path cannot be joined onto such a URL: it would land inside +// the query, and endpoints that answer 200 with an error page for an +// unrecognized parameter would then look like a repository. +func TestResolveSourceQueryMirrorlist(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + repo := testrepos.ServeDir(t, www) + + var probed []string + list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probed = append(probed, r.URL.String()) + if r.URL.Query().Get("repo") != "os" { + // Several real mirrorlist services answer 200 with a message. + w.Write([]byte("Invalid release/repo/arch combination\n")) + return + } + w.Write([]byte(repo.URL + "/el9\n")) + })) + t.Cleanup(list.Close) + + src, layoutURL, err := resolveSource(context.Background(), list.URL+"/?release=9&repo=os", RepoRPM) + if err != nil { + t.Fatal(err) + } + if bases := src.Bases(); len(bases) != 1 || bases[0] != repo.URL+"/el9" { + t.Errorf("bases = %v, want [%s]", bases, repo.URL+"/el9") + } + if layoutURL != repo.URL+"/el9" { + t.Errorf("layout URL = %q, want %q", layoutURL, repo.URL+"/el9") + } + for _, u := range probed { + if strings.Contains(u, "repodata") { + t.Errorf("probe folded a repository path into the query: %s", u) + } + } +} + +// TestResolveSourceMetalink verifies metalink documents expand into +// multi-mirror sources for RPM repositories. +func TestResolveSourceMetalink(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + metalink := `` + + `http://127.0.0.1:1/dead/repodata/repomd.xml` + + `` + srv.URL + `/el9/repodata/repomd.xml` + + `` + testrepos.WriteFile(t, filepath.Join(www, "metalink.xml"), []byte(metalink)) + + src, layoutURL, err := resolveSource(context.Background(), srv.URL+"/metalink.xml", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(src.Bases()) != 2 || src.Bases()[1] != srv.URL+"/el9" { + t.Errorf("bases = %v", src.Bases()) + } + if layoutURL != "http://127.0.0.1:1/dead" { + t.Errorf("layout URL = %q", layoutURL) + } + + // The dead first mirror must fail over to the live one. + resp, err := src.Get(context.Background(), "repodata/repomd.xml", fetch.GetOptions{}) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} diff --git a/mirror/rpm.go b/mirror/rpm.go new file mode 100644 index 0000000..ff411e0 --- /dev/null +++ b/mirror/rpm.go @@ -0,0 +1,328 @@ +package mirror + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// repoMD is the partial XML schema for a yum repomd.xml file. Only the +// fields the sync needs are decoded; unknown elements are ignored so +// upstream additions do not break parsing. +type repoMD struct { + Data []repoData `xml:"data"` +} + +// repoData is one metadata file referenced from repomd.xml. +type repoData struct { + Type string `xml:"type,attr"` + Checksum repoChecksum `xml:"checksum"` + Location repoLocation `xml:"location"` + Size int64 `xml:"size"` +} + +// repoChecksum is a checksum element with its algorithm type. +type repoChecksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +// repoLocation captures the relative href of a metadata or package file. +type repoLocation struct { + Href string `xml:"href,attr"` +} + +// rpmPackage is the minimal package element decoded from primary metadata, +// carrying the location, checksum, and size needed to mirror the file. +type rpmPackage struct { + Checksum repoChecksum `xml:"checksum"` + Size rpmSize `xml:"size"` + Location repoLocation `xml:"location"` +} + +// rpmSize carries the on-disk package size from primary metadata. +type rpmSize struct { + Package int64 `xml:"package,attr"` +} + +// syncRPM synchronizes one RPM repository from src into destDir. +func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + + // Stage repomd.xml so clients of a live tree keep a consistent view of + // the old repository until everything else is in place. + repomdDst, err := fetch.LocalJoin(destDir, "repodata/repomd.xml") + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, "repodata/repomd.xml", repomdDst, nil, true, false); err != nil { + return fmt.Errorf("fetch repomd.xml: %w", err) + } + md, err := readRepomd(fetch.StagedOrFinal(repomdDst)) + if err != nil { + return err + } + + // Download the metadata files referenced by repomd. Their names embed + // content hashes, so they can be written directly into place. A dry run + // only fetches the indexes that must be parsed, staged for discard. + var jobs, planned []fetch.Job + var primary *repoData + var deltaMDs []*repoData + var parseDsts []string + for i, data := range md.Data { + if data.Location.Href == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, data.Location.Href) + if err != nil { + return err + } + job := fetch.Job{ + ReqPath: data.Location.Href, + Dst: dst, + Want: fetch.MakeExpect(data.Size, data.Checksum.Type, data.Checksum.Value), + } + parsed := false + switch data.Type { + case "primary": + primary = &md.Data[i] + parsed = true + case "prestodelta", "deltainfo": + deltaMDs = append(deltaMDs, &md.Data[i]) + parsed = true + } + if opts.DryRun { + if !parsed { + planned = append(planned, job) + continue + } + job.Stage = true + parseDsts = append(parseDsts, dst) + } + jobs = append(jobs, job) + } + fetch.PlanJobs(planned, opts.Verify, keep) + if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil { + return fmt.Errorf("fetch repository metadata: %w", err) + } + if primary == nil { + return errors.New("repository metadata lists no primary index") + } + + // Collect package locations from the primary index, plus any delta + // packages referenced by prestodelta metadata. + primaryPath, err := fetch.LocalJoin(destDir, primary.Location.Href) + if err != nil { + return err + } + pkgs, err := readPrimary(fetch.StagedOrFinal(primaryPath)) + if err != nil { + return err + } + jobs = jobs[:0] + for _, pkg := range pkgs { + if pkg.Location.Href == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, pkg.Location.Href) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ + ReqPath: pkg.Location.Href, + Dst: dst, + Want: fetch.MakeExpect(pkg.Size.Package, pkg.Checksum.Type, pkg.Checksum.Value), + }) + } + for _, deltaMD := range deltaMDs { + deltaPath, err := fetch.LocalJoin(destDir, deltaMD.Location.Href) + if err != nil { + return err + } + deltas, err := readDeltas(fetch.StagedOrFinal(deltaPath)) + if err != nil { + return err + } + for _, delta := range deltas { + if delta.Filename == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, delta.Filename) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ + ReqPath: delta.Filename, + Dst: dst, + Want: fetch.MakeExpect(delta.Size, delta.Checksum.Type, delta.Checksum.Value), + }) + } + } + log.WithField("packages", len(jobs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // A dry run discards the indexes it staged for parsing. + for _, dst := range parseDsts { + _ = os.Remove(dst + fetch.StagedSuffix) + } + + // Stage the optional signature material so it is published together + // with the repomd.xml it signs, never beside the previous index. + var sigDsts []string + for _, extra := range []string{"repodata/repomd.xml.asc", "repodata/repomd.xml.key"} { + dst, err := fetch.LocalJoin(destDir, extra) + if err != nil { + return err + } + _, err = fetch.File(ctx, src, extra, dst, nil, true, false) + switch { + case err == nil: + sigDsts = append(sigDsts, dst) + case errors.Is(err, fetch.ErrNotFound): + // The upstream dropped the signature; drop the local copy so + // a stale signature is never served beside a new repomd.xml. + if !opts.DryRun { + fetch.RemoveStale(dst) + } + default: + return fmt.Errorf("fetch %s: %w", extra, err) + } + } + + // Promote the signatures and then repomd.xml so the published metadata + // chain is complete. + for _, dst := range sigDsts { + if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { + return err + } + if _, err := os.Stat(dst); err == nil { + keep.Add(dst) + } + } + if err := fetch.PromoteOrDiscard(repomdDst, opts.DryRun); err != nil { + return err + } + keep.Add(repomdDst) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + return nil +} + +// rpmDelta is one delta package referenced from prestodelta metadata. +type rpmDelta struct { + Filename string `xml:"filename"` + Size int64 `xml:"size"` + Checksum repoChecksum `xml:"checksum"` +} + +// readDeltas streams prestodelta metadata and collects each delta package +// reference so drpm files are mirrored alongside full packages. +func readDeltas(filename string) ([]rpmDelta, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + dec := xml.NewDecoder(r) + var deltas []rpmDelta + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse delta metadata %s: %w", filename, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "delta" { + continue + } + var delta rpmDelta + if err := dec.DecodeElement(&delta, &start); err != nil { + return nil, fmt.Errorf("parse delta entry %s: %w", filename, err) + } + if delta.Filename != "" { + deltas = append(deltas, delta) + } + } + return deltas, nil +} + +// readRepomd decodes a repomd.xml file. +func readRepomd(filename string) (*repoMD, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + var md repoMD + if err := xml.NewDecoder(f).Decode(&md); err != nil { + return nil, fmt.Errorf("parse repomd %s: %w", filename, err) + } + return &md, nil +} + +// readPrimary streams a primary metadata index and collects each package +// entry without loading the whole document into memory. +func readPrimary(filename string) ([]rpmPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + dec := xml.NewDecoder(r) + var pkgs []rpmPackage + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse primary metadata %s: %w", filename, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg rpmPackage + if err := dec.DecodeElement(&pkg, &start); err != nil { + return nil, fmt.Errorf("parse package entry %s: %w", filename, err) + } + if pkg.Location.Href != "" { + pkgs = append(pkgs, pkg) + } + } + return pkgs, nil +} diff --git a/mirror/rpm_test.go b/mirror/rpm_test.go new file mode 100644 index 0000000..e61c8d7 --- /dev/null +++ b/mirror/rpm_test.go @@ -0,0 +1,171 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// syncRPMFixture builds a served RPM repository, synchronizes it, and +// returns the fixture directory, repository URL, and local repository path. +func syncRPMFixture(t *testing.T, opts *Options) (string, string, string, map[string][]byte) { + t.Helper() + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + pkgs := testrepos.BuildRPMRepo(t, repoDir) + srv := testrepos.ServeDir(t, www) + repoURL := srv.URL + "/repos/el9" + + opts.Type = RepoRPM + if opts.Destination == "" { + opts.Destination = t.TempDir() + } + if opts.Workers == 0 { + opts.Workers = 2 + } + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + return repoDir, repoURL, filepath.Join(opts.Destination, "repos", "el9"), pkgs +} + +// TestSyncRPM verifies a full synchronization mirrors packages and metadata +// with no staged leftovers. +func TestSyncRPM(t *testing.T) { + _, _, local, pkgs := syncRPMFixture(t, &Options{}) + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, "Packages", name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml")); err != nil { + t.Error("repomd.xml not promoted:", err) + } + if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml.asc")); err != nil { + t.Error("repomd.xml.asc not promoted:", err) + } + drpm, err := os.ReadFile(filepath.Join(local, "drpms", "foo-0.9_1.0-1.x86_64.drpm")) + if err != nil { + t.Error("delta package not mirrored:", err) + } else if !bytes.Equal(drpm, testrepos.RPMDelta()) { + t.Error("delta package content mismatch") + } + staged, _ := filepath.Glob(filepath.Join(local, "repodata", "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncRPMVerify verifies size-only skipping keeps same-size corruption +// while the verify option repairs it. +func TestSyncRPMVerify(t *testing.T) { + opts := &Options{} + repoDir, repoURL, local, _ := syncRPMFixture(t, opts) + _ = repoDir + + // Corrupt a local package without changing its size. + victim := filepath.Join(local, "Packages", "foo-1.0-1.x86_64.rpm") + corrupt := bytes.Repeat([]byte("oof"), 500) + if err := os.WriteFile(victim, corrupt, 0644); err != nil { + t.Fatal(err) + } + + // A plain re-run trusts the matching size and keeps the corruption. + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(victim) + if !bytes.Equal(got, corrupt) { + t.Error("size-only run unexpectedly rewrote the file") + } + + // A verify run re-hashes local files and repairs the corruption. + opts.Verify = true + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + got, _ = os.ReadFile(victim) + if !bytes.Equal(got, bytes.Repeat([]byte("foo"), 500)) { + t.Error("verify run did not repair the corruption") + } +} + +// TestSyncRPMPrune verifies stale files are removed only when pruning is +// enabled. +func TestSyncRPMPrune(t *testing.T) { + opts := &Options{} + _, repoURL, local, _ := syncRPMFixture(t, opts) + + stale := filepath.Join(local, "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Error("file pruned without the prune option") + } + + opts.Prune = true + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale file survived pruning") + } +} + +// TestSyncRPMDroppedSignature verifies a signature the upstream stops +// serving is removed locally instead of lingering beside a new repomd.xml. +func TestSyncRPMDroppedSignature(t *testing.T) { + opts := &Options{} + repoDir, repoURL, local, _ := syncRPMFixture(t, opts) + + ascLocal := filepath.Join(local, "repodata", "repomd.xml.asc") + if _, err := os.Stat(ascLocal); err != nil { + t.Fatal("signature not mirrored:", err) + } + if err := os.Remove(filepath.Join(repoDir, "repodata", "repomd.xml.asc")); err != nil { + t.Fatal(err) + } + + if err := syncOne(context.Background(), repoURL, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ascLocal); !os.IsNotExist(err) { + t.Error("stale signature survived after the upstream dropped it") + } +} + +// TestSyncRPMChecksumMismatch verifies a package whose served content does +// not match the published checksum fails the synchronization. +func TestSyncRPMChecksumMismatch(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + testrepos.BuildRPMRepo(t, repoDir) + srv := testrepos.ServeDir(t, www) + + // Replace a served package with same-size different content after the + // metadata was generated. + victim := filepath.Join(repoDir, "Packages", "foo-1.0-1.x86_64.rpm") + if err := os.WriteFile(victim, bytes.Repeat([]byte("fox"), 500), 0644); err != nil { + t.Fatal(err) + } + + opts := &Options{Type: RepoRPM, Destination: t.TempDir(), Workers: 2} + err := syncOne(context.Background(), srv.URL+"/repos/el9", opts) + if err == nil || !strings.Contains(err.Error(), "checksum") { + t.Errorf("expected checksum mismatch error, got %v", err) + } +} diff --git a/mirror/sync.go b/mirror/sync.go new file mode 100644 index 0000000..aec4232 --- /dev/null +++ b/mirror/sync.go @@ -0,0 +1,177 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// lockDestination takes an exclusive lock under the destination directory +// so overlapping runs, such as from cron, cannot race on the same tree. +// The caller closes the returned file to release the lock. +func lockDestination(dest string) (*os.File, error) { + if err := os.MkdirAll(dest, 0755); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dest, fetch.LockFileName), os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("destination %s is locked by another repo-sync run", dest) + } + return f, nil +} + +// Sync synchronizes every configured repository URL into the destination, +// continuing after per-repository failures so one bad repository does not +// block the rest. +func Sync(ctx context.Context, opts *Options) error { + fetch.Reload() + + // Hold the destination for the whole run; the lock file itself stays + // behind, as removing it would race a waiting run. + lock, err := lockDestination(opts.Destination) + if err != nil { + return err + } + defer lock.Close() + + // Expand discovery URLs into concrete repository URLs. + repoURLs := opts.URLs + if opts.Discover { + repoURLs = nil + for _, base := range opts.URLs { + found, err := discoverRepos(ctx, base, opts.Type, opts.DiscoverDepth) + if err != nil { + return fmt.Errorf("discover repositories under %s: %w", base, err) + } + log.WithFields(log.Fields{"url": base, "repositories": len(found)}).Info("Discovered repositories.") + repoURLs = append(repoURLs, found...) + } + } + if len(repoURLs) == 0 { + return errors.New("no repositories to synchronize") + } + + // Synchronize each repository in turn. + var failures int + for _, repoURL := range repoURLs { + if err := ctx.Err(); err != nil { + return err + } + log.WithField("url", repoURL).Info("Synchronizing repository.") + fetch.Stats.Reset() + start := time.Now() + if err := syncOne(ctx, repoURL, opts); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + log.WithError(err).WithField("url", repoURL).Error("Repository synchronization failed.") + failures++ + continue + } + logSummary(repoURL, opts, time.Since(start)) + } + if failures > 0 { + return fmt.Errorf("%d of %d repositories failed to synchronize", failures, len(repoURLs)) + } + return nil +} + +// logSummary reports one repository's transfer activity after a successful +// synchronization. +func logSummary(repoURL string, opts *Options, elapsed time.Duration) { + fields := log.Fields{ + "url": repoURL, + "fetched": fetch.Stats.Fetched.Load(), + "transferred": fetch.FormatBytes(fetch.Stats.FetchedBytes.Load()), + "unchanged": fetch.Stats.Unchanged.Load(), + "duration": elapsed.Round(time.Millisecond).String(), + } + if opts.DryRun { + fields["would_fetch"] = fetch.Stats.Planned.Load() + fields["would_transfer"] = fetch.FormatBytes(fetch.Stats.PlannedBytes.Load()) + if opts.Prune { + fields["would_prune"] = fetch.Stats.Pruned.Load() + } + log.WithFields(fields).Info("Dry run complete.") + return + } + if opts.Prune { + fields["pruned"] = fetch.Stats.Pruned.Load() + } + log.WithFields(fields).Info("Repository synchronized.") +} + +// syncOne resolves the source for a single repository URL and runs the +// format-specific synchronization. Destination paths derive from the layout +// URL, so a mirror list maps to the first mirror's path rather than the +// list's own URL. +func syncOne(ctx context.Context, repoURL string, opts *Options) error { + src, layoutURL, err := resolveSource(ctx, repoURL, opts.Type) + if err != nil { + return err + } + switch opts.Type { + case RepoRPM: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncRPM(ctx, src, destDir, opts) + case RepoDeb: + return syncDeb(ctx, src, layoutURL, opts) + case RepoArch: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncArch(ctx, src, layoutURL, destDir, opts) + case RepoApk: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncApk(ctx, src, destDir, opts) + default: + return fmt.Errorf("unsupported repository type %q", opts.Type) + } +} + +// SyncInto synchronizes one repository into an explicit destination +// directory, bypassing URL-derived destination mapping. It exists for the +// mirror server, whose destinations follow request paths rather than +// upstream URL paths. For deb the destination is the archive root +// directory; for other types it is the repository directory itself. +// Concurrent callers interleave into fetch.Stats, so the counters carry no +// per-repository meaning on this path and are neither reset nor reported. +func SyncInto(ctx context.Context, typ RepoType, src *fetch.Source, repoURL, dest string, opts *Options) error { + switch typ { + case RepoRPM: + return syncRPM(ctx, src, dest, opts) + case RepoDeb: + // Copy before pinning the destination so the caller's Options is + // never mutated, keeping the struct safe to reuse or share. + debOpts := *opts + debOpts.destBase = dest + return syncDeb(ctx, src, repoURL, &debOpts) + case RepoArch: + return syncArch(ctx, src, repoURL, dest, opts) + case RepoApk: + return syncApk(ctx, src, dest, opts) + default: + return fmt.Errorf("unsupported repository type %q", typ) + } +} diff --git a/mirror/sync_test.go b/mirror/sync_test.go new file mode 100644 index 0000000..c9b6b62 --- /dev/null +++ b/mirror/sync_test.go @@ -0,0 +1,158 @@ +package mirror + +import ( + "context" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// countFiles returns the number of regular files under root. +func countFiles(t *testing.T, root string) int { + t.Helper() + count := 0 + err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + count++ + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return count +} + +// TestDryRun verifies a dry run leaves the destination unchanged: no +// packages downloaded, no metadata published, and no files pruned. +func TestDryRun(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + // A dry run against an empty destination writes no files at all. + dest := t.TempDir() + opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, DryRun: true, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + if n := countFiles(t, dest); n != 0 { + t.Errorf("dry run left %d files in the destination", n) + } + + // A real run works, and a later dry run with pruning keeps stale files. + opts.DryRun = false + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "el9", "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + opts.DryRun = true + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Error("dry run pruned a file:", err) + } +} + +// TestPruneGrace verifies stale files wait out the grace period across +// runs before being removed. +func TestPruneGrace(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, Prune: true, PruneGrace: time.Hour} + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "el9") + stale := filepath.Join(local, "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + + // The first run only starts the grace period. + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Fatal("file pruned before its grace period expired:", err) + } + statePath := filepath.Join(local, fetch.PruneStateName) + if _, err := os.Stat(statePath); err != nil { + t.Fatal("prune state not recorded:", err) + } + + // A second run within the grace period still keeps the file. + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Fatal("file pruned within its grace period:", err) + } + + // Age the recorded timestamp past the grace period; the next run + // prunes the file and clears the bookkeeping. + data, err := os.ReadFile(statePath) + if err != nil { + t.Fatal(err) + } + state := map[string]time.Time{} + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + for rel := range state { + state[rel] = time.Now().Add(-2 * time.Hour) + } + data, err = json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(statePath, data, 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale file survived past its grace period") + } + if _, err := os.Stat(statePath); err == nil { + t.Error("prune state lingered after all grace periods resolved") + } +} + +// TestLockDestination verifies a held destination lock blocks a second run +// and is released on close. +func TestLockDestination(t *testing.T) { + dest := t.TempDir() + lock, err := lockDestination(dest) + if err != nil { + t.Fatal(err) + } + if _, err := lockDestination(dest); err == nil { + t.Error("second lock unexpectedly succeeded") + } + lock.Close() + lock, err = lockDestination(dest) + if err != nil { + t.Error("lock not released on close:", err) + } else { + lock.Close() + } +} diff --git a/server/classify.go b/server/classify.go new file mode 100644 index 0000000..eac42c1 --- /dev/null +++ b/server/classify.go @@ -0,0 +1,98 @@ +package server + +import ( + "path" + "strings" + + "github.com/grmrgecko/repo-sync/mirror" +) + +// kindGeneric marks request paths that are not repository entry points. +const kindGeneric = "generic" + +// resource is a crawlable unit derived from a request path. +type resource struct { + // Kind is a RepoType string, or kindGeneric. + Kind string + // Key uniquely identifies the resource in state and the lock map. + Key string + // Path is the repository request path (the suite directory for deb). + Path string + // Root is the request path prefix covering every file the resource + // owns; deb uses the archive root so pool requests match. + Root string + // ReqPath preserves the original request path. + ReqPath string +} + +// classifyRequest maps a request path onto a crawlable resource. Only +// repository entry-point metadata registers a repository; every other path +// is generic and served on demand. +func classifyRequest(reqPath string) resource { + clean := path.Clean("/" + strings.TrimPrefix(reqPath, "/")) + base := path.Base(clean) + dir := path.Dir(clean) + + // RPM repositories are identified by any repodata request. + if idx := strings.Index(clean, "/repodata/"); idx >= 0 { + root := clean[:idx] + if root == "" { + root = "/" + } + return resource{ + Kind: string(mirror.RepoRPM), + Key: string(mirror.RepoRPM) + ":" + root, + Path: root, + Root: root, + ReqPath: clean, + } + } + + // Apt suites are identified by their release files; the archive root + // above dists also covers the pool. + if base == "InRelease" || base == "Release" || base == "Release.gpg" { + root := dir + if idx := strings.Index(clean, "/dists/"); idx >= 0 { + root = clean[:idx] + if root == "" { + root = "/" + } + } + return resource{ + Kind: string(mirror.RepoDeb), + Key: string(mirror.RepoDeb) + ":" + dir, + Path: dir, + Root: root, + ReqPath: clean, + } + } + + // Pacman repositories are identified by their database file. + if strings.HasSuffix(base, ".db") { + return resource{ + Kind: string(mirror.RepoArch), + Key: string(mirror.RepoArch) + ":" + dir, + Path: dir, + Root: dir, + ReqPath: clean, + } + } + + // Alpine repositories are identified by their index archive. + if base == mirror.APKIndexName { + return resource{ + Kind: string(mirror.RepoApk), + Key: string(mirror.RepoApk) + ":" + dir, + Path: dir, + Root: dir, + ReqPath: clean, + } + } + + return resource{ + Kind: kindGeneric, + Key: kindGeneric + ":" + clean, + Path: clean, + ReqPath: clean, + } +} diff --git a/server/classify_test.go b/server/classify_test.go new file mode 100644 index 0000000..202e8c4 --- /dev/null +++ b/server/classify_test.go @@ -0,0 +1,68 @@ +package server + +import ( + "testing" + "time" + + "github.com/grmrgecko/repo-sync/state" +) + +// TestClassifyRequest verifies request paths map to the right resource +// kinds, crawl paths, and keep-alive roots. +func TestClassifyRequest(t *testing.T) { + cases := []struct { + reqPath string + kind string + path string + root string + }{ + {"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"}, + {"/almalinux/9/BaseOS/x86_64/os/repodata/abc-primary.xml.gz", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"}, + {"/debian/dists/bookworm/InRelease", "deb", "/debian/dists/bookworm", "/debian"}, + {"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"}, + {"/flat/Release", "deb", "/flat", "/flat"}, + {"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"}, + {"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"}, + {"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""}, + {"/notes/README.txt", kindGeneric, "/notes/README.txt", ""}, + } + for _, c := range cases { + res := classifyRequest(c.reqPath) + if res.Kind != c.kind || res.Path != c.path || res.Root != c.root { + t.Errorf("classifyRequest(%q) = {%s %s %s}, want {%s %s %s}", + c.reqPath, res.Kind, res.Path, res.Root, c.kind, c.path, c.root) + } + } +} + +// TestRefreshTier verifies the tier walk and staleness budget. +func TestRefreshTier(t *testing.T) { + schedule := []time.Duration{4 * time.Hour, 12 * time.Hour, 24 * time.Hour} + if interval, stale := refreshTier(time.Hour, schedule); stale || interval != 4*time.Hour { + t.Errorf("fresh resource tier = %v, %v", interval, stale) + } + if interval, stale := refreshTier(10*time.Hour, schedule); stale || interval != 12*time.Hour { + t.Errorf("aging resource tier = %v, %v", interval, stale) + } + if _, stale := refreshTier(41*time.Hour, schedule); !stale { + t.Error("resource past the budget not stale") + } +} + +// TestNextCrawlAt verifies crawl gating for fresh, failed, and deferred +// entries. +func TestNextCrawlAt(t *testing.T) { + now := time.Now() + if !nextCrawlAt(state.Entry{}, time.Hour).IsZero() { + t.Error("never-crawled entry not due immediately") + } + failedAt := now.Add(30 * time.Minute) + failed := state.Entry{LastError: "boom", LastCrawled: now, NextCrawl: failedAt} + if got := nextCrawlAt(failed, time.Hour); !got.Equal(failedAt) { + t.Errorf("failed entry gate = %v, want %v", got, failedAt) + } + ok := state.Entry{LastCrawled: now, NextCrawl: now.Add(10 * time.Minute)} + if got := nextCrawlAt(ok, 2*time.Hour); !got.Equal(now.Add(2 * time.Hour)) { + t.Errorf("tier gate = %v, want %v", got, now.Add(2*time.Hour)) + } +} diff --git a/server/crawl_loop.go b/server/crawl_loop.go new file mode 100644 index 0000000..317ff0d --- /dev/null +++ b/server/crawl_loop.go @@ -0,0 +1,435 @@ +package server + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/mirror" + "github.com/grmrgecko/repo-sync/state" + log "github.com/sirupsen/logrus" +) + +// lockMu guards the per-resource lock table and the in-flight crawl set. +var lockMu sync.Mutex + +// resourceLock is one entry in the lock table; refs counts holders and +// waiters so the entry can be dropped once nobody references it. +type resourceLock struct { + mu sync.Mutex + refs int +} + +// locks serializes crawls of the same resource across request handlers and +// the scheduled loop. +var locks = map[string]*resourceLock{} + +// lockResource acquires the lock for a resource key and returns its unlock +// function. Entries are reference counted and removed when the last holder +// or waiter releases, so evicted resources do not accumulate in the table. +func lockResource(key string) func() { + lockMu.Lock() + l := locks[key] + if l == nil { + l = &resourceLock{} + locks[key] = l + } + l.refs++ + lockMu.Unlock() + + l.mu.Lock() + return func() { + l.mu.Unlock() + lockMu.Lock() + l.refs-- + if l.refs == 0 { + delete(locks, key) + } + lockMu.Unlock() + } +} + +// inventoryEvictionGrace shields resources still advertised by a mount's +// configuration from eviction after requests stop. +const inventoryEvictionGrace = 72 * time.Hour + +// inflight records resources with a crawl queued or running so concurrent +// dispatches from the request handlers and the scheduler collapse into one +// crawl. Guarded by lockMu. +var inflight = map[string]bool{} + +// crawlWG tracks dispatched crawl goroutines so shutdown can wait for them +// before the final state flush records their outcomes. +var crawlWG sync.WaitGroup + +// crawlCtx is the context dispatched crawls run under. CrawlLoop installs +// its own so shutdown cancels crawls; the background default covers tests +// and requests arriving before the loop starts. +// A pointer is stored rather than the interface itself: atomic.Value +// rejects a second store of a different concrete type, and the background +// default and the loop's cancelable context are different types. +var crawlCtx atomic.Pointer[context.Context] + +func init() { + ctx := context.Background() + crawlCtx.Store(&ctx) +} + +// Crawl slots bound how many crawls run at once; slotCond re-reads the +// configured limit on every wake so a reload takes effect. +var ( + slotMu sync.Mutex + slotCond = sync.NewCond(&slotMu) + slotsInUse int +) + +// acquireCrawlSlot blocks until a crawl slot is free under the configured +// concurrency limit. +func acquireCrawlSlot() { + slotMu.Lock() + for slotsInUse >= cfg.C.Crawler.ConcurrentCrawls { + slotCond.Wait() + } + slotsInUse++ + slotMu.Unlock() +} + +// releaseCrawlSlot frees a crawl slot and wakes the waiters. +func releaseCrawlSlot() { + slotMu.Lock() + slotsInUse-- + slotMu.Unlock() + slotCond.Broadcast() +} + +// startCrawl dispatches an asynchronous crawl of one resource, collapsing +// duplicate dispatches and bounding concurrency. Failures are logged; the +// retry gating lives in the schedulers. +func startCrawl(res resource) { + lockMu.Lock() + if inflight[res.Key] { + lockMu.Unlock() + return + } + inflight[res.Key] = true + lockMu.Unlock() + + crawlWG.Add(1) + go func() { + defer crawlWG.Done() + defer func() { + lockMu.Lock() + delete(inflight, res.Key) + lockMu.Unlock() + }() + acquireCrawlSlot() + defer releaseCrawlSlot() + ctx := *crawlCtx.Load() + if ctx.Err() != nil { + return + } + if err := crawlResource(ctx, res); err != nil { + log.WithError(err).WithField("key", res.Key).Error("Repository crawl failed.") + } + }() +} + +// WaitCrawls blocks until every dispatched crawl has finished. It is called +// at shutdown after the crawl context is canceled and before the final +// state flush. +func WaitCrawls() { + crawlWG.Wait() +} + +// crawlResource synchronizes one classified resource under its lock. Deb +// resources also hold an archive-root lock so suites sharing a pool never +// download the same file concurrently. +func crawlResource(ctx context.Context, res resource) error { + if res.Kind == string(mirror.RepoDeb) && res.Root != res.Path { + defer lockResource("deb-root:" + res.Root)() + } + defer lockResource(res.Key)() + + err := dispatchCrawl(ctx, res) + state.S.MarkCrawled(res.Key, time.Now(), err) + return err +} + +// pathBelow reports whether a request path sits strictly below a base path. +func pathBelow(p, base string) bool { + if p == base { + return false + } + if base == "/" { + return true + } + return strings.HasPrefix(p, base+"/") +} + +// prunableTree reports whether a crawl may prune its destination tree. Every +// sync prunes the repository directory, which for deb is the suite directory +// and otherwise the repository's own path. A crawl's keep set covers only +// that one repository, so a tree that also holds another mount's content or +// another tracked repository must not be pruned: the neighbour's files are +// unreferenced there and would be deleted. This matters most for a +// repository sitting at the mirror root, whose tree is the whole cache. +func prunableTree(conf *cfg.Config, res resource) bool { + for _, mount := range conf.Mounts { + if pathBelow(mount.Path, res.Path) { + log.WithFields(log.Fields{"key": res.Key, "mount": mount.Path}). + Debug("Pruning disabled; the repository tree contains another mount.") + return false + } + } + for key, entry := range state.S.Snapshot() { + if key == res.Key || entry.Kind == kindGeneric { + continue + } + if pathBelow(entry.Path, res.Path) { + log.WithFields(log.Fields{"key": res.Key, "nested": key}). + Debug("Pruning disabled; the repository tree contains another repository.") + return false + } + } + return true +} + +// dispatchCrawl runs the format-specific synchronization for a resource +// into the online tree. +func dispatchCrawl(ctx context.Context, res resource) error { + online := cfg.C.OnlineDomain() + mount, ok := cfg.C.MountFor(res.Path) + if !ok { + return fmt.Errorf("no mount configured for %s", res.Path) + } + + // upstreamFor maps a request path onto the mount's upstream URL. + upstreamFor := func(p string) string { + rel := p + if mount.Path != "/" { + rel = strings.TrimPrefix(p, mount.Path) + } + return mount.Upstream + rel + } + + // Generic files never reach the crawl path: they refresh on demand at + // serve time and expire through the failure cache and state eviction. + typ := mirror.RepoType(res.Kind) + opts := &mirror.Options{ + Type: typ, + Workers: cfg.C.Crawler.Workers, + Prune: prunableTree(cfg.C, res), + PruneGrace: cfg.C.Crawler.PruneGrace, + } + + // Deb destinations pin the archive root so pool files land beside + // their suites; every other type syncs into its own directory. + destPath := res.Path + if typ == mirror.RepoDeb { + destPath = res.Root + } + dest, err := fetch.LocalJoin(online.Root, destPath) + if err != nil { + return err + } + repoURL := upstreamFor(res.Path) + return mirror.SyncInto(ctx, typ, fetch.NewSource([]string{repoURL}), repoURL, dest, opts) +} + +// CrawlLoop keeps configured and discovered resources fresh until the +// context is canceled, starting with an immediate pass. Its context also +// governs request-dispatched crawls so shutdown cancels them. +func CrawlLoop(ctx context.Context) { + crawlCtx.Store(&ctx) + runScheduledCrawls(ctx) + t := time.NewTicker(cfg.C.CrawlCheckInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + fetchFailures.Sweep(time.Now()) + runScheduledCrawls(ctx) + // Re-arm from the active config so a reload takes effect. + t.Reset(cfg.C.CrawlCheckInterval) + } + } +} + +// runScheduledCrawls runs one pass of configured and request-discovered +// crawls. +func runScheduledCrawls(ctx context.Context) { + crawlConfigured(ctx) + crawlDue(ctx) +} + +// configuredResource builds the resource for a mount's configured +// repository. +func configuredResource(repo cfg.MountRepoConfig) resource { + root := repo.Path + if repo.Type == string(mirror.RepoDeb) { + if idx := strings.Index(repo.Path, "/dists/"); idx >= 0 { + root = repo.Path[:idx] + if root == "" { + root = "/" + } + } + } + return resource{ + Kind: repo.Type, + Key: repo.Type + ":" + repo.Path, + Path: repo.Path, + Root: root, + ReqPath: repo.Path, + } +} + +// crawlConfigured keeps every mount-configured repository synchronized +// regardless of client traffic. Crawls are dispatched asynchronously so a +// slow repository cannot stall the scheduler loop. +func crawlConfigured(ctx context.Context) { + now := time.Now() + for _, mount := range cfg.C.Mounts { + for _, repo := range mount.Repos { + if ctx.Err() != nil { + return + } + res := configuredResource(repo) + state.S.MarkSeenInInventory(res.Kind, res.Key, res.Path, res.Root, now) + entry, _ := state.S.Entry(res.Key) + if !dueForInventory(entry, now) { + continue + } + startCrawl(res) + } + } +} + +// dueForInventory gates configured crawls on the standard interval with +// faster retries after failures. +func dueForInventory(entry state.Entry, now time.Time) bool { + if entry.LastCrawled.IsZero() { + return true + } + if !entry.NextCrawl.IsZero() { + return !now.Before(entry.NextCrawl) + } + return now.Sub(entry.LastCrawled) >= cfg.C.RepoCrawlInterval +} + +// refreshTier walks the cumulative schedule, returning the refresh +// interval matching how long ago the resource was requested, or stale once +// the whole budget has elapsed. +func refreshTier(sinceRequested time.Duration, schedule []time.Duration) (time.Duration, bool) { + var cumulative time.Duration + for _, step := range schedule { + if step <= 0 { + continue + } + cumulative += step + if sinceRequested < cumulative { + return step, false + } + } + return 0, true +} + +// refreshSchedule is the tier list: the active crawl interval followed by +// the configured aging backoff steps. +func refreshSchedule() []time.Duration { + schedule := make([]time.Duration, 0, 1+len(cfg.C.Crawler.RefreshSchedule)) + schedule = append(schedule, cfg.C.RepoCrawlInterval) + return append(schedule, cfg.C.Crawler.RefreshSchedule...) +} + +// activeInInventory reports whether a mount still advertised the resource +// recently enough to shield it from eviction. +func activeInInventory(entry state.Entry, now time.Time) bool { + if entry.LastSeenInInventory.IsZero() { + return false + } + return now.Sub(entry.LastSeenInInventory) < inventoryEvictionGrace +} + +// nextCrawlAt returns when a resource should next crawl: immediately when +// never crawled, at the failure retry gate after errors, else at the tier +// interval. +func nextCrawlAt(entry state.Entry, tierInterval time.Duration) time.Time { + if entry.LastCrawled.IsZero() { + return time.Time{} + } + if entry.LastError != "" && !entry.NextCrawl.IsZero() { + return entry.NextCrawl + } + tierGate := entry.LastCrawled.Add(tierInterval) + if !entry.NextCrawl.IsZero() && entry.NextCrawl.After(tierGate) { + return entry.NextCrawl + } + return tierGate +} + +// crawlDue refreshes request-discovered resources on the tier schedule and +// evicts those whose traffic stopped past the whole budget. +func crawlDue(ctx context.Context) { + now := time.Now() + schedule := refreshSchedule() + for key, entry := range state.S.Snapshot() { + if ctx.Err() != nil { + return + } + // Inventory-only entries are managed by crawlConfigured. + if entry.LastRequested.IsZero() { + continue + } + interval, stale := refreshTier(now.Sub(entry.LastRequested), schedule) + if stale { + if activeInInventory(entry, now) { + continue + } + evictStale(key, entry) + continue + } + // Generic files refresh on demand at serve time. + if entry.Kind == kindGeneric { + continue + } + next := nextCrawlAt(entry, interval) + if !next.IsZero() && now.Before(next) { + continue + } + startCrawl(resource{Kind: entry.Kind, Key: key, Path: entry.Path, Root: entry.Root, ReqPath: entry.Path}) + } +} + +// evictStale removes a resource whose requests stopped long enough ago, +// re-checking under the lock so a racing request wins. +func evictStale(key string, entry state.Entry) { + // Deb resources share an archive root across sibling suites; hold that + // lock too, matching crawlResource, so eviction never races an + // in-flight sibling-suite crawl touching the shared pool. + if entry.Kind == string(mirror.RepoDeb) && entry.Root != entry.Path { + defer lockResource("deb-root:" + entry.Root)() + } + defer lockResource(key)() + + current, ok := state.S.Entry(key) + if ok && current.LastRequested.After(entry.LastRequested) { + return + } + target, err := fetch.LocalJoin(cfg.C.OnlineDomain().Root, entry.Path) + if err == nil && entry.Path != "/" && entry.Path != "" { + if err := os.RemoveAll(target); err != nil { + log.WithError(err).WithField("path", target).Warn("Failed to evict stale resource.") + return + } + log.WithFields(log.Fields{"key": key, "path": target}).Info("Evicted stale resource.") + } + state.S.Delete(key) +} diff --git a/server/failure_cache.go b/server/failure_cache.go new file mode 100644 index 0000000..1f854fb --- /dev/null +++ b/server/failure_cache.go @@ -0,0 +1,56 @@ +package server + +import ( + "sync" + "time" +) + +// failureEntry is one remembered upstream failure with its expiry. +type failureEntry struct { + status int + expires time.Time +} + +// failureCache remembers recent upstream failures for generic paths so a +// missing or erroring file does not trigger an upstream request per client. +// It is memory-only; nothing here reaches the state file. +type failureCache struct { + mu sync.Mutex + entries map[string]failureEntry +} + +// fetchFailures is the server's shared failure cache. +var fetchFailures = &failureCache{entries: map[string]failureEntry{}} + +// Hit returns the remembered status for a key, pruning expired entries. +func (c *failureCache) Hit(key string, now time.Time) (int, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok { + return 0, false + } + if !now.Before(e.expires) { + delete(c.entries, key) + return 0, false + } + return e.status, true +} + +// Add remembers a failure until now+ttl. +func (c *failureCache) Add(key string, status int, now time.Time, ttl time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.entries[key] = failureEntry{status: status, expires: now.Add(ttl)} +} + +// Sweep drops expired entries that nobody re-requested. +func (c *failureCache) Sweep(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, key) + } + } +} diff --git a/server/http.go b/server/http.go new file mode 100644 index 0000000..4da9c43 --- /dev/null +++ b/server/http.go @@ -0,0 +1,81 @@ +package server + +import ( + "context" + "net" + "net/http" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + log "github.com/sirupsen/logrus" +) + +// shutdownTimeout bounds the graceful shutdown of the listener, after which +// in-flight requests are abandoned. +const shutdownTimeout = 30 * time.Second + +// Server owns the mirror's HTTP listener. It binds from the active +// configuration and rebinds on reload when the listen address moves. +type Server struct { + server *http.Server + addr string +} + +// NewServer builds a mirror HTTP server from the active configuration. +func NewServer() *Server { + s := new(Server) + s.configure() + return s +} + +// configure rebuilds the underlying server from the active configuration. +// A running server is never reconfigured in place; Reload binds a new one. +func (s *Server) configure() { + s.addr = cfg.C.HTTP.ListenAddr() + s.server = &http.Server{ + Addr: s.addr, + Handler: Handler(), + ReadHeaderTimeout: cfg.C.HTTP.ReadHeaderTimeout, + ReadTimeout: cfg.C.HTTP.ReadTimeout, + WriteTimeout: cfg.C.HTTP.WriteTimeout, + IdleTimeout: cfg.C.HTTP.IdleTimeout, + } +} + +// Start binds the listen address and serves in the background. A bind +// failure is returned so the caller can abort startup rather than run +// without a listener. +func (s *Server) Start() error { + ln, err := net.Listen("tcp", s.addr) + if err != nil { + return err + } + log.WithField("addr", s.addr).Info("Mirror server listening.") + go func() { + if err := s.server.Serve(ln); err != nil && err != http.ErrServerClosed { + log.WithError(err).Error("HTTP server failed.") + } + }() + return nil +} + +// Reload rebinds the listener when the active configuration moved the +// listen address, and is a no-op otherwise. The old listener is shut down +// before the new one binds so the address is free. Other HTTP settings are +// read when the server is built, so they take effect on the next rebind. +func (s *Server) Reload() error { + if cfg.C.HTTP.ListenAddr() == s.addr { + return nil + } + s.Shutdown() + s.configure() + return s.Start() +} + +// Shutdown stops accepting connections and waits for in-flight requests, +// giving up after shutdownTimeout. +func (s *Server) Shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + _ = s.server.Shutdown(ctx) +} diff --git a/server/main_test.go b/server/main_test.go new file mode 100644 index 0000000..d7ec33b --- /dev/null +++ b/server/main_test.go @@ -0,0 +1,14 @@ +package server + +import ( + "os" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + fetch.Reload() + os.Exit(m.Run()) +} diff --git a/server/serve.go b/server/serve.go new file mode 100644 index 0000000..3c02443 --- /dev/null +++ b/server/serve.go @@ -0,0 +1,316 @@ +// Package server runs the caching mirror: it answers requests from the +// local tree, fetches misses through the configured upstream mounts, and +// crawls the repositories it discovers in the background so they stay +// current without holding a request open. +package server + +import ( + "context" + "errors" + "fmt" + "html" + "io" + "net/http" + "os" + "path" + "sort" + "strings" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/state" + log "github.com/sirupsen/logrus" +) + +// Handler returns the mirror server's HTTP handler. +func Handler() http.Handler { + return http.HandlerFunc(serveMirror) +} + +// regularFile stats a path, reporting only regular files as present. +func regularFile(name string) (os.FileInfo, bool) { + info, err := os.Stat(name) + if err != nil || !info.Mode().IsRegular() { + return nil, false + } + return info, true +} + +// serveLocal resolves a request path under a domain root, mapping +// directory requests to a cached index.html. +func serveLocal(root, reqPath string, isDir bool) (string, error) { + p := reqPath + if isDir || p == "/" { + p = strings.TrimSuffix(p, "/") + "/index.html" + } + return fetch.LocalJoin(root, p) +} + +// internalFile reports paths that belong to the mirror's own bookkeeping +// and must never be served. +func internalFile(reqPath string) bool { + base := path.Base(reqPath) + return base == fetch.LockFileName || base == fetch.PruneStateName || + strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix) +} + +// fetchUpstream retrieves one file through the mount covering the request +// path and stores it under root. +func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error { + mount, ok := cfg.C.MountFor(reqPath) + if !ok { + return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound) + } + rel := reqPath + if mount.Path != "/" { + rel = strings.TrimPrefix(reqPath, mount.Path) + } + if isDir { + rel = strings.TrimSuffix(rel, "/") + "/" + } + dst, err := serveLocal(root, reqPath, isDir) + if err != nil { + return err + } + // Concurrent on-demand fetches and background crawls can target the + // same destination; fetch.File serializes them by path. + src := fetch.NewSource([]string{mount.Upstream}) + _, err = fetch.File(ctx, src, rel, dst, nil, false, false) + return err +} + +// writeFetchError reports an upstream failure to the client. +func writeFetchError(w http.ResponseWriter, err error) { + if errors.Is(err, fetch.ErrNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + http.Error(w, "upstream fetch failed", http.StatusBadGateway) +} + +// writeCachedFailure reports a remembered upstream failure. +func writeCachedFailure(w http.ResponseWriter, status int) { + if status == http.StatusNotFound { + http.Error(w, "not found", http.StatusNotFound) + return + } + http.Error(w, "upstream fetch failed", http.StatusBadGateway) +} + +// writeIndexesDisabled answers a directory request when directory indexes +// are turned off, confirming the mirror is alive without listing content. +func writeIndexesDisabled(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, "\n\nrepo-sync mirror\n\n"+ + "

repo-sync mirror

\n

This mirror is running. Directory indexes are disabled.

\n"+ + "\n\n") +} + +// writeMountIndex answers the root directory request with a generated index +// of every mount and configured repository, giving clients a navigable +// entry point when no catch-all mount covers the root. +func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) { + seen := map[string]bool{} + var paths []string + add := func(p string) { + if p == "/" || seen[p] { + return + } + seen[p] = true + paths = append(paths, p) + } + for _, mount := range conf.Mounts { + add(mount.Path) + for _, repo := range mount.Repos { + add(repo.Path) + } + } + sort.Strings(paths) + + var b strings.Builder + b.WriteString("\n\nIndex of /\n\n

Index of /

\n
    \n") + for _, p := range paths { + href := html.EscapeString(p + "/") + fmt.Fprintf(&b, "
  • %s
    • \n", href, href) + } + b.WriteString("
\n\n\n") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, b.String()) +} + +// serveMirror handles every mirror request: domain lookup, classification, +// and dispatch to the role-specific handler. +func serveMirror(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + domain, ok := cfg.C.Domain(r.Host) + if !ok { + http.Error(w, "unknown mirror domain", http.StatusNotFound) + return + } + + reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/")) + isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/" + if internalFile(reqPath) { + http.Error(w, "not found", http.StatusNotFound) + return + } + + // Directory requests are answered before role dispatch: a notice page + // when indexes are disabled, and a generated index of the mounts for a + // root that no catch-all mount covers. + if isDir { + if !cfg.C.HTTP.DirectoryIndexes { + writeIndexesDisabled(w) + return + } + if reqPath == "/" { + if _, ok := cfg.C.MountFor("/"); !ok { + writeMountIndex(w, cfg.C) + return + } + } + } + res := classifyRequest(reqPath) + + if domain.Role == cfg.RoleOnline { + handleOnline(w, r, domain, reqPath, isDir, res) + return + } + handleOffline(w, r, domain, reqPath, isDir, res) +} + +// handleOnline serves from the writable cache, fetching and crawling on +// demand. +func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) { + local, err := serveLocal(domain.Root, reqPath, isDir) + if err != nil { + http.Error(w, "bad request path", http.StatusBadRequest) + return + } + now := time.Now() + + // Directory listings are cached pass-through pages with no tracking. + if isDir { + if _, exists := regularFile(local); !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil { + writeFetchError(w, err) + return + } + } + http.ServeFile(w, r, local) + return + } + + if res.Kind == kindGeneric { + // Files under a registered repository keep it alive and are + // fetched on demand until its crawl completes. + if state.S.TouchRepoMembers(reqPath, now) { + if _, exists := regularFile(local); !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + writeFetchError(w, err) + return + } + } + http.ServeFile(w, r, local) + return + } + handleGeneric(w, r, domain, reqPath, res, local, now) + return + } + + // A repository entry point registers the repository and dispatches a + // background crawl on first sight so the whole repository fills in + // behind this response without holding it open. The requested file + // itself is fetched on demand so the client is answered immediately. + // Repositories already crawled only re-dispatch here to retry a failed + // crawl; missing optional files must not trigger a full crawl per + // request. + entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now) + _, exists := regularFile(local) + needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "") + if !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + // A path whose first contact failed was never shown to be a + // repository, so the registration this request created is + // dropped and the scheduler never crawls it. Registrations an + // earlier request established are kept: every entry point of one + // repository shares a key, so a repository serving Release but + // not InRelease would otherwise deregister itself on the miss. + if !tracked { + state.S.Delete(res.Key) + } + writeFetchError(w, err) + return + } + } + if needCrawl { + startCrawl(res) + } + http.ServeFile(w, r, local) +} + +// handleGeneric serves a plain cached file, refreshing stale copies and +// remembering upstream failures. +func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) { + if status, hit := fetchFailures.Hit(res.Key, now); hit { + if _, exists := regularFile(local); !exists { + writeCachedFailure(w, status) + return + } + } else { + // GenericMaxAge bounds how often the upstream is revalidated, + // measured from the last upstream check recorded in state. The + // local file's modification time carries the upstream's + // Last-Modified, so it does not record when the mirror last checked. + entry, tracked := state.S.Entry(res.Key) + _, exists := regularFile(local) + if !exists || !tracked || now.Sub(entry.LastCrawled) > cfg.C.GenericMaxAge { + err := fetchUpstream(r.Context(), domain.Root, reqPath, false) + switch { + case errors.Is(err, fetch.ErrNotFound): + fetchFailures.Add(res.Key, http.StatusNotFound, now, cfg.C.NegativeCacheTTL) + state.S.Delete(res.Key) + if !exists { + writeFetchError(w, err) + return + } + log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.") + case err != nil: + fetchFailures.Add(res.Key, http.StatusBadGateway, now, cfg.C.TransientErrorTTL) + if !exists { + writeFetchError(w, err) + return + } + log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.") + default: + state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now) + // Record the upstream check so GenericMaxAge gates the next + // request. The scheduler skips generic entries, so the retry + // time this also sets is unused. + state.S.MarkCrawled(res.Key, now, nil) + } + } else { + state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now) + } + } + http.ServeFile(w, r, local) +} + +// handleOffline serves the read-only published tree, reading through the +// online cache for anything it does not hold. +func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) { + local, err := serveLocal(domain.Root, reqPath, isDir) + if err != nil { + http.Error(w, "bad request path", http.StatusBadRequest) + return + } + if _, exists := regularFile(local); exists { + http.ServeFile(w, r, local) + return + } + handleOnline(w, r, cfg.C.OnlineDomain(), reqPath, isDir, res) +} diff --git a/server/serve_test.go b/server/serve_test.go new file mode 100644 index 0000000..30c56aa --- /dev/null +++ b/server/serve_test.go @@ -0,0 +1,487 @@ +package server + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/grmrgecko/repo-sync/mirror" + "github.com/grmrgecko/repo-sync/state" +) + +// serverFixture builds an upstream with one repository of each type plus a +// generic file, configures the mirror server against it, and returns the +// mirror's test server and the online root. +func serverFixture(t *testing.T) (*httptest.Server, string, string) { + t.Helper() + + // Upstream content behind a single catch-all mount. + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "almalinux", "9", "BaseOS", "x86_64", "os")) + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + testrepos.BuildArchRepo(t, filepath.Join(www, "archlinux", "core", "os", "x86_64"), "core") + testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "main", "x86_64")) + testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file")) + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + offlineRoot := t.TempDir() + confDir := t.TempDir() + conf := fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s + - domain: offline.test + role: offline + root: %s +mounts: + - path: / + upstream: %s +`, confDir, onlineRoot, offlineRoot, upstream.URL) + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(conf)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + // Crawls dispatched by requests run in the background; drain them so + // they do not race the next test's state reload. + t.Cleanup(WaitCrawls) + return srv, onlineRoot, offlineRoot +} + +// waitFor polls a condition until it holds or the deadline passes; crawls +// dispatched by requests run asynchronously. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// get fetches a mirror URL with an optional Host override. +func get(t *testing.T, srv *httptest.Server, host, path string) (*http.Response, []byte) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil) + if err != nil { + t.Fatal(err) + } + if host != "" { + req.Host = host + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + return resp, body +} + +// TestServeRPMDiscovery verifies a repomd.xml request registers and crawls +// the repository, filling the local tree behind the response. +func TestServeRPMDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + repo := "/almalinux/9/BaseOS/x86_64/os" + resp, body := get(t, srv, "", repo+"/repodata/repomd.xml") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if len(body) == 0 { + t.Fatal("empty repomd.xml response") + } + + // The repository was registered and the background crawl mirrors its + // packages behind the response. + if _, ok := state.S.Entry("rpm:" + repo); !ok { + t.Error("repository not registered in state") + } + var pkgs []string + waitFor(t, "rpm crawl", func() bool { + pkgs, _ = filepath.Glob(filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), "Packages", "*.rpm")) + return len(pkgs) > 0 + }) + + // Package requests are served from the crawled tree. + resp, _ = get(t, srv, "", repo+"/Packages/"+filepath.Base(pkgs[0])) + if resp.StatusCode != http.StatusOK { + t.Errorf("package request status = %d", resp.StatusCode) + } +} + +// TestServeDebDiscovery verifies a Release request crawls the suite with +// pool files under the archive root. +func TestServeDebDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, _ := get(t, srv, "", "/debian/dists/test/Release") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if _, ok := state.S.Entry("deb:/debian/dists/test"); !ok { + t.Error("suite not registered in state") + } + waitFor(t, "deb crawl", func() bool { + _, err := os.Stat(filepath.Join(onlineRoot, "debian", "pool", "main", "h", "hello", "hello_1.0_amd64.deb")) + return err == nil + }) + + // A pool request under the archive root is a repository member. + resp, _ = get(t, srv, "", "/debian/pool/main/h/hello/hello_1.0_amd64.deb") + if resp.StatusCode != http.StatusOK { + t.Errorf("pool request status = %d", resp.StatusCode) + } +} + +// TestServeArchAndApkDiscovery verifies database and index requests crawl +// their repositories. +func TestServeArchAndApkDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, _ := get(t, srv, "", "/archlinux/core/os/x86_64/core.db") + if resp.StatusCode != http.StatusOK { + t.Fatalf("arch status = %d", resp.StatusCode) + } + waitFor(t, "arch crawl", func() bool { + pkgs, _ := filepath.Glob(filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "*.pkg.tar.zst")) + return len(pkgs) > 0 + }) + + resp, _ = get(t, srv, "", "/alpine/v3.24/main/x86_64/APKINDEX.tar.gz") + if resp.StatusCode != http.StatusOK { + t.Fatalf("apk status = %d", resp.StatusCode) + } + waitFor(t, "apk crawl", func() bool { + apks, _ := filepath.Glob(filepath.Join(onlineRoot, "alpine", "v3.24", "main", "x86_64", "*.apk")) + return len(apks) > 0 + }) +} + +// TestServeGeneric verifies plain files proxy through with caching and +// missing files are answered from the negative cache. +func TestServeGeneric(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, body := get(t, srv, "", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("generic = %d %q", resp.StatusCode, body) + } + if _, err := os.Stat(filepath.Join(onlineRoot, "notes", "README.txt")); err != nil { + t.Error("generic file not cached:", err) + } + + resp, _ = get(t, srv, "", "/notes/missing.txt") + if resp.StatusCode != http.StatusNotFound { + t.Errorf("missing file status = %d", resp.StatusCode) + } + if status, hit := fetchFailures.Hit("generic:/notes/missing.txt", time.Now()); !hit || status != http.StatusNotFound { + t.Error("missing file not negative-cached") + } +} + +// TestServeGenericRevalidation verifies GenericMaxAge gates how often a +// cached plain file is revalidated. The upstream file carries an old +// Last-Modified that the cached copy inherits, so the gate is measured from +// the recorded upstream check rather than the local modification time. +func TestServeGenericRevalidation(t *testing.T) { + www := t.TempDir() + name := filepath.Join(www, "notes", "README.txt") + testrepos.WriteFile(t, name, []byte("generic file")) + old := time.Now().Add(-30 * 24 * time.Hour) + if err := os.Chtimes(name, old, old); err != nil { + t.Fatal(err) + } + + var upstreamHits atomic.Int64 + files := http.FileServer(http.Dir(www)) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamHits.Add(1) + files.ServeHTTP(w, r) + })) + t.Cleanup(upstream.Close) + + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +generic_max_age: 6h +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: / + upstream: %s +`, confDir, t.TempDir(), upstream.URL))) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + for i := 0; i < 3; i++ { + resp, body := get(t, srv, "", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("request %d = %d %q", i, resp.StatusCode, body) + } + } + if got := upstreamHits.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 within generic_max_age", got) + } +} + +// TestServeOffline verifies offline domains serve their own tree first and +// read through the online cache otherwise. +func TestServeOffline(t *testing.T) { + srv, _, offlineRoot := serverFixture(t) + + // Not present offline: delegated to online, which proxies upstream. + resp, body := get(t, srv, "offline.test", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("offline read-through = %d %q", resp.StatusCode, body) + } + + // Present offline: served from the offline tree. + testrepos.WriteFile(t, filepath.Join(offlineRoot, "notes", "README.txt"), []byte("published copy")) + resp, body = get(t, srv, "offline.test", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "published copy" { + t.Errorf("offline copy = %d %q", resp.StatusCode, body) + } +} + +// TestServeDirectoryIndexes verifies the generated root index without a +// catch-all mount and the notice page when indexes are disabled. +func TestServeDirectoryIndexes(t *testing.T) { + www := t.TempDir() + testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file")) + upstream := testrepos.ServeDir(t, www) + + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + conf := fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: /notes + upstream: %s/notes + - path: /almalinux + upstream: %s/almalinux + repos: + - path: /almalinux/9/BaseOS/x86_64/os + type: rpm +`, confDir, t.TempDir(), upstream.URL, upstream.URL) + testrepos.WriteFile(t, confPath, []byte(conf)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + // Without a catch-all mount the root serves a generated index linking + // every mount and configured repository. + resp, body := get(t, srv, "", "/") + if resp.StatusCode != http.StatusOK { + t.Fatalf("index status = %d", resp.StatusCode) + } + for _, want := range []string{`"/notes/"`, `"/almalinux/"`, `"/almalinux/9/BaseOS/x86_64/os/"`} { + if !strings.Contains(string(body), want) { + t.Errorf("index missing %s in %q", want, body) + } + } + + // Disabling indexes answers every directory request with the notice. + disabled := strings.Replace(conf, "mounts:", "http:\n directory_indexes: false\nmounts:", 1) + testrepos.WriteFile(t, confPath, []byte(disabled)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + resp, body = get(t, srv, "", "/") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") { + t.Errorf("disabled root = %d %q", resp.StatusCode, body) + } + resp, body = get(t, srv, "", "/notes/") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") { + t.Errorf("disabled subdirectory = %d %q", resp.StatusCode, body) + } +} + +// TestServeRejections verifies unknown domains and internal bookkeeping +// files are refused. +func TestServeRejections(t *testing.T) { + srv, _, _ := serverFixture(t) + + resp, _ := get(t, srv, "unknown.test", "/notes/README.txt") + if resp.StatusCode != http.StatusNotFound { + t.Errorf("unknown domain status = %d", resp.StatusCode) + } + resp, _ = get(t, srv, "", path.Join("/almalinux", fetch.LockFileName)) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("internal file status = %d", resp.StatusCode) + } +} + +// TestPrunableTree verifies a crawl only prunes a tree it wholly owns, so a +// repository at the mirror root cannot delete another mount's cached files. +func TestPrunableTree(t *testing.T) { + serverFixture(t) + + conf := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}, {Path: "/other"}}} + if prunableTree(conf, resource{Key: "rpm:/", Path: "/"}) { + t.Error("a root repository may not prune a tree holding another mount") + } + if !prunableTree(conf, resource{Key: "rpm:/other/el9", Path: "/other/el9"}) { + t.Error("a repository owning its own tree must prune") + } + + // A repository nested under another one blocks the outer one's prune. + single := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}}} + state.S.MarkRequested("rpm", "rpm:/vendor/el9", "/vendor/el9", "/vendor/el9", time.Now()) + if prunableTree(single, resource{Key: "rpm:/vendor", Path: "/vendor"}) { + t.Error("an outer repository may not prune a tree holding a nested one") + } + if !prunableTree(single, resource{Key: "rpm:/vendor/el9", Path: "/vendor/el9"}) { + t.Error("a repository must not be blocked by its own registration") + } +} + +// TestServeRootRepositoryKeepsOtherMounts verifies a repository discovered at +// the mirror root does not prune content another mount cached beside it. +func TestServeRootRepositoryKeepsOtherMounts(t *testing.T) { + // Two upstreams: a repository at its own root, and unrelated content. + repoWWW := t.TempDir() + testrepos.BuildRPMRepo(t, repoWWW) + repoUpstream := testrepos.ServeDir(t, repoWWW) + otherWWW := t.TempDir() + testrepos.WriteFile(t, filepath.Join(otherWWW, "notes.txt"), []byte("other mount content")) + otherUpstream := testrepos.ServeDir(t, otherWWW) + + onlineRoot := t.TempDir() + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: / + upstream: %s + - path: /other + upstream: %s +`, confDir, onlineRoot, repoUpstream.URL, otherUpstream.URL))) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + t.Cleanup(WaitCrawls) + + // Cache a file from the unrelated mount, then crawl the root repository. + if resp, _ := get(t, srv, "", "/other/notes.txt"); resp.StatusCode != http.StatusOK { + t.Fatalf("notes status = %d", resp.StatusCode) + } + cached := filepath.Join(onlineRoot, "other", "notes.txt") + if _, err := os.Stat(cached); err != nil { + t.Fatal("unrelated file not cached:", err) + } + if resp, _ := get(t, srv, "", "/repodata/repomd.xml"); resp.StatusCode != http.StatusOK { + t.Fatalf("repomd status = %d", resp.StatusCode) + } + WaitCrawls() + + if _, err := os.Stat(cached); err != nil { + t.Error("root repository crawl pruned another mount's cached file:", err) + } +} + +// TestServeSiblingEntryPointMiss verifies a repository keeps its +// registration when a sibling entry-point path the upstream does not serve +// is requested. Every entry point of one repository shares a state key, and +// clients probe all of them: apt asks for InRelease, Release, and +// Release.gpg, and dnf asks for signature material beside repomd.xml. +func TestServeSiblingEntryPointMiss(t *testing.T) { + srv, _, _ := serverFixture(t) + + for _, tc := range []struct{ found, missing, key string }{ + {"/debian/dists/test/Release", "/debian/dists/test/InRelease", "deb:/debian/dists/test"}, + { + "/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", + "/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml.key", + "rpm:/almalinux/9/BaseOS/x86_64/os", + }, + } { + if resp, _ := get(t, srv, "", tc.found); resp.StatusCode != http.StatusOK { + t.Fatalf("%s status = %d", tc.found, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); !ok { + t.Fatalf("%s did not register %s", tc.found, tc.key) + } + if resp, _ := get(t, srv, "", tc.missing); resp.StatusCode != http.StatusNotFound { + t.Fatalf("%s status = %d, want 404", tc.missing, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); !ok { + t.Errorf("%s deregistered %s", tc.missing, tc.key) + } + } +} + +// TestServeUnresolvedEntryPoint verifies a path that only looks like a +// repository entry point is not left registered for the scheduler to crawl. +func TestServeUnresolvedEntryPoint(t *testing.T) { + srv, _, _ := serverFixture(t) + + for _, tc := range []struct{ path, key string }{ + {"/nope/repodata/repomd.xml", "rpm:/nope"}, + {"/nope/dists/sid/InRelease", "deb:/nope/dists/sid"}, + {"/nope/database.db", "arch:/nope"}, + {"/nope/" + mirror.APKIndexName, "apk:/nope"}, + } { + resp, _ := get(t, srv, "", tc.path) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s status = %d", tc.path, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); ok { + t.Errorf("%s left %s registered", tc.path, tc.key) + } + } +} diff --git a/server_cmd.go b/server_cmd.go new file mode 100644 index 0000000..495cad5 --- /dev/null +++ b/server_cmd.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/coreos/go-systemd/v22/daemon" + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/server" + "github.com/grmrgecko/repo-sync/state" + "github.com/kardianos/service" + log "github.com/sirupsen/logrus" +) + +// stopChan carries a shutdown request from the service manager. It is +// buffered so a stop delivered after the signal loop exits cannot block the +// service supervisor. +var stopChan = make(chan struct{}, 1) + +// ServerCmd runs the caching mirror server. +type ServerCmd struct { + Bind string `name:"http.bind" help:"Override the configured bind address." optional:""` + Port uint `name:"http.port" help:"Override the configured port." optional:""` +} + +// updateCFG applies the command line overrides onto the active +// configuration. The overridden fields are read only when the listener is +// built, which happens on this goroutine, so a reload can amend them in +// place after Init has published the new configuration. +func (c *ServerCmd) updateCFG() { + if c.Bind != "" { + cfg.C.HTTP.BindAddr = c.Bind + } + if c.Port != 0 { + cfg.C.HTTP.Port = c.Port + } +} + +// Run starts the mirror server: configuration, state, crawl loops, and the +// HTTP listener, then handles signals until shutdown. +func (c *ServerCmd) Run() error { + // main installed the configuration; the server additionally needs the + // domains and mounts the sync commands do without. + c.updateCFG() + if err := cfg.C.RequireServer(); err != nil { + return err + } + + // Load persisted crawl state. + if err := state.Load(); err != nil { + return err + } + + // Start the background loops and the listener. + loopCtx, loopCancel := context.WithCancel(context.Background()) + defer loopCancel() + go state.S.FlushLoop(loopCtx) + crawlDone := make(chan struct{}) + go func() { + defer close(crawlDone) + server.CrawlLoop(loopCtx) + }() + + // stopCrawls cancels the loops and waits for in-flight crawls so their + // outcomes are recorded before the final state flush. The crawl loop + // must return first: a scheduler pass in flight can still dispatch + // crawls, and a WaitGroup Add racing Wait is a misuse panic. + stopCrawls := func() { + loopCancel() + <-crawlDone + server.WaitCrawls() + } + + httpServer := server.NewServer() + if err := httpServer.Start(); err != nil { + return err + } + + // Attach to the service manager when not run interactively, so a stop + // request arrives on stopChan, then report readiness now that the + // listener is bound. + if !service.Interactive() { + svc, err := new(ServiceCmd).service() + if err != nil { + return err + } + go svc.Run() + } + _, _ = daemon.SdNotify(false, daemon.SdNotifyReady) + + // Handle signals: reload on SIGHUP, shut down on SIGINT/SIGTERM or on a + // stop from the service manager. + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) +SigLoop: + for { + select { + case <-stopChan: + break SigLoop + case sig := <-signals: + if sig != syscall.SIGHUP { + break SigLoop + } + log.Info("Reloading configuration.") + if err := cfg.InitServer(flags.ConfigPath); err != nil { + log.WithError(err).Error("Unable to reload configuration; keeping previous values.") + continue + } + applyGlobalFlags() + c.updateCFG() + if err := httpServer.Reload(); err != nil { + // Exiting on a failed rebind must still flush crawl state. + stopCrawls() + if saveErr := state.S.Save(); saveErr != nil { + log.WithError(saveErr).Error("Failed to save state while shutting down.") + } + return err + } + } + } + + // Shut down: stop the listener, stop the loops, then flush state. + log.Info("Shutting down.") + httpServer.Shutdown() + stopCrawls() + return state.S.Save() +} diff --git a/service_cmd.go b/service_cmd.go new file mode 100644 index 0000000..bb30789 --- /dev/null +++ b/service_cmd.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/kardianos/service" +) + +// systemdScript is the unit template used when installing the service. It +// replaces the library default to run as a notify service with automatic +// restart, so systemd only considers the mirror started once the listener +// is bound. +const systemdScript = `[Unit] +Description={{Description}} +ConditionFileIsExecutable={{Path | cmdEscape}} +{{range Dependencies}}{{.}} +{{end}}StartLimitIntervalSec=500 +StartLimitBurst=5 + +[Service] +Type=notify +ExecStart={{Path | cmdEscape}}{{range Arguments}} {{. | cmd}}{{end}} +{{if ChRoot}}RootDirectory={{ChRoot | cmd}} +{{end}}{{if WorkingDirectory}}WorkingDirectory={{WorkingDirectory | cmdEscape}} +{{end}}{{if UserName}}User={{UserName}} +{{end}}{{if ReloadSignal}}ExecReload=/bin/kill -{{ReloadSignal}} "$MAINPID" +{{end}}{{if PIDFile}}PIDFile={{PIDFile | cmd}} +{{end}}{{if OutputFileSupport}}StandardOutput=file:{{LogDirectory}}/{{Name}}.out +StandardError=file:{{LogDirectory}}/{{Name}}.err +{{end}}{{if LimitNOFILE}}LimitNOFILE={{LimitNOFILE}} +{{end}}{{if Restart}}Restart={{Restart}} +{{end}}{{if SuccessExitStatus}}SuccessExitStatus={{SuccessExitStatus}} +{{end}}RestartSec=5 +EnvironmentFile=-/etc/sysconfig/{{Name}} + +{{range EnvVars}}{{.}} +{{end}}[Install] +WantedBy=multi-user.target +` + +// ServiceAction lists the accepted service actions, in the order the Run +// switch handles them. +var ServiceAction = []string{"start", "stop", "status", "restart", "install", "uninstall"} + +// ServiceCmd manages the mirror server as a system service. +type ServiceCmd struct { + Action string `arg:"" enum:"${serviceActions}" help:"${serviceActions}" required:""` +} + +// action returns the requested service action. +func (s *ServiceCmd) action() string { + return s.Action +} + +// Run performs the requested action against the installed service. +func (s *ServiceCmd) Run() (err error) { + svc, err := s.service() + if err != nil { + return err + } + switch s.action() { + case ServiceAction[0]: + err = svc.Start() + case ServiceAction[1]: + err = svc.Stop() + case ServiceAction[2]: + var status service.Status + status, err = svc.Status() + if err == nil { + switch status { + case service.StatusRunning: + fmt.Println("Service is running.") + case service.StatusStopped: + fmt.Println("Service is stopped.") + default: + fmt.Println("Service is in an unknown state.") + } + } + case ServiceAction[3]: + err = svc.Restart() + case ServiceAction[4]: + err = svc.Install() + case ServiceAction[5]: + err = svc.Uninstall() + } + if err != nil { + return err + } + // Status already printed its own result. + if s.action() != ServiceAction[2] { + fmt.Println("Command executed successfully.") + } + return +} + +// service builds the service definition shared by the management actions +// and by the server when it is started by the service manager. +func (s *ServiceCmd) service() (service.Service, error) { + svcConfig := &service.Config{ + Name: cfg.Name, + DisplayName: cfg.DisplayName, + Description: cfg.Description, + Arguments: []string{"server"}, + Dependencies: []string{"After=network.target"}, + Option: service.KeyValue{ + "SystemdScript": systemdScript, + "ReloadSignal": "HUP", + "Restart": "always", + }, + } + return service.New(s, svcConfig) +} + +// Start satisfies service.Interface. The server is already running in the +// foreground by the time the supervisor attaches. +func (s *ServiceCmd) Start(svc service.Service) error { + return nil +} + +// Stop satisfies service.Interface, signalling the server's shutdown. +func (s *ServiceCmd) Stop(svc service.Service) error { + stopChan <- struct{}{} + return nil +} diff --git a/state/state.go b/state/state.go new file mode 100644 index 0000000..010c614 --- /dev/null +++ b/state/state.go @@ -0,0 +1,236 @@ +// Package state persists the mirror server's per-resource crawl tracking +// so discovered repositories survive restarts. +package state + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "gopkg.in/yaml.v3" +) + +// Entry tracks one resource: a discovered or configured repository, or a +// generic cached file. +type Entry struct { + // Kind is the repository format, or "generic" for plain files. + Kind string `yaml:"kind"` + // Path is the request path of the resource (a deb entry uses the + // suite directory). + Path string `yaml:"path"` + // Root is the request path prefix covering every file the resource + // owns; requests below it keep the resource alive. + Root string `yaml:"root,omitempty"` + LastRequested time.Time `yaml:"last_requested,omitempty"` + LastCrawled time.Time `yaml:"last_crawled,omitempty"` + NextCrawl time.Time `yaml:"next_crawl,omitempty"` + LastSeenInInventory time.Time `yaml:"last_seen_in_inventory,omitempty"` + LastError string `yaml:"last_error,omitempty"` +} + +// file is the on-disk YAML layout. +type file struct { + Entries map[string]*Entry `yaml:"entries"` +} + +// Store is the in-memory state with dirty tracking for periodic flushes. +type Store struct { + mu sync.Mutex + file file + dirty bool +} + +// S is the active store, installed by Load. +var S *Store + +// Load reads the state file, treating a missing or empty file as a fresh +// store. Other read failures abort startup so a state file the process +// cannot read is never silently replaced. +func Load() error { + s := &Store{file: file{Entries: map[string]*Entry{}}} + name := cfg.C.StatePath + data, err := os.ReadFile(name) + switch { + case err == nil && len(data) > 0: + if err := yaml.Unmarshal(data, &s.file); err != nil { + return err + } + // A null or absent entries mapping unmarshals over the initialized + // map with a nil one, so the map is restored before any write. + if s.file.Entries == nil { + s.file.Entries = map[string]*Entry{} + } + // Drop null entries a hand-edited file may carry, as lookups + // dereference the stored pointer. + for key, e := range s.file.Entries { + if e == nil { + delete(s.file.Entries, key) + } + } + case err != nil && !errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("read state %s: %w", name, err) + } + S = s + return nil +} + +// MarkRequested records a client request for a resource, creating it if +// needed. It returns a copy of the entry and whether the resource was +// already tracked, so a caller can tell a registration this request created +// from one an earlier request established. +func (s *Store) MarkRequested(kind, key, path, root string, now time.Time) (Entry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, tracked := s.file.Entries[key] + if !tracked { + e = &Entry{} + s.file.Entries[key] = e + } + e.Kind = kind + e.Path = path + e.Root = root + e.LastRequested = now + s.dirty = true + return *e, tracked +} + +// MarkSeenInInventory records that a configured mount still advertises the +// resource, shielding it from eviction. +func (s *Store) MarkSeenInInventory(kind, key, path, root string, now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.file.Entries[key] + if e == nil { + e = &Entry{} + s.file.Entries[key] = e + } + e.Kind = kind + e.Path = path + e.Root = root + e.LastSeenInInventory = now + s.dirty = true +} + +// MarkCrawled records a crawl outcome, scheduling the next attempt sooner +// after failures. +func (s *Store) MarkCrawled(key string, now time.Time, crawlErr error) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.file.Entries[key] + if e == nil { + return + } + e.LastCrawled = now + if crawlErr != nil { + e.LastError = crawlErr.Error() + e.NextCrawl = now.Add(cfg.C.FailureRetryInterval) + } else { + e.LastError = "" + e.NextCrawl = now.Add(cfg.C.RepoCrawlInterval) + } + s.dirty = true +} + +// Delete removes a resource from tracking. +func (s *Store) Delete(key string) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.file.Entries[key]; ok { + delete(s.file.Entries, key) + s.dirty = true + } +} + +// Entry returns a copy of a tracked resource. +func (s *Store) Entry(key string) (Entry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.file.Entries[key] + if !ok { + return Entry{}, false + } + return *e, true +} + +// Snapshot returns a copy of every tracked resource. +func (s *Store) Snapshot() map[string]Entry { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]Entry, len(s.file.Entries)) + for key, e := range s.file.Entries { + out[key] = *e + } + return out +} + +// TouchRepoMembers refreshes the last-requested time of every repository +// whose root covers a request path, keeping repositories alive while their +// files are fetched. It reports whether any repository matched. +func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + matched := false + for _, e := range s.file.Entries { + if e.Kind == "generic" || e.Root == "" { + continue + } + under := reqPath == e.Root || e.Root == "/" || + (len(reqPath) > len(e.Root) && reqPath[:len(e.Root)] == e.Root && reqPath[len(e.Root)] == '/') + if !under { + continue + } + e.LastRequested = now + matched = true + s.dirty = true + } + return matched +} + +// Save writes the state atomically when it has changed. +func (s *Store) Save() error { + s.mu.Lock() + defer s.mu.Unlock() + if !s.dirty { + return nil + } + data, err := yaml.Marshal(s.file) + if err != nil { + return err + } + name := cfg.C.StatePath + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + return err + } + tmp := name + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + return err + } + if err := os.Rename(tmp, name); err != nil { + return err + } + s.dirty = false + return nil +} + +// FlushLoop saves periodically and once more at shutdown. +func (s *Store) FlushLoop(ctx context.Context) { + t := time.NewTicker(cfg.C.StateFlushInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + _ = s.Save() + return + case <-t.C: + _ = s.Save() + // Re-arm from the active config so a reload takes effect. + t.Reset(cfg.C.StateFlushInterval) + } + } +} diff --git a/state/state_test.go b/state/state_test.go new file mode 100644 index 0000000..1d36882 --- /dev/null +++ b/state/state_test.go @@ -0,0 +1,93 @@ +package state + +import ( + "os" + "path/filepath" + "testing" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" +) + +// initConfig installs a minimal configuration with the given state path. +func initConfig(t *testing.T, statePath string) { + t.Helper() + conf := ` +state_path: ` + statePath + ` +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://example.com"} +` + name := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(name, []byte(conf), 0644); err != nil { + t.Fatal(err) + } + if err := cfg.Init(name); err != nil { + t.Fatal(err) + } +} + +// TestLoadMissing verifies a missing state file yields a fresh store. +func TestLoadMissing(t *testing.T) { + initConfig(t, filepath.Join(t.TempDir(), "state.yaml")) + if err := Load(); err != nil { + t.Fatal(err) + } + if len(S.Snapshot()) != 0 { + t.Error("fresh store is not empty") + } +} + +// TestLoadUnreadable verifies read failures other than a missing file are +// reported to the caller. +func TestLoadUnreadable(t *testing.T) { + initConfig(t, t.TempDir()) + if err := Load(); err == nil { + t.Fatal("expected an error for an unreadable state path") + } +} + +// TestLoadNullEntryMap verifies a state file whose entries mapping is null +// yields a writable store. +func TestLoadNullEntryMap(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.yaml") + if err := os.WriteFile(statePath, []byte("entries:\n"), 0644); err != nil { + t.Fatal(err) + } + initConfig(t, statePath) + if err := Load(); err != nil { + t.Fatal(err) + } + S.MarkRequested("rpm", "rpm:/repo", "/repo", "/repo", time.Now()) + if _, ok := S.Entry("rpm:/repo"); !ok { + t.Error("entry not recorded after loading a null entries mapping") + } +} + +// TestLoadNullEntries verifies hand-edited null entries are dropped at load +// so later lookups only see valid entries. +func TestLoadNullEntries(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.yaml") + content := ` +entries: + rpm:/repo: null + deb:/suite: + kind: deb + path: /suite +` + if err := os.WriteFile(statePath, []byte(content), 0644); err != nil { + t.Fatal(err) + } + initConfig(t, statePath) + if err := Load(); err != nil { + t.Fatal(err) + } + if _, ok := S.Entry("rpm:/repo"); ok { + t.Error("null entry survived load") + } + if _, ok := S.Entry("deb:/suite"); !ok { + t.Error("valid entry lost on load") + } + S.TouchRepoMembers("/repo/repodata/repomd.xml", time.Now()) +} diff --git a/sync_cmd.go b/sync_cmd.go new file mode 100644 index 0000000..8db1daf --- /dev/null +++ b/sync_cmd.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/mirror" +) + +// SyncArgs holds the flags and positional arguments shared by every +// repository type. +type SyncArgs struct { + Trim int `help:"Number of leading URL path components to drop when building the destination path." default:"0"` + Flat bool `help:"Place repositories directly in the destination directory without copying the URL path."` + Discover bool `help:"Treat each URL as a directory index and crawl it for repositories."` + Depth int `help:"Maximum directory depth crawled in discovery mode." default:"5"` + Workers int `help:"Number of concurrent download workers; defaults to the configured crawler workers."` + Verify bool `help:"Re-verify checksums of files that already exist locally."` + Prune bool `help:"Delete local files that are no longer part of the repository."` + + PruneGrace time.Duration `help:"Keep files that left the repository for this long before pruning them (e.g. 72h); defaults to the configured crawler prune grace."` + DryRun bool `help:"Report what would be downloaded or pruned without changing the repository."` + + Args []string `arg:"" name:"url" help:"Repository or mirrorlist URLs followed by the destination directory."` +} + +// options validates the shared arguments and builds the mirror options. +func (s *SyncArgs) options(typ mirror.RepoType) (*mirror.Options, error) { + if len(s.Args) < 2 { + return nil, errors.New("expected at least one repository URL and a destination directory") + } + urls := s.Args[:len(s.Args)-1] + for _, raw := range urls { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("invalid repository URL %q", raw) + } + } + dest, err := filepath.Abs(s.Args[len(s.Args)-1]) + if err != nil { + return nil, err + } + if s.Trim < 0 { + return nil, errors.New("trim must not be negative") + } + if s.Depth < 0 { + return nil, errors.New("depth must not be negative") + } + if s.PruneGrace < 0 { + return nil, errors.New("prune grace must not be negative") + } + + // Unset worker and grace flags fall back to the crawler configuration, + // so a config file tunes the sync commands the same way it tunes the + // server's background crawls. + if s.Workers == 0 { + s.Workers = cfg.C.Crawler.Workers + } + if s.PruneGrace == 0 { + s.PruneGrace = cfg.C.Crawler.PruneGrace + } + return &mirror.Options{ + Type: typ, + URLs: urls, + Destination: dest, + Trim: s.Trim, + Flat: s.Flat, + Discover: s.Discover, + DiscoverDepth: s.Depth, + Workers: s.Workers, + Verify: s.Verify, + Prune: s.Prune, + PruneGrace: s.PruneGrace, + DryRun: s.DryRun, + }, nil +} + +// run executes a synchronization with signal-aware cancellation. +func (s *SyncArgs) run(opts *mirror.Options) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + return mirror.Sync(ctx, opts) +} + +// RPMCmd synchronizes RPM repositories. +type RPMCmd struct { + SyncArgs `embed:""` +} + +// Run performs the RPM synchronization. +func (c *RPMCmd) Run() error { + opts, err := c.options(mirror.RepoRPM) + if err != nil { + return err + } + return c.run(opts) +} + +// ArchCmd synchronizes Arch Linux repositories. +type ArchCmd struct { + SyncArgs `embed:""` +} + +// Run performs the Arch Linux synchronization. +func (c *ArchCmd) Run() error { + opts, err := c.options(mirror.RepoArch) + if err != nil { + return err + } + return c.run(opts) +} + +// ApkCmd synchronizes Alpine Linux repositories. +type ApkCmd struct { + SyncArgs `embed:""` +} + +// Run performs the Alpine Linux synchronization. +func (c *ApkCmd) Run() error { + opts, err := c.options(mirror.RepoApk) + if err != nil { + return err + } + return c.run(opts) +} + +// DebCmd synchronizes DEB repositories. +type DebCmd struct { + Component []string `help:"Limit synchronization to these components."` + Arch []string `help:"Limit synchronization to these architectures; include \"source\" to keep source indexes."` + + SyncArgs `embed:""` +} + +// Run performs the DEB synchronization. +func (c *DebCmd) Run() error { + opts, err := c.options(mirror.RepoDeb) + if err != nil { + return err + } + opts.Components = c.Component + opts.Architectures = c.Arch + return c.run(opts) +}