commit 5af1dd8928d2014c3cfc88598328fff89d45618e Author: James Coleman Date: Mon Jul 27 14:16:32 2026 -0500 first commit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9527d4f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,31 @@ +on: + release: + types: [created] + +permissions: + contents: write + packages: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - + name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: '~> v2' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_golang.yaml b/.github/workflows/test_golang.yaml new file mode 100644 index 0000000..01faee4 --- /dev/null +++ b/.github/workflows/test_golang.yaml @@ -0,0 +1,21 @@ +name: Go package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a7f429a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/repo-sync +/dist/ + +*.exe +*.test +*.out +*.prof diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d6d3422 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,61 @@ +# GoReleaser config for repo-sync. +# https://goreleaser.com +# +# CGO is disabled so the binary is fully static (no glibc dependency) and runs +# unmodified across modern Linux distributions. +version: 2 + +project_name: repo-sync + +before: + hooks: + - go mod tidy + - go test ./... + +builds: + - id: repo-sync + main: . + binary: repo-sync + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -X github.com/grmrgecko/repo-sync/config.Version={{ .Version }} + - -X github.com/grmrgecko/repo-sync/config.Commit={{ .ShortCommit }} + - -X github.com/grmrgecko/repo-sync/config.Date={{ .Date }} + - -X github.com/grmrgecko/repo-sync/config.Mode=release + goos: + - linux + goarch: + - "386" + - amd64 + - arm + - arm64 + - ppc64le + goarm: + - "6" + +archives: + - id: default + formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + files: + - README.md + - LICENSE + +checksum: + name_template: "checksums.txt" + +snapshot: + version_template: "{{ incpatch .Version }}-snapshot" + +changelog: + use: git + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..83d4a90 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2026 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5470b7c --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +BINARY := repo-sync +MODULE := github.com/grmrgecko/repo-sync/config +# VERSION is the single source of truth for the version string. COMMIT and DATE +# are derived from git and the build clock. +VERSION ?= $(shell cat VERSION 2>/dev/null || echo dev) +COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) +DATE := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ') +LDFLAGS := -s -w -X $(MODULE).Version=$(VERSION) -X $(MODULE).Commit=$(COMMIT) -X $(MODULE).Date=$(DATE) + +.PHONY: all build test vet fmt snapshot release clean + +all: build + +## build: native static binary into dist/ +build: + CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(BINARY) . + +## test: run the unit tests +test: + go test ./... + +vet: + go vet ./... + +fmt: + gofmt -w . + +## snapshot: local GoReleaser build without publishing (artifacts in dist/) +snapshot: + goreleaser release --snapshot --clean + +## release: full GoReleaser release (CI runs this on a tag) +release: + goreleaser release --clean + +clean: + rm -rf dist diff --git a/README.md b/README.md new file mode 100644 index 0000000..1970e91 --- /dev/null +++ b/README.md @@ -0,0 +1,241 @@ +# repo-sync + +A universal Linux package repository synchronization tool. It mirrors remote +repositories into a local directory tree, preserving the upstream layout so +the result can be served directly to package managers. + +Supported repository types, named by `--type` when a run has to be limited +to some of them: + +- **rpm** — yum/dnf repositories (`repodata/repomd.xml`), including + plain-text mirrorlist and metalink URLs with failover between mirrors. +- **deb** — apt repositories, both standard (`dists/` with a shared + `pool/`) and flat layouts, including `Acquire-By-Hash` population. +- **arch** — pacman repositories (`.db`), including detached package + signatures and companion metadata (`.files`, `.db.tar.gz`, `.links.tar.gz`). +- **apk** — Alpine Linux repositories (`APKINDEX.tar.gz` with packages + beside it). + +## Usage + +``` +repo-sync sync [flags] [ ...] +``` + +The last argument is always the destination directory; every argument before +it is a repository or mirrorlist URL, synchronized one after the other. + +Each repository is identified by its own metadata, so no type has to be +given: an upstream publishing both yum and apt repositories is mirrored in +one run. `--type` limits which formats are accepted, and is repeatable; a +run limited to a single type synchronizes every URL as that type rather than +identifying it, which is what a mirrorlist or metalink URL needs. + +A configuration file is optional: without one the built-in defaults apply. +When one is present its `crawler` section supplies the defaults for every +command, so `workers`, `prune_grace`, `request_timeout`, `user_agent`, +`missing_mode`, `missing_retries`, and `discover_cache` are shared by the +sync commands and the server. The global flags `--config-path`, `--log-level`, `--user-agent`, +and `--request-timeout` override the configuration for any command. + +```sh +# Mirror one repository. The URL path is copied below the destination, so +# this produces ./mirror/repos/CentOS/7/EA4/. +repo-sync sync https://example.com/repos/CentOS/7/EA4/ ./mirror + +# Trim the first two path components: ./mirror/7/EA4/. +repo-sync sync --trim 2 https://example.com/repos/CentOS/7/EA4/ ./mirror + +# No path copying at all: the repository lands directly in ./mirror. +repo-sync sync --flat https://example.com/repos/CentOS/7/EA4/ ./mirror + +# Multiple repositories in one run. +repo-sync sync https://example.com/repos/a/ https://example.com/repos/b/ ./mirror + +# Crawl directory listings for repositories, at most 4 levels deep. +repo-sync sync --discover --depth 4 https://example.com/repos/ ./mirror + +# A pacman repository. The database name is discovered from the directory +# listing, or by probing URL path segments when listings are disabled. +repo-sync sync https://example.com/archlinux/core/os/x86_64/ ./mirror + +# An Alpine repository is one arch directory; discovery syncs every arch +# of a release/repo tree in one run. +repo-sync sync https://dl-cdn.alpinelinux.org/alpine/v3.24/community/x86_64/ ./mirror +repo-sync sync --discover --depth 1 https://dl-cdn.alpinelinux.org/alpine/v3.24/community/ ./mirror + +# A Fedora-style metalink URL; mirrors are used in preference order. Nothing +# can be crawled or probed through a metalink, so the type is given. The +# mirrors' paths differ, so --flat keeps the destination stable. +repo-sync sync --type rpm --flat 'https://mirrors.fedoraproject.org/metalink?repo=epel-9&arch=x86_64' ./mirror/epel9 + +# An apt suite. Pool files resolve against the archive root, producing +# ./mirror/debian/dists/bookworm/ and ./mirror/debian/pool/. +repo-sync sync https://deb.example.com/debian/dists/bookworm ./mirror + +# Limit an apt mirror to specific components and architectures. Include +# "source" as an architecture to keep source indexes. +repo-sync sync --component main --arch amd64,source https://deb.example.com/debian/dists/bookworm ./mirror + +# Limit a crawl to one format when an upstream publishes several. +repo-sync sync --type rpm --discover https://repo.example.com/ ./mirror +``` + +## Discovery + +`--discover` treats each URL as a directory index and crawls it for +repositories, at most `--depth` levels down. A directory holding repository +metadata is mirrored as a whole and is not crawled further, so a vendor +archive is mirrored by pointing one run at its root. + +`--exclude` keeps parts of that tree out of the crawl. Each pattern is a +glob, repeatable, and matched against the entry name wherever it appears in +the tree; a pattern containing a slash is matched against the path below the +crawled URL instead, pinning it to one place, and a leading slash anchors it +to that URL. An excluded directory is never listed, so nothing below it is +mirrored either. + +`--include-file` mirrors loose files the crawl passes on its way, which no +repository's metadata lists — signing keys, release notes, and packages +published outside a repository. Each pattern is a glob, repeatable, and +matched by name or by path on the same rules as `--exclude`, so `'*.rpm'` +takes loose packages from anywhere in the tree while `'/*.rpm'` takes only +those beside the crawled URL itself. Only files in directories that are +*not* repositories are considered: a repository's own files come from its +metadata, and pruning there would remove anything else. These files carry no +published checksum, so they are revalidated against their modification time +rather than re-downloaded, and `--prune` never removes them. + +Crawling a vendor archive costs hundreds of directory listings, so what a +crawl found is cached at the destination root in `.repo-sync-state.json` and +reused until it ages out. `--discover-cache` sets that lifetime, three days +by default; `0` scans on every run, and changing `--depth`, `--type`, +`--exclude`, or `--include-file` scans again regardless, so a new pattern +never waits out the cache. + +The cache is also the record of what the upstream used to publish. A scan +that no longer finds a repository or a loose file it recorded before marks it +withdrawn, and `--prune` then removes it from the mirror once it has been +gone for `--prune-grace`, so a vendor dropping a distribution does not leave +its tree behind forever. A crawl that could not list part of the tree +withdraws nothing: an upstream that failed to answer is not one that dropped +a repository. Without `--prune` the withdrawal is recorded and logged but +nothing is removed. + +```sh +# Mirror MySQL's yum and apt repositories in one run, skipping the +# distributions this mirror does not serve, and keeping the signing keys +# and loose packages published outside the repositories. +repo-sync sync --discover --depth 6 \ + --exclude sles --exclude 'fc*' --exclude docker \ + --include-file '*.rpm' --include-file '*.deb' --include-file 'RPM-GPG-KEY-mysql*' \ + http://repo.mysql.com/ ./mirror +``` + +## Trace files + +A public mirror is expected to say something about itself: who runs it, +where it is, and when it last synchronized. Debian archives established a +convention for this — a file at `project/trace/` inside the archive — +and downstream mirrors and mirror checkers read it. `--trace` publishes one +into every repository synchronized, and the mirror server publishes one +into every repository it crawls when the `trace` section is enabled. + +```sh +repo-sync sync --trace --trace-maintainer 'Jane ' \ + https://example.com/repos/CentOS/7/EA4/ ./mirror +``` + +``` +Mon Jul 27 20:31:58 UTC 2026 +Date: Mon, 27 Jul 2026 20:31:58 +0000 +Date-Started: Mon, 27 Jul 2026 20:29:14 +0000 +Creator: repo-sync 0.1.0 +Running on host: mirror.example.com +Maintainer: Jane +Repository type: rpm +Upstream-mirror: https://example.com/repos/CentOS/7/EA4 +Total bytes received: 5726208 +Total time spent syncing: 164 +Average rate: 34915 B/s +``` + +`Upstream-mirror` records the mirror actually fetched from, which for a +mirrorlist or metalink is the one that answered rather than the list's own +URL — the reason a trace is written here rather than by whatever schedules +the sync, which cannot know that. + +The file is placed in the repository directory, or for apt at the archive +root, where the convention puts it. It is registered as part of the +repository, so `--prune` never removes it even when the repository and the +trace share a directory. `--dry-run` reports what would change without +writing one. + +Everything but the dates and transfer figures comes from configuration, +and each field has a matching flag: `--trace-host` (defaults to the system +hostname), `--trace-maintainer`, `--trace-sponsor`, `--trace-country`, +`--trace-location`, and `--trace-throughput`. Setting them in the `trace` +section of a config file avoids repeating them on every run; `--trace` and +`--no-trace` then switch tracing on or off per run. Fields left unset are +omitted from the file rather than written empty. + +Upstream's own traces are mirrored alongside, so `project/trace/` names +every mirror the content passed through and a reader can follow the chain +back to the archive it originated from. A trace upstream publishes under +this mirror's own host name is skipped: that file describes a different +mirror, and the one written here has to win. Upstreams that publish no +traces, or that do not serve a directory index for them, are not an error; +nothing is copied and the run continues. + +Traces work the same on the mirror server, whose crawls are concurrent: +each crawl accounts for its own transfer rather than reading the shared +counters, so the totals in each trace describe that crawl alone. Every +completed crawl republishes the repository's trace, so the timestamps +track how current the cached copy is. + +## Mirror server + +`repo-sync server` runs a caching mirror in front of upstream repositories. +Repositories are discovered from client requests and then kept synchronized +in the background, so package managers can be pointed straight at it. + +```sh +repo-sync server --config-path ./config.yaml +``` + +Without `--config-path` the config is read from `./config.yaml`, +`~/.config/repo-sync/config.yaml`, or `/etc/repo-sync/config.yaml`. See +[config.example.yaml](config.example.yaml) for a documented configuration +covering the listener, domains, mounts, and crawler tuning. The server +needs at least one domain and one mount; the sync commands do not. + +To run it as a systemd service, install `/etc/repo-sync/config.yaml` and: + +```sh +repo-sync service install +repo-sync service start +``` + +`service` also accepts `stop`, `restart`, `status`, and `uninstall`. The +installed unit runs `repo-sync server` as a notify service, restarts on +failure, and reloads its configuration on `systemctl reload repo-sync`. + +## Building + +```sh +make +``` + +The build stamps the binary with the contents of the `VERSION` file plus the +git commit and build date, shown by `repo-sync --version`. A plain +`go build` also works but reports the version as `dev`. + +## Testing + +```sh +make test +``` + +The suite is hermetic: fixture repositories for every supported format +(rpm, deb, arch, apk) are generated in temp directories and served over +local HTTP, so no network access is required. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..0504dc5 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,154 @@ +# Example repo-sync mirror server configuration. +# +# The server proxies and caches upstream package mirrors. Repositories are +# discovered from client requests: fetching a repository's entry-point +# metadata (repodata/repomd.xml, dists//InRelease, .db, +# APKINDEX.tar.gz) registers it for full synchronization and periodic +# refreshing. Repositories listed under a mount's `repos` are kept +# synchronized regardless of traffic. + +# state_path persists discovered repositories across restarts. When unset +# it defaults to state.yaml next to the config file (./config.yaml keeps +# state in ./state.yaml), or /etc/repo-sync/state.yaml without one. +state_path: /etc/repo-sync/state.yaml +state_flush_interval: 30s + +# repo_crawl_interval is how often known repositories are refreshed while +# they keep being requested. failure_retry_interval schedules the next +# attempt after a failed crawl. crawl_check_interval is the scheduler tick. +repo_crawl_interval: 4h +failure_retry_interval: 1h +crawl_check_interval: 30m + +# generic_max_age bounds how stale a plain cached file (anything outside a +# repository) may be before it is revalidated upstream. Upstream 404s are +# remembered for negative_cache_ttl and transient errors for +# transient_error_ttl so failures are not amplified per client. +generic_max_age: 6h +negative_cache_ttl: 1h +transient_error_ttl: 1m + +http: + bind_addr: "" + port: 8080 + read_header_timeout: 10s + read_timeout: 5m + write_timeout: 30m + idle_timeout: 2m + # directory_indexes enables directory listings: upstream index pages are + # passed through, and when no catch-all "/" mount exists the root serves + # a generated index of the mounts and their configured repositories. + # When false, directory requests answer with a static notice page. + directory_indexes: true + +log: + # Severity threshold: debug, info, warn, error. + level: info + # Formatter: console (text) or json. + type: console + # Outputs may include "console" (stderr) or explicit file paths. + outputs: + - console + +# domains map the request Host header to a filesystem tree. Exactly one +# domain has the online role: its tree is the writable cache that misses +# are fetched into and crawls write to. Offline domains are read-only +# published trees that fall back to the online tree for anything missing, +# and are promoted out-of-band (e.g. by snapshot tooling). +domains: + - domain: mirror.example.com + role: online + root: /var/lib/repo-sync/online + - domain: stage.mirror.example.com + role: offline + root: /var/lib/repo-sync/stage + +# mounts map base request paths onto upstream mirrors; the longest matching +# path wins, and "/" is allowed as a catch-all. A request for +# /almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml resolves through the +# /almalinux mount and registers that path as an RPM repository. Each mount +# may also pin repositories that stay synchronized without client traffic. +mounts: + - path: /almalinux + upstream: https://mirror.example.com/almalinux + repos: + - path: /almalinux/9/BaseOS/x86_64/os + type: rpm + - path: /centos + upstream: https://vault.centos.org + - path: /ubuntu + upstream: https://mirror.example.com/ubuntu + - path: /archlinux + upstream: https://mirror.example.com/archlinux + - path: /alpine + upstream: https://dl-cdn.alpinelinux.org/alpine + +# The crawler section also applies to the sync commands: they use these +# workers, prune_grace, request_timeout, and user_agent unless the +# equivalent flag overrides them. +crawler: + # workers is the number of download workers inside a single crawl; + # concurrent_crawls bounds how many repository crawls run at once. + workers: 4 + concurrent_crawls: 4 + # request_timeout bounds how long an upstream may take to start + # responding to a request (response header timeout). + request_timeout: 2m + # user_agent overrides the default repo-sync/. + user_agent: "" + # prune_grace keeps files that left a repository for this long before + # crawl-time pruning removes them. + prune_grace: 24h + # missing_mode decides what happens when an upstream does not serve a + # package its repository metadata lists, which published repositories are + # not rare in doing: + # retry - mirror the rest of the repository and keep reporting the + # absence as a failure until the same file has been missing for + # missing_retries consecutive runs, after which it is recorded + # in the repository as a known absence and stops failing runs + # fail - stop the repository at the first missing file + # ignore - skip missing files without ever failing + # A repository's own metadata is always required regardless of the mode. + missing_mode: retry + missing_retries: 3 + # discover_cache is how long "sync --discover" reuses a crawl's results + # before scanning the tree again. The results are kept at the destination + # root in .repo-sync-state.json, which is also what tells a later run + # which repositories the upstream has dropped since. Zero scans on every + # run. + discover_cache: 72h + # Backoff steps applied once a repository stops being requested. The + # first tier is repo_crawl_interval; each entry extends the refresh + # interval as the resource ages, and once the total budget elapses the + # resource is evicted from the online cache. + refresh_schedule: + - 12h + - 24h + - 48h + - 72h + - 96h + - 120h + +# The trace section publishes a file describing this mirror into every +# repository the sync commands synchronize and the server crawls, following +# the convention Debian archives established at project/trace/. +# Downstream mirrors and mirror checkers read it to learn who runs a mirror, +# where it is, and when it last synchronized. The traces upstream publishes +# are mirrored beside it, so the directory names every mirror the content +# passed through. It is disabled by default, and every field can be +# overridden with the matching --trace-* flag. +trace: + # Traces are written only when this is enabled, or --trace is passed. + enabled: false + # The name this mirror is traced under, which is also the trace file's + # name. Defaults to the system hostname. + host: "" + # Contact for whoever runs this mirror, in "name " form. + maintainer: "" + # Who provides the hardware or bandwidth, if anyone. + sponsor: "" + # Where the mirror is, for clients choosing a nearby one. + country: "" + location: "" + # How much bandwidth the mirror has available. + throughput: "" diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..007540c --- /dev/null +++ b/config/config.go @@ -0,0 +1,513 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "time" + + "github.com/go-playground/validator/v10" + "github.com/spf13/viper" +) + +// Domain roles: the online domain is the writable upstream cache, offline +// domains are read-only published trees that fall back to the online tree. +const ( + RoleOnline = "online" + RoleOffline = "offline" +) + +// HTTPConfig configures the mirror server listener. +type HTTPConfig struct { + BindAddr string `mapstructure:"bind_addr" yaml:"bind_addr"` + Port uint `mapstructure:"port" yaml:"port" validate:"gt=0,lte=65535"` + ReadHeaderTimeout time.Duration `mapstructure:"read_header_timeout" yaml:"read_header_timeout" validate:"gt=0"` + ReadTimeout time.Duration `mapstructure:"read_timeout" yaml:"read_timeout" validate:"gt=0"` + WriteTimeout time.Duration `mapstructure:"write_timeout" yaml:"write_timeout" validate:"gt=0"` + IdleTimeout time.Duration `mapstructure:"idle_timeout" yaml:"idle_timeout" validate:"gt=0"` + // DirectoryIndexes enables directory listings: upstream index pages are + // passed through, and the root serves a generated index of the mounts + // when no catch-all mount covers it. When disabled, every directory + // request answers with a static notice page instead. + DirectoryIndexes bool `mapstructure:"directory_indexes" yaml:"directory_indexes"` +} + +// ListenAddr renders the bind address and port for net.Listen, bracketing +// IPv6 addresses as required. +func (h HTTPConfig) ListenAddr() string { + return net.JoinHostPort(h.BindAddr, strconv.Itoa(int(h.Port))) +} + +// LogConfig configures the server's logging output. +type LogConfig struct { + Level string `mapstructure:"level" yaml:"level" validate:"oneof=debug info warn error"` + Type string `mapstructure:"type" yaml:"type" validate:"oneof=console json"` + Outputs []string `mapstructure:"outputs" yaml:"outputs" validate:"required,min=1,dive,required"` +} + +// DomainConfig maps a request Host header to a filesystem root and role. +type DomainConfig struct { + Domain string `mapstructure:"domain" yaml:"domain" validate:"required"` + Root string `mapstructure:"root" yaml:"root" validate:"required"` + Role string `mapstructure:"role" yaml:"role" validate:"required,oneof=online offline"` +} + +// MountRepoConfig is a repository kept synchronized regardless of client +// traffic, identified by its request path and format. +type MountRepoConfig struct { + Path string `mapstructure:"path" yaml:"path" validate:"required"` + Type string `mapstructure:"type" yaml:"type" validate:"required,oneof=rpm deb arch apk"` +} + +// MountConfig maps a base request path onto an upstream mirror URL. +type MountConfig struct { + Path string `mapstructure:"path" yaml:"path" validate:"required"` + Upstream string `mapstructure:"upstream" yaml:"upstream" validate:"required,url"` + Repos []MountRepoConfig `mapstructure:"repos" yaml:"repos" validate:"omitempty,dive"` +} + +// CrawlerConfig tunes the background synchronization behavior. +type CrawlerConfig struct { + Workers int `mapstructure:"workers" yaml:"workers" validate:"gt=0"` + ConcurrentCrawls int `mapstructure:"concurrent_crawls" yaml:"concurrent_crawls" validate:"gt=0"` + RequestTimeout time.Duration `mapstructure:"request_timeout" yaml:"request_timeout" validate:"gt=0"` + UserAgent string `mapstructure:"user_agent" yaml:"user_agent"` + PruneGrace time.Duration `mapstructure:"prune_grace" yaml:"prune_grace" validate:"gte=0"` + RefreshSchedule []time.Duration `mapstructure:"refresh_schedule" yaml:"refresh_schedule" validate:"required,min=1,dive,gt=0"` + // MissingMode decides what a synchronization does with a package the + // repository metadata lists but the upstream does not serve. The mode + // names are the mirror's own; the fetch package parses them. + MissingMode string `mapstructure:"missing_mode" yaml:"missing_mode" validate:"omitempty,oneof=fail retry ignore"` + // MissingRetries is how many consecutive runs must miss a file before + // the retry mode accepts it as gone. + MissingRetries int `mapstructure:"missing_retries" yaml:"missing_retries" validate:"gte=0"` + // DiscoverCache is how long a discovery crawl's results are reused + // before the tree is scanned again. + DiscoverCache time.Duration `mapstructure:"discover_cache" yaml:"discover_cache" validate:"gte=0"` +} + +// TraceConfig describes the mirror this instance publishes, and is written +// into each synchronized or crawled repository as a trace file when +// enabled. The fields follow the trace convention Debian archives +// established, which downstream mirrors and mirror checkers read to learn +// who runs a mirror, where it is, and when it last synchronized. +type TraceConfig struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled"` + // Host names this mirror in the trace, and is also the trace file's + // name. It defaults to the system hostname. + Host string `mapstructure:"host" yaml:"host"` + Maintainer string `mapstructure:"maintainer" yaml:"maintainer"` + Sponsor string `mapstructure:"sponsor" yaml:"sponsor"` + Country string `mapstructure:"country" yaml:"country"` + Location string `mapstructure:"location" yaml:"location"` + Throughput string `mapstructure:"throughput" yaml:"throughput"` +} + +// HostName returns the name this mirror is traced under, falling back to +// the system hostname when none is configured. The fallback lives here +// rather than in the defaults so it also applies to the built-in +// configuration, which is what a run without a config file reads. +func (t TraceConfig) HostName() string { + if t.Host != "" { + return t.Host + } + if name, err := os.Hostname(); err == nil && name != "" { + return name + } + return "unknown" +} + +// Config is the mirror server configuration. +type Config struct { + HTTP HTTPConfig `mapstructure:"http" yaml:"http"` + Log LogConfig `mapstructure:"log" yaml:"log"` + StatePath string `mapstructure:"state_path" yaml:"state_path" validate:"required"` + StateFlushInterval time.Duration `mapstructure:"state_flush_interval" yaml:"state_flush_interval" validate:"gt=0"` + RepoCrawlInterval time.Duration `mapstructure:"repo_crawl_interval" yaml:"repo_crawl_interval" validate:"gt=0"` + FailureRetryInterval time.Duration `mapstructure:"failure_retry_interval" yaml:"failure_retry_interval" validate:"gt=0"` + CrawlCheckInterval time.Duration `mapstructure:"crawl_check_interval" yaml:"crawl_check_interval" validate:"gt=0"` + GenericMaxAge time.Duration `mapstructure:"generic_max_age" yaml:"generic_max_age" validate:"gt=0"` + NegativeCacheTTL time.Duration `mapstructure:"negative_cache_ttl" yaml:"negative_cache_ttl" validate:"gt=0"` + TransientErrorTTL time.Duration `mapstructure:"transient_error_ttl" yaml:"transient_error_ttl" validate:"gt=0"` + Domains []DomainConfig `mapstructure:"domains" yaml:"domains" validate:"omitempty,dive"` + Mounts []MountConfig `mapstructure:"mounts" yaml:"mounts" validate:"omitempty,dive"` + Crawler CrawlerConfig `mapstructure:"crawler" yaml:"crawler"` + Trace TraceConfig `mapstructure:"trace" yaml:"trace"` + + domainsByName map[string]int + onlineIndex int + mountOrder []int +} + +// C is the active server configuration. A reload builds and validates a +// replacement in full before repointing C at it, so readers always see a +// complete configuration; requests already in flight finish against the +// one they picked up. +var C *Config + +// C starts out holding the built-in defaults so it is never nil. Commands +// that run without a configuration file, and packages exercised by tests +// that never call Init, read those defaults instead of guarding for nil. +func init() { + C = defaults() +} + +// defaults builds a configuration from the registered defaults alone, +// consulting no file. Nothing here can fail: the values are static and the +// indexes start empty, so validation is left to Init. +func defaults() *Config { + v := viper.New() + setDefaults(v) + c := &Config{} + // Unmarshaling static defaults into their own struct cannot fail. + _ = v.Unmarshal(c) + c.StatePath = "/etc/repo-sync/state.yaml" + c.domainsByName = map[string]int{} + c.onlineIndex = -1 + return c +} + +// loadConfig reads and validates a configuration file. Init installs the +// result; callers work with the C singleton rather than loading their own. +func loadConfig(configPath string) (*Config, error) { + v := viper.New() + setDefaults(v) + + file := findConfig(configPath) + if file != "" { + v.SetConfigFile(file) + if err := v.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read config %s: %w", file, err) + } + } + + c := &Config{} + if err := v.Unmarshal(c); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + + // Default the state file next to the config file, so ./config.yaml + // keeps state in ./state.yaml, or to the system path without one. + if c.StatePath == "" { + if file != "" { + c.StatePath = filepath.Join(filepath.Dir(file), "state.yaml") + } else { + c.StatePath = "/etc/repo-sync/state.yaml" + } + } + + if err := c.finalize(); err != nil { + return nil, err + } + return c, nil +} + +// Init loads and validates the configuration, installs it as the active +// configuration, and applies its logging settings. A failed reload leaves +// the previous configuration in place. +func Init(configPath string) error { + c, err := loadConfig(configPath) + if err != nil { + return err + } + install(c) + return nil +} + +// InitServer is Init with the mirror server's additional requirements +// checked before the configuration is installed, so a bad reload leaves +// the running server on the configuration it is already serving. +func InitServer(configPath string) error { + c, err := loadConfig(configPath) + if err != nil { + return err + } + if err := c.RequireServer(); err != nil { + return err + } + install(c) + return nil +} + +// install publishes a configuration and applies the settings derived from +// it. +func install(c *Config) { + C = c + SetupLogging(c.Log) +} + +// findConfig resolves the configuration file path: the explicit flag, the +// working directory, the user config directory, then the system directory. +func findConfig(explicit string) string { + if explicit != "" { + return explicit + } + candidates := []string{"config.yaml"} + if home, err := os.UserHomeDir(); err == nil { + candidates = append(candidates, filepath.Join(home, ".config", "repo-sync", "config.yaml")) + } + candidates = append(candidates, "/etc/repo-sync/config.yaml") + for _, name := range candidates { + if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() { + return name + } + } + return "" +} + +// setDefaults registers every configuration default. +func setDefaults(v *viper.Viper) { + v.SetDefault("state_flush_interval", 30*time.Second) + v.SetDefault("repo_crawl_interval", 4*time.Hour) + v.SetDefault("failure_retry_interval", time.Hour) + v.SetDefault("crawl_check_interval", 30*time.Minute) + v.SetDefault("generic_max_age", 6*time.Hour) + v.SetDefault("negative_cache_ttl", time.Hour) + v.SetDefault("transient_error_ttl", time.Minute) + + v.SetDefault("http.port", 8080) + v.SetDefault("http.directory_indexes", true) + v.SetDefault("http.read_header_timeout", 10*time.Second) + v.SetDefault("http.read_timeout", 5*time.Minute) + v.SetDefault("http.write_timeout", 30*time.Minute) + v.SetDefault("http.idle_timeout", 2*time.Minute) + + v.SetDefault("log.level", "info") + v.SetDefault("log.type", "console") + v.SetDefault("log.outputs", []string{"console"}) + + v.SetDefault("crawler.workers", 4) + v.SetDefault("crawler.concurrent_crawls", 4) + v.SetDefault("crawler.request_timeout", 2*time.Minute) + v.SetDefault("crawler.user_agent", Name+"/"+Version) + v.SetDefault("crawler.prune_grace", 0) + v.SetDefault("crawler.missing_mode", "retry") + v.SetDefault("crawler.missing_retries", 3) + v.SetDefault("crawler.discover_cache", 72*time.Hour) + v.SetDefault("crawler.refresh_schedule", []time.Duration{ + 12 * time.Hour, 24 * time.Hour, 48 * time.Hour, + 72 * time.Hour, 96 * time.Hour, 120 * time.Hour, + }) + + v.SetDefault("trace.enabled", false) +} + +// finalize canonicalizes, validates, and indexes a parsed configuration. +func (c *Config) finalize() error { + c.canonicalize() + if err := validateConfig(c); err != nil { + return err + } + if err := c.indexDomains(); err != nil { + return err + } + return c.indexMounts() +} + +// canonicalize normalizes user-supplied values before validation. +func (c *Config) canonicalize() { + c.StatePath = expandHome(strings.TrimSpace(c.StatePath)) + for i := range c.Domains { + c.Domains[i].Domain = strings.ToLower(strings.TrimSpace(c.Domains[i].Domain)) + c.Domains[i].Role = strings.ToLower(strings.TrimSpace(c.Domains[i].Role)) + c.Domains[i].Root = expandHome(strings.TrimSpace(c.Domains[i].Root)) + } + // Trace values are written verbatim into a published file, so trim them + // and drop anything that would break the one-field-per-line format. + c.Trace.Host = TraceValue(c.Trace.Host) + c.Trace.Maintainer = TraceValue(c.Trace.Maintainer) + c.Trace.Sponsor = TraceValue(c.Trace.Sponsor) + c.Trace.Country = TraceValue(c.Trace.Country) + c.Trace.Location = TraceValue(c.Trace.Location) + c.Trace.Throughput = TraceValue(c.Trace.Throughput) + for i := range c.Mounts { + c.Mounts[i].Path = cleanRequestPath(c.Mounts[i].Path) + c.Mounts[i].Upstream = strings.TrimRight(strings.TrimSpace(c.Mounts[i].Upstream), "/") + for j := range c.Mounts[i].Repos { + c.Mounts[i].Repos[j].Path = cleanRequestPath(c.Mounts[i].Repos[j].Path) + c.Mounts[i].Repos[j].Type = strings.ToLower(strings.TrimSpace(c.Mounts[i].Repos[j].Type)) + } + } +} + +// cleanRequestPath normalizes a request path to a single leading slash with +// no trailing slash. +func cleanRequestPath(p string) string { + return path.Clean("/" + strings.Trim(strings.TrimSpace(p), "/")) +} + +// TraceValue normalizes a trace field for publication. Each field occupies +// one line of the trace file, so embedded newlines are folded to spaces +// rather than trimmed away, which would let a configured value forge +// additional trace fields. +func TraceValue(s string) string { + s = strings.Map(func(r rune) rune { + if r == '\n' || r == '\r' || r == '\t' { + return ' ' + } + // Drop the remaining control characters outright. + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) + return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) +} + +// expandHome resolves a leading ~ against the user's home directory. +func expandHome(p string) string { + if p == "~" || strings.HasPrefix(p, "~/") { + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, strings.TrimPrefix(p, "~")) + } + } + return p +} + +// validateConfig applies the declarative validation rules, reporting +// failures by their YAML key names. +func validateConfig(c *Config) error { + validate := validator.New() + validate.RegisterTagNameFunc(func(field reflect.StructField) string { + name := strings.SplitN(field.Tag.Get("mapstructure"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + err := validate.Struct(c) + if err == nil { + return nil + } + var verrs validator.ValidationErrors + if !errors.As(err, &verrs) { + return err + } + var fields []string + for _, verr := range verrs { + fields = append(fields, fmt.Sprintf("%s (%s)", strings.ToLower(verr.Namespace()), verr.Tag())) + } + return fmt.Errorf("invalid configuration: %s", strings.Join(fields, ", ")) +} + +// indexDomains builds the host lookup and enforces exactly one online +// domain. +func (c *Config) indexDomains() error { + c.domainsByName = map[string]int{} + c.onlineIndex = -1 + for i := range c.Domains { + d := c.Domains[i] + if _, ok := c.domainsByName[d.Domain]; ok { + return fmt.Errorf("domain %s is configured more than once", d.Domain) + } + c.domainsByName[d.Domain] = i + if d.Role == RoleOnline { + if c.onlineIndex != -1 { + return fmt.Errorf("only one online domain may be configured") + } + c.onlineIndex = i + } + } + return nil +} + +// RequireServer reports whether the configuration carries what the mirror +// server needs. Domains and mounts are only meaningful to the server, so +// they are checked here rather than at load; the sync commands run against +// a configuration that has neither. +func (c *Config) RequireServer() error { + if len(c.Domains) == 0 { + return errors.New("at least one domain must be configured") + } + if c.onlineIndex == -1 { + return errors.New("one online domain must be configured") + } + if len(c.Mounts) == 0 { + return errors.New("at least one mount must be configured") + } + return nil +} + +// indexMounts validates mount uniqueness and upstream URLs, orders mounts +// longest path first for prefix lookup, and checks configured repositories +// sit under their mount. +func (c *Config) indexMounts() error { + seen := map[string]bool{} + for i := range c.Mounts { + m := &c.Mounts[i] + if seen[m.Path] { + return fmt.Errorf("mount path %s is configured more than once", m.Path) + } + seen[m.Path] = true + u, err := url.Parse(m.Upstream) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("mount %s upstream must include scheme and host", m.Path) + } + // Request URLs are built by joining a relative path onto the + // upstream; a query or fragment would swallow that path, so reject + // them rather than compose a malformed URL later. + if u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("mount %s upstream must not contain a query or fragment", m.Path) + } + for _, repo := range m.Repos { + if !pathUnder(repo.Path, m.Path) { + return fmt.Errorf("repository %s is not under mount %s", repo.Path, m.Path) + } + } + } + c.mountOrder = make([]int, len(c.Mounts)) + for i := range c.mountOrder { + c.mountOrder[i] = i + } + sort.Slice(c.mountOrder, func(a, b int) bool { + return len(c.Mounts[c.mountOrder[a]].Path) > len(c.Mounts[c.mountOrder[b]].Path) + }) + return nil +} + +// pathUnder reports whether a request path equals or sits below a base +// path. +func pathUnder(p, base string) bool { + if base == "/" { + return true + } + return p == base || strings.HasPrefix(p, base+"/") +} + +// Domain looks up the configuration for a request Host header. The port is +// stripped and IPv6 literals are unbracketed before the lookup. +func (c *Config) Domain(host string) (DomainConfig, bool) { + host = strings.ToLower(strings.TrimSpace(host)) + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.Trim(host, "[]") + i, ok := c.domainsByName[host] + if !ok { + return DomainConfig{}, false + } + return c.Domains[i], true +} + +// OnlineDomain returns the single writable cache domain. +func (c *Config) OnlineDomain() DomainConfig { + return c.Domains[c.onlineIndex] +} + +// MountFor finds the longest mount matching a request path. +func (c *Config) MountFor(reqPath string) (MountConfig, bool) { + for _, i := range c.mountOrder { + if pathUnder(reqPath, c.Mounts[i].Path) { + return c.Mounts[i], true + } + } + return MountConfig{}, false +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..48bd831 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,142 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// writeConfig writes a temp config file and returns its path. +func writeConfig(t *testing.T, content string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(name, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return name +} + +// TestInitExample verifies the shipped example configuration is valid. +func TestInitExample(t *testing.T) { + if err := Init("../config.example.yaml"); err != nil { + t.Fatal(err) + } + if _, ok := C.Domain("mirror.example.com:443"); !ok { + t.Error("online domain not resolvable with a port") + } + if C.OnlineDomain().Domain != "mirror.example.com" { + t.Errorf("online domain = %q", C.OnlineDomain().Domain) + } + mount, ok := C.MountFor("/almalinux/9/BaseOS/x86_64/os") + if !ok || mount.Path != "/almalinux" { + t.Errorf("mount = %+v, %v", mount, ok) + } +} + +// TestStatePathDefault verifies an unset state path lands next to the +// config file. +func TestStatePathDefault(t *testing.T) { + path := writeConfig(t, ` +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://example.com"} +`) + if err := Init(path); err != nil { + t.Fatal(err) + } + want := filepath.Join(filepath.Dir(path), "state.yaml") + if C.StatePath != want { + t.Errorf("StatePath = %q, want %q", C.StatePath, want) + } +} + +// TestMountLongestPrefix verifies the most specific mount wins. +func TestMountLongestPrefix(t *testing.T) { + path := writeConfig(t, ` +state_path: /tmp/state.yaml +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://fallback.example.com"} + - {path: /centos, upstream: "https://centos.example.com"} + - {path: /centos/vault, upstream: "https://vault.example.com"} +`) + if err := Init(path); err != nil { + t.Fatal(err) + } + cases := map[string]string{ + "/centos/vault/7/os": "https://vault.example.com", + "/centos/8/BaseOS": "https://centos.example.com", + "/debian/dists/testy": "https://fallback.example.com", + } + for reqPath, want := range cases { + mount, ok := C.MountFor(reqPath) + if !ok || mount.Upstream != want { + t.Errorf("MountFor(%q) = %q, want %q", reqPath, mount.Upstream, want) + } + } +} + +// TestInitServerRejections verifies invalid server configurations fail +// with clear errors. +func TestInitServerRejections(t *testing.T) { + cases := map[string]string{ + "two online domains": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} + - {domain: b.test, role: online, root: /tmp/b} +mounts: + - {path: /, upstream: "https://example.com"} +`, + "no online domain": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: offline, root: /tmp/a} +mounts: + - {path: /, upstream: "https://example.com"} +`, + "repo outside mount": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} +mounts: + - path: /centos + upstream: "https://example.com" + repos: + - {path: /debian/dists/x, type: deb} +`, + "bad repo type": ` +state_path: /tmp/state.yaml +domains: + - {domain: a.test, role: online, root: /tmp/a} +mounts: + - path: /centos + upstream: "https://example.com" + repos: + - {path: /centos/7/os, type: gentoo} +`, + } + for name, content := range cases { + if err := InitServer(writeConfig(t, content)); err == nil { + t.Errorf("%s: configuration unexpectedly accepted", name) + } + } +} + +// TestInitWithoutServerSections verifies a configuration carrying no +// domains or mounts loads for the sync commands but is refused for the +// server. +func TestInitWithoutServerSections(t *testing.T) { + path := writeConfig(t, "state_path: /tmp/state.yaml\ncrawler:\n workers: 9\n") + if err := Init(path); err != nil { + t.Fatalf("Init rejected a sync-only configuration: %v", err) + } + if C.Crawler.Workers != 9 { + t.Errorf("Crawler.Workers = %d, want 9", C.Crawler.Workers) + } + if err := InitServer(path); err == nil { + t.Error("InitServer accepted a configuration with no domains") + } +} diff --git a/config/info.go b/config/info.go new file mode 100644 index 0000000..b1aed1c --- /dev/null +++ b/config/info.go @@ -0,0 +1,19 @@ +// Package config loads and validates the mirror server configuration, +// publishes it for reload-aware readers, configures logging, and carries +// the application identity and build identifiers. +package config + +// Build identifiers populated at build time via -ldflags. +var ( + Version = "dev" + Commit = "" + Date = "" + Mode = "dev" +) + +// Application identifiers used by the CLI and the upstream user agent. +const ( + Name = "repo-sync" + DisplayName = "Repository Sync" + Description = "Synchronize Linux package repositories into a local directory tree." +) diff --git a/config/log.go b/config/log.go new file mode 100644 index 0000000..008adfa --- /dev/null +++ b/config/log.go @@ -0,0 +1,56 @@ +package config + +import ( + "io" + "os" + + log "github.com/sirupsen/logrus" +) + +// logFiles holds the files opened by the previous SetupLogging call so a +// reload can close them once the new outputs are installed. +var logFiles []*os.File + +// SetupLogging applies the configured log level, formatter, and outputs to +// the global logger. Unknown output paths are treated as files opened for +// appending; failures fall back to stderr so the server always logs. +func SetupLogging(lc LogConfig) { + level, err := log.ParseLevel(lc.Level) + if err != nil { + level = log.InfoLevel + } + log.SetLevel(level) + + if lc.Type == "json" { + log.SetFormatter(&log.JSONFormatter{}) + } else { + log.SetFormatter(&log.TextFormatter{FullTimestamp: true}) + } + + var writers []io.Writer + var files []*os.File + for _, output := range lc.Outputs { + if output == "console" { + writers = append(writers, os.Stderr) + continue + } + f, err := os.OpenFile(expandHome(output), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + log.WithError(err).WithField("output", output).Warn("Unable to open log output; skipping.") + continue + } + writers = append(writers, f) + files = append(files, f) + } + if len(writers) == 0 { + writers = append(writers, os.Stderr) + } + log.SetOutput(io.MultiWriter(writers...)) + + // Close the previous reload's files only after the new writer is + // installed; the logger serializes writes, so none remain in flight. + for _, f := range logFiles { + f.Close() + } + logFiles = files +} diff --git a/config/trace_test.go b/config/trace_test.go new file mode 100644 index 0000000..6579353 --- /dev/null +++ b/config/trace_test.go @@ -0,0 +1,83 @@ +package config + +import ( + "os" + "strings" + "testing" +) + +// TestTraceValue verifies trace fields are reduced to a single line, so a +// configured value cannot forge additional trace fields. +func TestTraceValue(t *testing.T) { + cases := []struct{ in, want string }{ + {" Example Inc ", "Example Inc"}, + {"a\nb", "a b"}, + {"a\r\nb", "a b"}, + {"a\tb", "a b"}, + {"name ", "name "}, + {"Maintainer\nCountry: XX", "Maintainer Country: XX"}, + {"has\x00null", "hasnull"}, + {" ", ""}, + } + for _, c := range cases { + if got := TraceValue(c.in); got != c.want { + t.Errorf("TraceValue(%q) = %q, want %q", c.in, got, c.want) + } + } + // Whatever the input, the result never spans lines. + for _, in := range []string{"a\nb\nc", "x\r\ny", "p\tq"} { + if strings.ContainsAny(TraceValue(in), "\r\n") { + t.Errorf("TraceValue(%q) still contains a line break", in) + } + } +} + +// TestTraceHostName verifies the host falls back to the system hostname +// when none is configured. +func TestTraceHostName(t *testing.T) { + if got := (TraceConfig{Host: "mirror.example.com"}).HostName(); got != "mirror.example.com" { + t.Errorf("configured host = %q", got) + } + want, err := os.Hostname() + if err != nil || want == "" { + t.Skip("system hostname unavailable") + } + if got := (TraceConfig{}).HostName(); got != want { + t.Errorf("fallback host = %q, want %q", got, want) + } +} + +// TestTraceConfigLoads verifies the trace section is read and normalized, +// and that tracing stays off unless it is asked for. +func TestTraceConfigLoads(t *testing.T) { + path := writeConfig(t, ` +trace: + enabled: true + host: mirror.example.com + maintainer: "Test Maintainer " + sponsor: "Example Inc" + country: US +`) + if err := Init(path); err != nil { + t.Fatal(err) + } + if !C.Trace.Enabled { + t.Error("trace not enabled") + } + if C.Trace.Host != "mirror.example.com" { + t.Errorf("host = %q", C.Trace.Host) + } + if C.Trace.Maintainer != "Test Maintainer " { + t.Errorf("maintainer = %q, want internal whitespace collapsed", C.Trace.Maintainer) + } + if C.Trace.Location != "" || C.Trace.Throughput != "" { + t.Error("unset trace fields should stay empty") + } + + if err := Init(writeConfig(t, "log:\n level: info\n")); err != nil { + t.Fatal(err) + } + if C.Trace.Enabled { + t.Error("tracing must default to off") + } +} diff --git a/fetch/fetch.go b/fetch/fetch.go new file mode 100644 index 0000000..5b79c43 --- /dev/null +++ b/fetch/fetch.go @@ -0,0 +1,798 @@ +// Package fetch downloads repository files over HTTP with mirror failover, +// verifying published sizes and checksums, staging updates so a live tree +// is never served a partially updated repository, and pruning files the +// upstream no longer lists. +package fetch + +import ( + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "hash/fnv" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/klauspost/compress/zstd" + log "github.com/sirupsen/logrus" + "github.com/ulikunitz/xz" +) + +// client is the shared HTTP client used for all repository fetches. It is +// atomic because a SIGHUP reload swaps it while requests are in flight. +var client atomic.Pointer[http.Client] + +func init() { + // Build a client immediately so the package is usable without an + // explicit Reload. The config package is initialized first, so its + // defaults are already installed. + Reload() +} + +// Reload rebuilds the shared HTTP client from the active crawler +// configuration. A header timeout is used instead of a whole-request +// timeout so large package downloads are not cut short. +func Reload() { + old := client.Load() + client.Store(&http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: cfg.C.Crawler.RequestTimeout, + }, + }) + // Release the replaced transport's idle pool; in-flight requests on it + // finish undisturbed. + if old != nil { + old.CloseIdleConnections() + } +} + +// ErrNotFound marks an upstream file that does not exist, letting callers +// treat missing optional files differently from transport failures. +var ErrNotFound = errors.New("upstream file not found") + +// ErrNotModified marks a conditional request the upstream answered with +// 304, meaning the local copy is already current. +var ErrNotModified = errors.New("upstream file not modified") + +// StagedSuffix is appended to staged downloads until they are promoted. +const StagedSuffix = ".staged" + +// PartialSuffix is appended to in-progress downloads. Partials of files +// with published checksums survive failures so the next run can resume. +const PartialSuffix = ".partial" + +// LockFileName is the flock target guarding a destination directory. +const LockFileName = ".repo-sync.lock" + +// PruneStateName is the prune grace bookkeeping file kept at the root of a +// pruned tree. +const PruneStateName = ".repo-sync-prune.json" + +// DiscoverStateName is the discovery cache kept at the root of a +// destination tree, recording what each crawl found and when. +const DiscoverStateName = ".repo-sync-state.json" + +// Expect describes the size and checksums a fetched file must match. A size +// of -1 means the size is unknown, and Sums maps lowercase algorithm names +// to hex digests. +type Expect struct { + Size int64 + Sums map[string]string +} + +// MakeExpect builds an Expect from a size and a single checksum, treating +// non-positive sizes and empty checksums as unknown. +func MakeExpect(size int64, algo, digest string) *Expect { + e := &Expect{Size: size, Sums: map[string]string{}} + if size <= 0 { + e.Size = -1 + } + if algo != "" && digest != "" { + e.Sums[strings.ToLower(algo)] = strings.ToLower(digest) + } + return e +} + +// hashOrder lists checksum algorithms from most to least preferred. +var hashOrder = []string{"sha512", "sha384", "sha256", "sha224", "sha1", "sha", "md5", "md5sum"} + +// newHasher returns a hash implementation for a checksum algorithm name as +// found in repository metadata, or nil when the algorithm is unknown. +func newHasher(algo string) hash.Hash { + switch strings.ToLower(algo) { + case "md5", "md5sum": + return md5.New() + case "sha", "sha1": + return sha1.New() + case "sha224": + return sha256.New224() + case "sha256": + return sha256.New() + case "sha384": + return sha512.New384() + case "sha512": + return sha512.New() + } + return nil +} + +// bestSum picks the strongest checksum available in the expectation. +func (e *Expect) bestSum() (string, string) { + for _, algo := range hashOrder { + if digest, ok := e.Sums[algo]; ok && digest != "" { + return algo, digest + } + } + return "", "" +} + +// FileState reports what File did with a file. It is only meaningful when +// the accompanying error is nil. +type FileState int + +const ( + // FileMissing marks an optional file the upstream does not serve. + FileMissing FileState = iota + // FileUnchanged marks an existing local file that matched expectations. + FileUnchanged + // FileFetched marks a file downloaded into its final location. + FileFetched + // FileStaged marks a file downloaded next to its final location for a + // later promote. + FileStaged +) + +// verifyLocal reports whether an existing regular file already matches the +// expected size and, when checkHash is set, the expected checksum. +func verifyLocal(name string, want *Expect, checkHash bool) bool { + info, err := os.Stat(name) + if err != nil || !info.Mode().IsRegular() { + return false + } + if want.Size >= 0 && info.Size() != want.Size { + return false + } + if !checkHash { + return true + } + algo, digest := want.bestSum() + if digest == "" { + return true + } + sum, err := hashFile(name, algo) + if err != nil { + return false + } + return sum == digest +} + +// hashFile computes the named checksum of a file, returned as lowercase hex. +func hashFile(name, algo string) (string, error) { + h := newHasher(algo) + if h == nil { + return "", fmt.Errorf("unsupported checksum algorithm %q", algo) + } + f, err := os.Open(name) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// fileStripes bounds the destination lock table. Downloads to the same +// destination are serialized by stripe; a collision between unrelated +// destinations only delays a download briefly. +const fileStripes = 512 + +// fileLocks serializes downloads by destination path. The mirror server's +// on-demand fetches and background crawls can target the same file +// concurrently, and their partial downloads must not interleave. +var fileLocks [fileStripes]sync.Mutex + +// lockFile holds the lock for a destination path and returns its unlock +// function. +func lockFile(dst string) func() { + h := fnv.New32a() + _, _ = io.WriteString(h, dst) + m := &fileLocks[h.Sum32()%fileStripes] + m.Lock() + return m.Unlock +} + +// File downloads reqPath from the source into dst, verifying size and +// checksum when known. Existing files that already match are left alone. +// With stage set the download is written next to dst for a later promote so +// readers of a live tree never see a partially updated repository. +func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting bool) (FileState, error) { + // Serialize writers to this destination so concurrent callers, such + // as the mirror server's on-demand fetches and background crawls, + // cannot interleave partial downloads of the same file. + unlock := lockFile(dst) + defer unlock() + + // Clear any staged leftover from an interrupted run so a later promote + // can never publish stale content. + if stage { + _ = os.Remove(dst + StagedSuffix) + } + + // Skip files that already match the expected content. + if want != nil && verifyLocal(dst, want, verifyExisting) { + recordUnchanged(ctx) + return FileUnchanged, nil + } + + // Checksummed files with a known size can resume a failed download's + // partial file with a ranged request. + var resumeFrom int64 + if want != nil && want.Size > 0 { + if _, digest := want.bestSum(); digest != "" { + if info, err := os.Stat(dst + PartialSuffix); err == nil && info.Size() > 0 && info.Size() < want.Size { + resumeFrom = info.Size() + } + } + } + + for { + state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom) + if err != nil && resumeFrom > 0 && !errors.Is(err, ErrNotFound) && ctx.Err() == nil { + // The partial may not be a prefix of the current upstream file; + // retry once from scratch. + log.WithError(err).WithField("path", reqPath).Debug("Resumed download failed; retrying in full.") + _ = os.Remove(dst + PartialSuffix) + resumeFrom = 0 + continue + } + return state, err + } +} + +// download performs one transfer attempt for File, appending to the +// partial file when resuming from a prior failure. +func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64) (FileState, error) { + // Files without published checksums cannot be verified locally, so an + // existing copy is revalidated with a conditional request instead of + // being re-downloaded every run. + var modifiedSince time.Time + if want == nil { + if info, err := os.Stat(dst); err == nil && info.Mode().IsRegular() { + modifiedSince = info.ModTime() + } + } + + // Request the file, failing over between mirrors as needed. + resp, err := src.Get(ctx, reqPath, GetOptions{ModifiedSince: modifiedSince, RangeFrom: resumeFrom}) + if errors.Is(err, ErrNotModified) { + recordUnchanged(ctx) + log.WithField("path", reqPath).Debug("Kept unchanged repository file.") + return FileUnchanged, nil + } + if err != nil { + return FileMissing, err + } + defer resp.Body.Close() + + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return FileMissing, err + } + + var hasher hash.Hash + var algo, digest string + if want != nil { + algo, digest = want.bestSum() + if digest != "" { + hasher = newHasher(algo) + } + } + // Partials are only kept for files whose content the next attempt can + // verify by checksum. + keepPartial := digest != "" && want.Size > 0 + + // A 206 continues the partial file, whose existing bytes must feed the + // hash first; any other success replaces it from the start. + partial := dst + PartialSuffix + resumed := resumeFrom > 0 && resp.StatusCode == http.StatusPartialContent + openFlags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC + if resumed { + openFlags = os.O_CREATE | os.O_WRONLY | os.O_APPEND + if hasher != nil { + pf, err := os.Open(partial) + if err != nil { + return FileMissing, err + } + _, err = io.Copy(hasher, pf) + pf.Close() + if err != nil { + return FileMissing, err + } + } + log.WithFields(log.Fields{"path": reqPath, "offset": resumeFrom}).Debug("Resuming partial download.") + } + + // Stream the body into the partial file so a crash never leaves an + // incomplete file at the final location. + f, err := os.OpenFile(partial, openFlags, 0644) + if err != nil { + return FileMissing, err + } + w := io.Writer(f) + if hasher != nil { + w = io.MultiWriter(f, hasher) + } + written, copyErr := io.Copy(w, resp.Body) + closeErr := f.Close() + if copyErr != nil { + if !keepPartial { + _ = os.Remove(partial) + } + return FileMissing, fmt.Errorf("download %s: %w", reqPath, copyErr) + } + if closeErr != nil { + if !keepPartial { + _ = os.Remove(partial) + } + return FileMissing, closeErr + } + + // Verify the transfer against the published size and checksum. A + // mismatch invalidates the partial file entirely. + total := written + if resumed { + total += resumeFrom + } + if want != nil && want.Size >= 0 && total != want.Size { + _ = os.Remove(partial) + return FileMissing, fmt.Errorf("download %s: size %d does not match expected %d", reqPath, total, want.Size) + } + if hasher != nil { + if sum := hex.EncodeToString(hasher.Sum(nil)); sum != digest { + _ = os.Remove(partial) + return FileMissing, fmt.Errorf("download %s: %s checksum %s does not match expected %s", reqPath, algo, sum, digest) + } + } + + // Move the download into place, or next to it when staging. + final := dst + state := FileFetched + if stage { + final = dst + StagedSuffix + state = FileStaged + } + if err := os.Rename(partial, final); err != nil { + _ = os.Remove(partial) + return FileMissing, err + } + + // Carry the upstream modification time so later runs can revalidate + // with conditional requests. Promotion preserves it. + if lm, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil { + _ = os.Chtimes(final, time.Now(), lm) + } + recordFetched(ctx, written) + log.WithField("path", reqPath).Debug("Fetched repository file.") + return state, nil +} + +// PlanJobs records what a real run would download without transferring +// anything, still marking every destination as part of the repository so a +// dry-run prune report stays accurate. Optional files the upstream may not +// serve are counted, so the result is an upper bound. +func PlanJobs(jobs []Job, verifyExisting bool, keep *KeepSet) { + for _, job := range jobs { + keep.Add(job.Dst) + if job.Want != nil && verifyLocal(job.Dst, job.Want, verifyExisting) { + Stats.Unchanged.Add(1) + continue + } + Stats.Planned.Add(1) + if job.Want != nil && job.Want.Size > 0 { + Stats.PlannedBytes.Add(job.Want.Size) + } + log.WithField("path", job.ReqPath).Debug("Would fetch repository file.") + } +} + +// RemoveStale deletes a local file the upstream no longer serves, so an +// outdated copy is never served beside refreshed metadata. Missing files +// are not an error. +func RemoveStale(name string) { + if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", name).Warn("Failed to remove stale file.") + } +} + +// StagedOrFinal returns the staged copy of dst when one exists, falling +// back to the promoted file for runs where the upstream was unchanged. +func StagedOrFinal(dst string) string { + staged := dst + StagedSuffix + if _, err := os.Stat(staged); err == nil { + return staged + } + return dst +} + +// PromoteStaged moves a staged download into its final location. Missing +// staged files are not an error, as unchanged files are never staged. +func PromoteStaged(dst string) error { + staged := dst + StagedSuffix + if _, err := os.Stat(staged); errors.Is(err, fs.ErrNotExist) { + return nil + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + return os.Rename(staged, dst) +} + +// PromoteOrDiscard finishes a staged file: promoted normally, or removed in +// a dry run so no published repository state changes. +func PromoteOrDiscard(dst string, dryRun bool) error { + if dryRun { + _ = os.Remove(dst + StagedSuffix) + return nil + } + return PromoteStaged(dst) +} + +// Job describes one file for Many to download. +type Job struct { + ReqPath string + Dst string + Want *Expect + Optional bool + Stage bool +} + +// Many downloads jobs with a fixed worker pool, recording each job's +// resulting state. The first hard failure aborts the remaining work. Files +// the upstream does not serve abort it too, unless the job is optional or +// miss tolerates the absence. +func Many(ctx context.Context, src *Source, jobs []Job, workers int, verifyExisting bool, keep *KeepSet, miss *Missing) ([]FileState, error) { + states := make([]FileState, len(jobs)) + indexes := make(chan int) + errs := make(chan error, 1) + var wg sync.WaitGroup + + if workers < 1 { + workers = 1 + } + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for idx := range indexes { + job := jobs[idx] + state, err := File(ctx, src, job.ReqPath, job.Dst, job.Want, job.Stage, verifyExisting) + if errors.Is(err, ErrNotFound) { + if job.Optional { + log.WithField("path", job.ReqPath).Debug("Skipped file the upstream does not serve.") + continue + } + // A file the metadata lists is expected to exist, so its + // absence is counted and reported however it is handled. + recordMissing(ctx) + if miss.Tolerate(job.ReqPath) { + continue + } + } + if err != nil { + select { + case errs <- err: + default: + } + return + } + states[idx] = state + keep.Add(job.Dst) + } + }() + } + + for idx := range jobs { + select { + case err := <-errs: + close(indexes) + wg.Wait() + return states, err + case <-ctx.Done(): + close(indexes) + wg.Wait() + return states, ctx.Err() + case indexes <- idx: + } + } + close(indexes) + wg.Wait() + + select { + case err := <-errs: + return states, err + default: + return states, nil + } +} + +// KeepSet tracks the files that belong to the current sync so pruning can +// remove everything else. A nil KeepSet ignores all operations. +type KeepSet struct { + mu sync.Mutex + paths map[string]bool +} + +// NewKeepSet returns a tracking set when pruning is enabled, or nil so all +// tracking becomes a no-op. +func NewKeepSet(enabled bool) *KeepSet { + if !enabled { + return nil + } + return &KeepSet{paths: map[string]bool{}} +} + +// Add records a local file as part of the repository. +func (k *KeepSet) Add(name string) { + if k == nil { + return + } + abs, err := filepath.Abs(name) + if err != nil { + return + } + k.mu.Lock() + k.paths[abs] = true + k.mu.Unlock() +} + +// has reports whether a local file is part of the repository. +func (k *KeepSet) has(name string) bool { + if k == nil { + return false + } + abs, err := filepath.Abs(name) + if err != nil { + return false + } + k.mu.Lock() + defer k.mu.Unlock() + return k.paths[abs] +} + +// loadPruneState reads the grace bookkeeping for a pruned tree, mapping +// root-relative paths to when they were first seen unreferenced. +func loadPruneState(root string) map[string]time.Time { + state := map[string]time.Time{} + data, err := os.ReadFile(filepath.Join(root, PruneStateName)) + if err != nil { + return state + } + if err := json.Unmarshal(data, &state); err != nil { + log.WithError(err).WithField("path", root).Warn("Failed to parse prune state; starting fresh.") + return map[string]time.Time{} + } + return state +} + +// savePruneState persists the grace bookkeeping, or removes it when no +// files are waiting out their grace period. +func savePruneState(root string, state map[string]time.Time) { + name := filepath.Join(root, PruneStateName) + if len(state) == 0 { + _ = os.Remove(name) + return + } + data, err := json.Marshal(state) + if err == nil { + err = os.WriteFile(name, data, 0644) + } + if err != nil { + log.WithError(err).WithField("path", name).Warn("Failed to save prune state.") + } +} + +// PruneTree removes files under root that are not part of the current sync, +// then clears out any directories left empty. A grace period defers +// deletion until a file has been unreferenced across runs for that long, +// so clients holding earlier metadata keep working. Failures are logged +// and do not fail the sync. +func PruneTree(root string, keep *KeepSet, grace time.Duration, dryRun bool) { + if keep == nil { + return + } + var state map[string]time.Time + if grace > 0 { + state = loadPruneState(root) + } + now := time.Now() + + var dirs []string + err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if name != root { + dirs = append(dirs, name) + } + return nil + } + base := filepath.Base(name) + if base == PruneStateName || base == MissingStateName || + base == DiscoverStateName || base == LockFileName { + return nil + } + rel, relErr := filepath.Rel(root, name) + if relErr != nil { + return nil + } + if keep.has(name) { + delete(state, rel) + return nil + } + + // Internal work files are removed regardless of grace; anything + // else waits out the grace period across runs first. + internal := strings.HasSuffix(base, StagedSuffix) || strings.HasSuffix(base, PartialSuffix) + if grace > 0 && !internal { + first, ok := state[rel] + if !ok { + // A dry run records nothing, so report the pending grace + // entry instead of staying silent about the file. + if dryRun { + log.WithField("path", name).Info("Stale file would enter the prune grace period.") + return nil + } + state[rel] = now + log.WithField("path", name).Debug("Stale file entered the prune grace period.") + return nil + } + if now.Sub(first) < grace { + return nil + } + } + if dryRun { + Stats.Pruned.Add(1) + log.WithField("path", name).Info("Would prune stale file.") + return nil + } + if err := os.Remove(name); err != nil { + log.WithError(err).WithField("path", name).Warn("Failed to prune stale file.") + return nil + } + delete(state, rel) + Stats.Pruned.Add(1) + log.WithField("path", name).Debug("Pruned stale file.") + return nil + }) + if err != nil { + log.WithError(err).WithField("path", root).Warn("Failed to walk repository for pruning.") + } + + // Drop bookkeeping for files that no longer exist, then persist it. + if grace > 0 && !dryRun { + for rel := range state { + if _, err := os.Stat(filepath.Join(root, rel)); errors.Is(err, fs.ErrNotExist) { + delete(state, rel) + } + } + savePruneState(root, state) + } + if dryRun { + return + } + + // Remove now-empty directories deepest first. Separator count is the + // true depth, so a parent is always removed after its children even + // when a sibling's child has a longer name. + sep := string(os.PathSeparator) + sort.Slice(dirs, func(i, j int) bool { + return strings.Count(dirs[i], sep) > strings.Count(dirs[j], sep) + }) + for _, dir := range dirs { + _ = os.Remove(dir) + } +} + +// Decompressor wraps a reader with the decompressor matching the file name +// extension, returning an optional close function for the wrapper. Staged +// suffixes are ignored so staged indexes can be parsed before promotion. +func Decompressor(r io.Reader, filename string) (io.Reader, func(), error) { + filename = strings.TrimSuffix(filename, StagedSuffix) + switch { + case strings.HasSuffix(filename, ".gz"): + gz, err := gzip.NewReader(r) + if err != nil { + return nil, nil, err + } + return gz, func() { _ = gz.Close() }, nil + case strings.HasSuffix(filename, ".bz2"): + return bzip2.NewReader(r), nil, nil + case strings.HasSuffix(filename, ".xz"): + xr, err := xz.NewReader(r) + if err != nil { + return nil, nil, err + } + return xr, nil, nil + case strings.HasSuffix(filename, ".zst"): + zr, err := zstd.NewReader(r) + if err != nil { + return nil, nil, err + } + return zr, zr.Close, nil + default: + return r, nil, nil + } +} + +// SniffDecompressor wraps a reader with the decompressor matching its +// leading magic bytes, for formats whose file names carry no extension. +func SniffDecompressor(r io.Reader) (io.Reader, func(), error) { + br := bufio.NewReader(r) + magic, err := br.Peek(6) + if err != nil && !errors.Is(err, io.EOF) { + return nil, nil, err + } + switch { + case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b: + gz, err := gzip.NewReader(br) + if err != nil { + return nil, nil, err + } + return gz, func() { _ = gz.Close() }, nil + case len(magic) >= 3 && magic[0] == 'B' && magic[1] == 'Z' && magic[2] == 'h': + return bzip2.NewReader(br), nil, nil + case len(magic) >= 6 && bytes.Equal(magic[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): + xr, err := xz.NewReader(br) + if err != nil { + return nil, nil, err + } + return xr, nil, nil + case len(magic) >= 4 && bytes.Equal(magic[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}): + zr, err := zstd.NewReader(br) + if err != nil { + return nil, nil, err + } + return zr, zr.Close, nil + default: + return br, nil, nil + } +} + +// LocalJoin resolves a repository-relative path under root, rejecting paths +// that would escape the destination tree. +func LocalJoin(root, rel string) (string, error) { + clean := path.Clean("/" + strings.ReplaceAll(rel, "\\", "/")) + dst := filepath.Join(root, filepath.FromSlash(clean)) + rootClean := filepath.Clean(root) + prefix := rootClean + string(os.PathSeparator) + if rootClean == string(os.PathSeparator) { + prefix = rootClean + } + if dst != rootClean && !strings.HasPrefix(dst, prefix) { + return "", fmt.Errorf("path %q escapes destination %q", rel, root) + } + return dst, nil +} diff --git a/fetch/fetch_test.go b/fetch/fetch_test.go new file mode 100644 index 0000000..b9b3f03 --- /dev/null +++ b/fetch/fetch_test.go @@ -0,0 +1,212 @@ +package fetch + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// resumeFixture serves a distinct-content file and records the Range +// headers of incoming requests. +func resumeFixture(t *testing.T, stripRange bool) (*Source, []byte, *[]string) { + t.Helper() + content := make([]byte, 3000) + for i := range content { + content[i] = byte(i % 251) + } + www := t.TempDir() + testrepos.WriteFile(t, filepath.Join(www, "big.bin"), content) + + var mu sync.Mutex + ranges := &[]string{} + fs := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + *ranges = append(*ranges, r.Header.Get("Range")) + mu.Unlock() + if stripRange { + r.Header.Del("Range") + } + fs.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return NewSource([]string{srv.URL}), content, ranges +} + +// TestFetchFileResume verifies a valid partial file is completed with a +// ranged request. +func TestFetchFileResume(t *testing.T) { + src, content, ranges := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, content[:1000]) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Error("resumed content mismatch") + } + if len(*ranges) != 1 || (*ranges)[0] != "bytes=1000-" { + t.Errorf("ranges = %v, want one bytes=1000- request", *ranges) + } +} + +// TestFetchFileResumeIgnored verifies a server that ignores ranges still +// produces a correct full download. +func TestFetchFileResumeIgnored(t *testing.T) { + src, content, _ := resumeFixture(t, true) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, content[:1000]) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, _ := os.ReadFile(dst) + if !bytes.Equal(got, content) { + t.Error("content mismatch after ignored range") + } +} + +// TestFetchFileResumeBadPartial verifies a partial that is not a prefix of +// the upstream file falls back to a full download. +func TestFetchFileResumeBadPartial(t *testing.T) { + src, content, ranges := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + testrepos.WriteFile(t, dst+PartialSuffix, bytes.Repeat([]byte("X"), 1000)) + state, err := File(context.Background(), src, "big.bin", dst, want, false, false) + if err != nil || state != FileFetched { + t.Fatalf("fetch = %v, %v", state, err) + } + got, _ := os.ReadFile(dst) + if !bytes.Equal(got, content) { + t.Error("content mismatch after bad partial") + } + if len(*ranges) != 2 || (*ranges)[1] != "" { + t.Errorf("ranges = %v, want a ranged then a full request", *ranges) + } +} + +// TestFetchFileConditional verifies files without published checksums are +// revalidated with conditional requests: unchanged upstreams are kept, and +// changed upstreams are fetched again. +func TestFetchFileConditional(t *testing.T) { + www := t.TempDir() + upstream := filepath.Join(www, "meta.tar.gz") + testrepos.WriteFile(t, upstream, []byte("metadata")) + srv := testrepos.ServeDir(t, www) + src := NewSource([]string{srv.URL}) + + dst := filepath.Join(t.TempDir(), "meta.tar.gz") + state, err := File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileStaged { + t.Fatalf("first fetch = %v, %v", state, err) + } + if err := PromoteStaged(dst); err != nil { + t.Fatal(err) + } + + // An unchanged upstream answers the conditional request with 304. + state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileUnchanged { + t.Errorf("unchanged fetch = %v, %v", state, err) + } + + // A newer upstream file must be fetched again. The modification time + // is pushed forward because Last-Modified has one-second resolution. + testrepos.WriteFile(t, upstream, []byte("metadata-v2")) + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(upstream, future, future); err != nil { + t.Fatal(err) + } + state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false) + if err != nil || state != FileStaged { + t.Fatalf("changed fetch = %v, %v", state, err) + } + got, err := os.ReadFile(dst + StagedSuffix) + if err != nil { + t.Fatal(err) + } + if string(got) != "metadata-v2" { + t.Errorf("staged content = %q", got) + } +} + +// TestFetchFileConcurrent verifies concurrent downloads to the same +// destination are serialized: no caller's partial download can interleave +// with another's. +func TestFetchFileConcurrent(t *testing.T) { + src, content, _ := resumeFixture(t, false) + want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}} + + dst := filepath.Join(t.TempDir(), "big.bin") + var wg sync.WaitGroup + errs := make(chan error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := File(context.Background(), src, "big.bin", dst, want, false, true); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + got, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, content) { + t.Error("content mismatch after concurrent downloads") + } +} + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + Reload() + os.Exit(m.Run()) +} + +// TestLocalJoin verifies traversal attempts cannot escape the destination. +func TestLocalJoin(t *testing.T) { + root := t.TempDir() + got, err := LocalJoin(root, "../../etc/passwd") + if err != nil { + t.Fatal(err) + } + want := filepath.Join(root, "etc", "passwd") + if got != want { + t.Errorf("got %q, want %q", got, want) + } + if _, err := LocalJoin(root, "a/../b/c.rpm"); err != nil { + t.Errorf("clean relative path rejected: %v", err) + } + got, err = LocalJoin("/", "etc/passwd") + if err != nil { + t.Fatalf("filesystem root destination rejected: %v", err) + } + if got != "/etc/passwd" { + t.Errorf("got %q, want %q", got, "/etc/passwd") + } +} diff --git a/fetch/missing.go b/fetch/missing.go new file mode 100644 index 0000000..82d9042 --- /dev/null +++ b/fetch/missing.go @@ -0,0 +1,238 @@ +package fetch + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// MissingMode selects what a synchronization does with a file its +// repository metadata lists but the upstream does not serve. Published +// repositories are not always complete: an index can outlive the packages it +// references, and a whole repository is not worth abandoning over one of +// them. +type MissingMode string + +const ( + // MissingFail stops a repository at the first missing file. + MissingFail MissingMode = "fail" + // MissingRetry keeps synchronizing the rest of the repository and fails + // the run until the same file has been missing across enough consecutive + // runs to be accepted as gone. + MissingRetry MissingMode = "retry" + // MissingIgnore skips missing files without ever failing the run. + MissingIgnore MissingMode = "ignore" +) + +// MissingModes lists every supported mode, in the order they escalate from +// tolerant to strict. +var MissingModes = []MissingMode{MissingIgnore, MissingRetry, MissingFail} + +// MissingModeNames lists the supported mode names, for user facing messages. +func MissingModeNames() []string { + names := make([]string, 0, len(MissingModes)) + for _, mode := range MissingModes { + names = append(names, string(mode)) + } + return names +} + +// ParseMissingMode converts a configured mode name into a MissingMode. An +// empty name selects the strict mode, which is what a caller that configured +// none expects. +func ParseMissingMode(name string) (MissingMode, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return MissingFail, nil + } + mode := MissingMode(name) + if !slices.Contains(MissingModes, mode) { + return "", fmt.Errorf("unsupported missing file mode %q; expected one of %s", + name, strings.Join(MissingModeNames(), ", ")) + } + return mode, nil +} + +// MissingStateName is the known-missing bookkeeping file kept in a +// synchronized repository. +const MissingStateName = ".repo-sync-missing.json" + +// missingRecord is one file's absence: when it was first and last missed, +// how many consecutive runs have missed it, and whether it has been accepted +// as permanently gone. +type missingRecord struct { + First time.Time `json:"first"` + Last time.Time `json:"last"` + Runs int `json:"runs"` + Known bool `json:"known"` +} + +// Missing tracks the files an upstream did not serve during one repository +// synchronization and decides whether those absences fail it. Records live +// in the repository itself so an absence that repeats across runs can be +// told apart from one that has only just appeared, which is the difference +// between a package that was withdrawn and an upstream having a bad day. +// +// A nil Missing is the strict mode: nothing is tracked and every absence +// fails the run. +type Missing struct { + mode MissingMode + retries int + root string + dryRun bool + + // mu guards everything below; download workers report concurrently. + mu sync.Mutex + records map[string]*missingRecord + seen map[string]bool + // unresolved counts the files missed this run that are still inside + // their retry budget, and first names one of them for the error. + unresolved int + first string +} + +// NewMissing starts tracking the files an upstream does not serve for a +// repository rooted at dir, loading what earlier runs recorded there. The +// strict mode tracks nothing, so it returns nil and every absence keeps +// failing the run. +func NewMissing(dir string, mode MissingMode, retries int, dryRun bool) *Missing { + if mode != MissingRetry && mode != MissingIgnore { + return nil + } + m := &Missing{ + mode: mode, + retries: retries, + root: dir, + dryRun: dryRun, + records: map[string]*missingRecord{}, + seen: map[string]bool{}, + } + m.load() + return m +} + +// load reads the records an earlier run left in the repository. A missing or +// unreadable file starts fresh: the records only decide how much longer an +// absence is retried, so losing them costs retries rather than correctness. +func (m *Missing) load() { + data, err := os.ReadFile(filepath.Join(m.root, MissingStateName)) + if err != nil { + return + } + records := map[string]*missingRecord{} + if err := json.Unmarshal(data, &records); err != nil { + log.WithError(err).WithField("path", m.root).Warn("Failed to parse the missing file state; starting fresh.") + return + } + // Drop null records a hand-edited file may carry, as every lookup + // dereferences the stored pointer. + for name, rec := range records { + if rec != nil { + m.records[name] = rec + } + } +} + +// Tolerate records that the upstream did not serve reqPath and reports +// whether the synchronization may continue without it. A file inside its +// retry budget is still tolerated for this run, so the rest of the +// repository is mirrored either way; Finish is what reports it as a failure. +func (m *Missing) Tolerate(reqPath string) bool { + if m == nil { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + + // Count each file once per run: a repository may list the same file from + // several indexes, and the budget counts runs rather than references. + if m.seen[reqPath] { + return true + } + m.seen[reqPath] = true + + now := time.Now() + rec := m.records[reqPath] + if rec == nil { + rec = &missingRecord{First: now} + m.records[reqPath] = rec + } + rec.Runs++ + rec.Last = now + + // A file missing for its whole retry budget is accepted as gone and + // stops failing runs from here on. It is still requested every run, so + // an upstream that publishes it again is picked up without intervention. + if rec.Runs >= m.retries { + rec.Known = true + } + if m.mode == MissingIgnore || rec.Known { + log.WithField("path", reqPath).Debug("Skipped a file the upstream does not serve.") + return true + } + + m.unresolved++ + if m.first == "" { + m.first = reqPath + } + log.WithFields(log.Fields{"path": reqPath, "runs": rec.Runs, "retries": m.retries}). + Warn("Upstream does not serve a file its metadata lists.") + return true +} + +// Finish persists what this run observed and reports the absences that have +// not yet been accepted as permanent, which is what fails the repository +// while a missing file is still being retried. +func (m *Missing) Finish() error { + if m == nil { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + m.save() + if m.unresolved == 0 { + return nil + } + return fmt.Errorf("%d file(s) listed by the metadata are missing upstream and will be retried, starting with %s", + m.unresolved, m.first) +} + +// save rewrites the records for the files missed this run, and removes the +// file once nothing is missing. Only files missed this run are kept: +// anything else is served again, or is no longer listed at all, and either +// way its retry budget starts over. +func (m *Missing) save() { + if m.dryRun { + return + } + name := filepath.Join(m.root, MissingStateName) + if len(m.seen) == 0 { + if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", name).Warn("Failed to remove the missing file state.") + } + return + } + kept := make(map[string]*missingRecord, len(m.seen)) + for reqPath := range m.seen { + kept[reqPath] = m.records[reqPath] + } + data, err := json.Marshal(kept) + if err == nil { + err = os.MkdirAll(m.root, 0755) + } + if err == nil { + err = os.WriteFile(name, data, 0644) + } + if err != nil { + log.WithError(err).WithField("path", name).Warn("Failed to save the missing file state.") + } +} diff --git a/fetch/missing_test.go b/fetch/missing_test.go new file mode 100644 index 0000000..d917c70 --- /dev/null +++ b/fetch/missing_test.go @@ -0,0 +1,149 @@ +package fetch + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// readMissingState reads the records a run left in a repository. +func readMissingState(t *testing.T, root string) map[string]*missingRecord { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, MissingStateName)) + if err != nil { + t.Fatal(err) + } + records := map[string]*missingRecord{} + if err := json.Unmarshal(data, &records); err != nil { + t.Fatal(err) + } + return records +} + +// TestParseMissingMode verifies mode names are accepted case insensitively +// and that an unconfigured mode is the strict one. +func TestParseMissingMode(t *testing.T) { + cases := map[string]MissingMode{ + "": MissingFail, + "fail": MissingFail, + "retry": MissingRetry, + " Retry": MissingRetry, + "ignore": MissingIgnore, + } + for name, want := range cases { + got, err := ParseMissingMode(name) + if err != nil || got != want { + t.Errorf("ParseMissingMode(%q) = %q, %v, want %q", name, got, err, want) + } + } + if _, err := ParseMissingMode("skip"); err == nil { + t.Error("expected an unsupported mode name to be rejected") + } +} + +// TestMissingStrictMode verifies the strict mode tracks nothing and +// tolerates nothing, so every absence keeps failing the run. +func TestMissingStrictMode(t *testing.T) { + root := t.TempDir() + m := NewMissing(root, MissingFail, 3, false) + if m != nil { + t.Fatal("the strict mode installed a tracker") + } + if m.Tolerate("pool/gone.deb") { + t.Error("the strict mode tolerated a missing file") + } + if err := m.Finish(); err != nil { + t.Errorf("Finish = %v, want nil", err) + } + if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) { + t.Error("the strict mode wrote a state file") + } +} + +// TestMissingRetryBudget verifies an absence has to repeat across +// consecutive runs before it stops failing them, and that a file the +// upstream serves again starts its budget over. +func TestMissingRetryBudget(t *testing.T) { + root := t.TempDir() + const gone = "pool/main/h/hello/hello_1.0_amd64.deb" + + // The first two runs mirror the rest of the repository but still report + // the absence as a failure. + for run := 1; run <= 2; run++ { + m := NewMissing(root, MissingRetry, 3, false) + if !m.Tolerate(gone) { + t.Fatalf("run %d aborted on a missing file", run) + } + if err := m.Finish(); err == nil { + t.Errorf("run %d reported success while the file was still being retried", run) + } + if rec := readMissingState(t, root)[gone]; rec == nil || rec.Runs != run || rec.Known { + t.Errorf("run %d recorded %+v", run, rec) + } + } + + // The third run exhausts the budget, so the file is accepted as gone. + m := NewMissing(root, MissingRetry, 3, false) + m.Tolerate(gone) + if err := m.Finish(); err != nil { + t.Errorf("run 3 still failed after the retry budget: %v", err) + } + if rec := readMissingState(t, root)[gone]; rec == nil || !rec.Known { + t.Errorf("run 3 recorded %+v, want a known absence", rec) + } + + // A later run that misses nothing clears the bookkeeping entirely. + m = NewMissing(root, MissingRetry, 3, false) + if err := m.Finish(); err != nil { + t.Errorf("a run that missed nothing failed: %v", err) + } + if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) { + t.Error("records lingered after the upstream served everything again") + } +} + +// TestMissingRepeatedPath verifies a file listed by several indexes counts +// once per run, as the budget counts runs rather than references. +func TestMissingRepeatedPath(t *testing.T) { + root := t.TempDir() + m := NewMissing(root, MissingRetry, 3, false) + m.Tolerate("pool/gone.deb") + m.Tolerate("pool/gone.deb") + if err := m.Finish(); err == nil { + t.Fatal("a single run exhausted the retry budget") + } + if rec := readMissingState(t, root)["pool/gone.deb"]; rec == nil || rec.Runs != 1 { + t.Errorf("recorded %+v, want one run", rec) + } +} + +// TestMissingIgnoreMode verifies absences never fail a run in the ignore +// mode, while still being recorded for whoever reads the repository. +func TestMissingIgnoreMode(t *testing.T) { + root := t.TempDir() + m := NewMissing(root, MissingIgnore, 3, false) + if !m.Tolerate("pool/gone.deb") { + t.Fatal("the ignore mode aborted on a missing file") + } + if err := m.Finish(); err != nil { + t.Errorf("Finish = %v, want nil", err) + } + if rec := readMissingState(t, root)["pool/gone.deb"]; rec == nil { + t.Error("the ignore mode recorded nothing") + } +} + +// TestMissingDryRun verifies a dry run reports absences without leaving +// bookkeeping behind, so it never spends a run's retry budget. +func TestMissingDryRun(t *testing.T) { + root := t.TempDir() + m := NewMissing(root, MissingRetry, 3, true) + m.Tolerate("pool/gone.deb") + if err := m.Finish(); err == nil { + t.Error("a dry run hid a missing file") + } + if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) { + t.Error("a dry run wrote a state file") + } +} diff --git a/fetch/source.go b/fetch/source.go new file mode 100644 index 0000000..501b14a --- /dev/null +++ b/fetch/source.go @@ -0,0 +1,260 @@ +package fetch + +import ( + "bufio" + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + log "github.com/sirupsen/logrus" +) + +// Source is the set of base URLs a repository can be fetched from. Direct +// repositories have a single base; mirrorlists provide several with +// failover between them. +type Source struct { + // mu guards bases and active. + mu sync.Mutex + bases []string + active int +} + +// NewSource builds a Source from base URLs, normalizing trailing slashes. +func NewSource(bases []string) *Source { + s := &Source{} + for _, base := range bases { + s.bases = append(s.bases, strings.TrimRight(base, "/")) + } + return s +} + +// Rebase trims a common relative suffix off every base URL so later fetches +// resolve against the parent directory. Bases without the suffix are kept +// unchanged. +func (s *Source) Rebase(suffix string) { + suffix = "/" + strings.Trim(suffix, "/") + s.mu.Lock() + defer s.mu.Unlock() + for i, base := range s.bases { + if strings.HasSuffix(base, suffix) { + s.bases[i] = strings.TrimSuffix(base, suffix) + } else { + log.WithField("mirror", base).Warn("Mirror URL does not share the expected path suffix; using it unchanged.") + } + } +} + +// GetOptions adjusts a single Source request. A non-zero ModifiedSince +// adds a conditional header, surfacing a 304 as ErrNotModified. A positive +// RangeFrom requests the remainder of a partially downloaded file. +type GetOptions struct { + ModifiedSince time.Time + RangeFrom int64 +} + +// Get requests a repository-relative path, failing over to the next mirror +// on transport or server errors. A 404 from the responding mirror is +// treated as authoritative so optional files do not trigger failover. +func (s *Source) Get(ctx context.Context, reqPath string, o GetOptions) (*http.Response, error) { + // Snapshot the bases so a concurrent Rebase cannot race the loop. + s.mu.Lock() + start := s.active + bases := append([]string(nil), s.bases...) + s.mu.Unlock() + if len(bases) == 0 { + return nil, errors.New("source has no mirror URLs") + } + + var lastErr error + for i := 0; i < len(bases); i++ { + idx := (start + i) % len(bases) + target := bases[idx] + "/" + strings.TrimPrefix(reqPath, "/") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + if !o.ModifiedSince.IsZero() { + req.Header.Set("If-Modified-Since", o.ModifiedSince.UTC().Format(http.TimeFormat)) + } + if o.RangeFrom > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", o.RangeFrom)) + } + resp, err := client.Load().Do(req) + if err != nil { + lastErr = fmt.Errorf("fetch %s: %w", target, err) + log.WithError(err).WithField("url", target).Debug("Mirror request failed.") + continue + } + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + resp.Body.Close() + return nil, fmt.Errorf("fetch %s: %w", target, ErrNotFound) + } + if resp.StatusCode == http.StatusNotModified && !o.ModifiedSince.IsZero() { + resp.Body.Close() + return nil, fmt.Errorf("fetch %s: %w", target, ErrNotModified) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + lastErr = fmt.Errorf("fetch %s: status %d", target, resp.StatusCode) + log.WithField("url", target).WithField("status", resp.StatusCode).Debug("Mirror request failed.") + continue + } + if idx != start { + s.mu.Lock() + s.active = idx + s.mu.Unlock() + } + return resp, nil + } + return nil, lastErr +} + +// ReadURL retrieves a raw URL fully into memory, limited to 4 MiB. It is +// used for probing mirrorlists, whose URLs may carry query strings that a +// Source cannot join paths onto. +func ReadURL(ctx context.Context, rawURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + resp, err := client.Load().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + return nil, fmt.Errorf("fetch %s: %w", rawURL, ErrNotFound) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("fetch %s: status %d", rawURL, resp.StatusCode) + } + // Read one byte past the cap so an oversized document can be detected + // and reported rather than silently truncated into a bad parse. + const maxSize = 4 << 20 + data, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxSize { + log.WithField("url", rawURL).Warn("Response exceeds the read limit and was truncated.") + data = data[:maxSize] + } + return data, nil +} + +// ParseMirrorlist extracts mirror base URLs from a mirrorlist body, +// skipping comments and lines that are not absolute HTTP URLs. +func ParseMirrorlist(body []byte) []string { + seen := map[string]bool{} + var bases []string + scanner := bufio.NewScanner(bytes.NewReader(body)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + u, err := url.Parse(line) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + continue + } + base := strings.TrimRight(line, "/") + if seen[base] { + continue + } + seen[base] = true + bases = append(bases, base) + } + return bases +} + +// ParseMetalink extracts mirror base URLs from a metalink document's +// repomd.xml entries, keeping the document's preference order. +func ParseMetalink(body []byte) []string { + const repomdSuffix = "/repodata/repomd.xml" + var ml struct { + Files []struct { + Name string `xml:"name,attr"` + URLs []struct { + Value string `xml:",chardata"` + } `xml:"resources>url"` + } `xml:"files>file"` + } + if xml.Unmarshal(body, &ml) != nil { + return nil + } + seen := map[string]bool{} + var bases []string + for _, file := range ml.Files { + if file.Name != "repomd.xml" { + continue + } + for _, u := range file.URLs { + v := strings.TrimSpace(u.Value) + if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") { + continue + } + base := strings.TrimSuffix(v, repomdSuffix) + if base == v { + continue + } + base = strings.TrimRight(base, "/") + if seen[base] { + continue + } + seen[base] = true + bases = append(bases, base) + } + } + return bases +} + +// agent returns the user agent sent with every upstream request, falling +// back to the build identity when the configuration clears it. +func agent() string { + if ua := cfg.C.Crawler.UserAgent; ua != "" { + return ua + } + return cfg.Name + "/" + cfg.Version +} + +// Get performs a plain GET of a raw URL with the mirror user agent, used +// for directory listings and other non-repository requests. +func Get(ctx context.Context, rawURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", agent()) + return client.Load().Do(req) +} + +// Bases returns a copy of the mirror base URLs, primarily for tests and +// diagnostics. +func (s *Source) Bases() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.bases...) +} + +// Active returns the base URL of the mirror currently being fetched from, +// which for a mirrorlist or metalink is the one that last answered rather +// than the list's own URL. It is empty when the source has no mirrors. +func (s *Source) Active() string { + s.mu.Lock() + defer s.mu.Unlock() + if s.active < 0 || s.active >= len(s.bases) { + return "" + } + return s.bases[s.active] +} diff --git a/fetch/source_test.go b/fetch/source_test.go new file mode 100644 index 0000000..01ea993 --- /dev/null +++ b/fetch/source_test.go @@ -0,0 +1,97 @@ +package fetch + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseMirrorlist verifies URL extraction, comment skipping, and +// deduplication. +func TestParseMirrorlist(t *testing.T) { + body := "# comment\n\nhttp://mirror1.example.com/repo/\nhttps://mirror2.example.com/repo\nhttp://mirror1.example.com/repo\n/bare/path\nnot a url\n" + bases := ParseMirrorlist([]byte(body)) + want := []string{"http://mirror1.example.com/repo", "https://mirror2.example.com/repo"} + if len(bases) != len(want) { + t.Fatalf("got %v, want %v", bases, want) + } + for i := range want { + if bases[i] != want[i] { + t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i]) + } + } +} + +// TestSourceFailover verifies a dead first mirror fails over to a working +// one and that missing files surface the not-found sentinel. +func TestSourceFailover(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + src := NewSource([]string{"http://127.0.0.1:1/el9", srv.URL + "/el9"}) + resp, err := src.Get(context.Background(), "repodata/repomd.xml", GetOptions{}) + if err != nil { + t.Fatal("failover did not reach the working mirror:", err) + } + resp.Body.Close() + if src.active != 1 { + t.Errorf("active mirror = %d, want 1", src.active) + } + + if _, err := src.Get(context.Background(), "no/such/file", GetOptions{}); !errors.Is(err, ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } +} + +// TestParseMetalink verifies base URL extraction from metalink documents. +func TestParseMetalink(t *testing.T) { + body := `` + + `` + + `https://mirror1.example.com/fedora/repodata/repomd.xml` + + `rsync://mirror2.example.com/fedora/repodata/repomd.xml` + + `http://mirror3.example.com/fedora/repodata/repomd.xml` + + `https://mirror1.example.com/fedora/repodata/repomd.xml` + + `` + bases := ParseMetalink([]byte(body)) + want := []string{"https://mirror1.example.com/fedora", "http://mirror3.example.com/fedora"} + if len(bases) != len(want) { + t.Fatalf("got %v, want %v", bases, want) + } + for i := range want { + if bases[i] != want[i] { + t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i]) + } + } + if ParseMetalink([]byte("not xml at all")) != nil { + t.Error("non-XML input produced bases") + } +} + +// TestSourceNoBases verifies a Source with no mirrors reports an error and +// no response. +func TestSourceNoBases(t *testing.T) { + src := NewSource(nil) + resp, err := src.Get(context.Background(), "anything", GetOptions{}) + if err == nil { + t.Fatal("expected an error for a source without bases") + } + if resp != nil { + t.Error("response must be nil on error") + } +} + +// TestSourceRebase verifies suffix trimming for suite-relative sources. +func TestSourceRebase(t *testing.T) { + src := NewSource([]string{"http://example.com/debian/dists/test", "http://other.example.com/mirror"}) + src.Rebase("dists/test") + if src.bases[0] != "http://example.com/debian" { + t.Errorf("bases[0] = %q", src.bases[0]) + } + if src.bases[1] != "http://other.example.com/mirror" { + t.Errorf("bases[1] changed unexpectedly: %q", src.bases[1]) + } +} diff --git a/fetch/stats.go b/fetch/stats.go new file mode 100644 index 0000000..f22959c --- /dev/null +++ b/fetch/stats.go @@ -0,0 +1,103 @@ +package fetch + +import ( + "context" + "fmt" + "sync/atomic" +) + +// Counters accumulates transfer activity for one repository's summary log +// line. Counters are atomic because download workers run concurrently. +type Counters struct { + Fetched atomic.Int64 + FetchedBytes atomic.Int64 + Unchanged atomic.Int64 + Pruned atomic.Int64 + Planned atomic.Int64 + PlannedBytes atomic.Int64 + Missing atomic.Int64 +} + +// Stats collects transfer activity for the repository currently +// synchronizing. The CLI synchronizes repositories sequentially, so one +// package-level collector suffices there. The server's concurrent crawls +// interleave into these counters, so their values are only meaningful in +// the sequential path; the server must not report them. +var Stats = &Counters{} + +// statsKey carries a caller's own collector on a context. +type statsKey struct{} + +// WithStats returns a context whose transfers accumulate into c as well as +// into Stats. It gives one synchronization its own totals even while other +// synchronizations run concurrently, which is what the mirror server's +// crawls and their trace files need. +func WithStats(ctx context.Context, c *Counters) context.Context { + if c == nil { + return ctx + } + return context.WithValue(ctx, statsKey{}, c) +} + +// runStats returns the caller's collector carried by ctx, or nil when the +// caller installed none. +func runStats(ctx context.Context) *Counters { + c, _ := ctx.Value(statsKey{}).(*Counters) + return c +} + +// recordFetched counts a completed download in the shared collector and in +// the caller's own, when one is installed. +func recordFetched(ctx context.Context, bytes int64) { + Stats.Fetched.Add(1) + Stats.FetchedBytes.Add(bytes) + if c := runStats(ctx); c != nil { + c.Fetched.Add(1) + c.FetchedBytes.Add(bytes) + } +} + +// recordUnchanged counts a file that needed no transfer in both collectors. +func recordUnchanged(ctx context.Context) { + Stats.Unchanged.Add(1) + if c := runStats(ctx); c != nil { + c.Unchanged.Add(1) + } +} + +// recordMissing counts a file the upstream did not serve in both +// collectors, whether or not the run tolerates its absence. +func recordMissing(ctx context.Context) { + Stats.Missing.Add(1) + if c := runStats(ctx); c != nil { + c.Missing.Add(1) + } +} + +// Reset clears the collector for the next repository. +func (s *Counters) Reset() { + s.Fetched.Store(0) + s.FetchedBytes.Store(0) + s.Unchanged.Store(0) + s.Pruned.Store(0) + s.Planned.Store(0) + s.PlannedBytes.Store(0) + s.Missing.Store(0) +} + +// FormatBytes renders a byte count in a human readable unit. +func FormatBytes(n int64) string { + const unit = 1024 + const prefixes = "KMGTPE" + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + // Cap the exponent at the last available prefix so an unexpectedly + // large count can never index past the prefix table. + for m := n / unit; m >= unit && exp < len(prefixes)-1; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), prefixes[exp]) +} diff --git a/flags.go b/flags.go new file mode 100644 index 0000000..d337fbe --- /dev/null +++ b/flags.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "runtime/debug" + "strings" + "time" + + "github.com/alecthomas/kong" + cfg "github.com/grmrgecko/repo-sync/config" +) + +// VersionFlag prints build information and exits. +type VersionFlag bool + +// Decode satisfies kong.MapperValue. The flag is treated as a boolean toggle. +func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil } + +// IsBool reports the flag as a boolean for kong's parser. +func (v VersionFlag) IsBool() bool { return true } + +// BeforeApply emits version information then exits before the rest of the +// command is executed. +func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error { + fmt.Printf("%s: %s (%s)\n", cfg.Name, cfg.Version, cfg.Mode) + if cfg.Commit != "" { + fmt.Printf(" commit: %s\n", cfg.Commit) + } + if cfg.Date != "" { + fmt.Printf(" built: %s\n", cfg.Date) + } + if cfg.Commit == "" { + if bi, ok := debug.ReadBuildInfo(); ok { + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + fmt.Printf(" commit: %s\n", s.Value) + case "vcs.time": + fmt.Printf(" built: %s\n", s.Value) + } + } + } + } + app.Exit(0) + return nil +} + +// Flags is the root command line configuration. The global flags override +// the configuration file for every command, so a sync run and the server +// honor the same settings; each is empty when unset so the configured +// value stands. +type Flags struct { + ConfigPath string `help:"The path to the config file." optional:"" type:"existingfile"` + LogLevel string `help:"Override the configured log level." enum:",debug,info,warn,error" default:""` + UserAgent string `help:"Override the configured user agent sent to upstreams." optional:""` + RequestTimeout time.Duration `help:"Override the configured upstream response timeout (e.g. 2m)." optional:""` + Version VersionFlag `name:"version" help:"Print version information and quit"` + + // Trace flags override the configuration's trace section. Enabling is a + // pointer so an unset flag leaves the configured value alone while + // --trace and --no-trace each override it. + Trace *bool `help:"Write a trace file into each synchronized repository." negatable:""` + TraceHost string `help:"Override the host name this mirror is traced under; defaults to the system hostname." optional:""` + TraceMaintainer string `help:"Override the maintainer recorded in traces, in \"name \" form." optional:""` + TraceSponsor string `help:"Override the sponsor recorded in traces." optional:""` + TraceCountry string `help:"Override the country recorded in traces." optional:""` + TraceLocation string `help:"Override the location recorded in traces." optional:""` + TraceThroughput string `help:"Override the throughput recorded in traces." optional:""` + + Sync SyncCmd `cmd:"" name:"sync" help:"Synchronize package repositories of any supported type."` + Server ServerCmd `cmd:"" name:"server" help:"Run the caching mirror server."` + Service ServiceCmd `cmd:"" name:"service" help:"Manage the mirror server system service."` +} + +// flags holds the parsed command line configuration. +var flags Flags + +// ParseFlags parses the command line, exiting with usage information on +// error. +func ParseFlags() *kong.Context { + return kong.Parse(&flags, + kong.Name(cfg.Name), + kong.Description(cfg.Description), + kong.UsageOnError(), + kong.ConfigureHelp(kong.HelpOptions{ + Compact: true, + }), + kong.Vars{ + "serviceActions": strings.Join(ServiceAction, ","), + }, + ) +} diff --git a/flags_test.go b/flags_test.go new file mode 100644 index 0000000..8083c28 --- /dev/null +++ b/flags_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "testing" + + cfg "github.com/grmrgecko/repo-sync/config" +) + +// TestApplyTraceFlags verifies the trace flags override the configuration +// only when given, so a configured value stands on its own. +func TestApplyTraceFlags(t *testing.T) { + configured := cfg.TraceConfig{ + Enabled: true, + Host: "configured.example.com", + Maintainer: "Configured ", + Country: "US", + } + yes, no := true, false + + tests := []struct { + name string + flags Flags + want cfg.TraceConfig + }{ + { + name: "unset flags leave the configuration alone", + flags: Flags{}, + want: configured, + }, + { + name: "--no-trace disables tracing", + flags: Flags{Trace: &no}, + want: func() cfg.TraceConfig { c := configured; c.Enabled = false; return c }(), + }, + { + name: "--trace enables tracing", + flags: Flags{Trace: &yes}, + want: func() cfg.TraceConfig { c := configured; c.Enabled = true; return c }(), + }, + { + name: "value flags override individually", + flags: Flags{TraceHost: "flag.example.com", TraceSponsor: "Flag Sponsor"}, + want: func() cfg.TraceConfig { + c := configured + c.Host = "flag.example.com" + c.Sponsor = "Flag Sponsor" + return c + }(), + }, + { + name: "flag values are normalized like configured ones", + flags: Flags{TraceMaintainer: "Flag\nMaintainer"}, + want: func() cfg.TraceConfig { + c := configured + c.Maintainer = "Flag Maintainer" + return c + }(), + }, + } + + // applyTraceFlags reads the package level flags and configuration, so + // each case installs its own and restores them afterwards. + origFlags := flags + origTrace := cfg.C.Trace + t.Cleanup(func() { + flags = origFlags + cfg.C.Trace = origTrace + }) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg.C.Trace = configured + flags = tc.flags + applyTraceFlags() + if cfg.C.Trace != tc.want { + t.Errorf("trace config = %+v, want %+v", cfg.C.Trace, tc.want) + } + }) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5f9996a --- /dev/null +++ b/go.mod @@ -0,0 +1,36 @@ +module github.com/grmrgecko/repo-sync + +go 1.26.4 + +require ( + github.com/alecthomas/kong v1.16.0 + github.com/coreos/go-systemd/v22 v22.7.0 + github.com/go-playground/validator/v10 v10.30.3 + github.com/kardianos/service v1.3.0 + github.com/klauspost/compress v1.19.1 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/viper v1.21.0 + github.com/ulikunitz/xz v0.5.16 + golang.org/x/net v0.57.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..890d18f --- /dev/null +++ b/go.sum @@ -0,0 +1,81 @@ +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4= +github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= +github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI= +github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= +github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/testrepos/testrepos.go b/internal/testrepos/testrepos.go new file mode 100644 index 0000000..7a9630f --- /dev/null +++ b/internal/testrepos/testrepos.go @@ -0,0 +1,258 @@ +// Package testrepos builds small fixture repositories for tests across +// the fetch, mirror, and server packages. +package testrepos + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// WriteFile creates a file with parent directories under a fixture tree. +func WriteFile(t *testing.T, name string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, data, 0644); err != nil { + t.Fatal(err) + } +} + +// GzipBytes compresses data with gzip for index fixtures. +func GzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := gz.Write(data); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// SHA256Hex returns the hex SHA-256 of data. +func SHA256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// MD5Hex returns the hex MD5 of data. +func MD5Hex(data []byte) string { + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]) +} + +// ServeDir serves a fixture tree over HTTP with directory listings, closing +// the server when the test finishes. +func ServeDir(t *testing.T, dir string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.FileServer(http.Dir(dir))) + t.Cleanup(srv.Close) + return srv +} + +// BuildRPMRepo writes a minimal yum repository into dir and returns the +// package contents by file name. +func BuildRPMRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "foo-1.0-1.x86_64.rpm": bytes.Repeat([]byte("foo"), 500), + "bar-2.0-1.x86_64.rpm": bytes.Repeat([]byte("bar"), 700), + } + var pkgXML bytes.Buffer + for name, data := range pkgs { + WriteFile(t, filepath.Join(dir, "Packages", name), data) + fmt.Fprintf(&pkgXML, + `%s%s`, + name, SHA256Hex(data), len(data), name) + } + primary := []byte(`` + pkgXML.String() + ``) + primaryGz := GzipBytes(t, primary) + primaryName := SHA256Hex(primaryGz) + "-primary.xml.gz" + WriteFile(t, filepath.Join(dir, "repodata", primaryName), primaryGz) + + // A prestodelta index references one delta package under drpms/. + drpm := RPMDelta() + WriteFile(t, filepath.Join(dir, "drpms", "foo-0.9_1.0-1.x86_64.drpm"), drpm) + presto := []byte(fmt.Sprintf( + `drpms/foo-0.9_1.0-1.x86_64.drpm%d%s`, + len(drpm), SHA256Hex(drpm))) + prestoGz := GzipBytes(t, presto) + prestoName := SHA256Hex(prestoGz) + "-prestodelta.xml.gz" + WriteFile(t, filepath.Join(dir, "repodata", prestoName), prestoGz) + + repomd := fmt.Sprintf( + `%s%d%s%d`, + SHA256Hex(primaryGz), primaryName, len(primaryGz), + SHA256Hex(prestoGz), prestoName, len(prestoGz)) + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml"), []byte(repomd)) + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.asc"), []byte("fake signature")) + return pkgs +} + +// RPMDelta is the delta package content used by BuildRPMRepo. +func RPMDelta() []byte { + return bytes.Repeat([]byte("drpm"), 100) +} + +// debPackagesStanza renders one binary package stanza for a Packages index. +func debPackagesStanza(name, version, poolPath string, data []byte) string { + return fmt.Sprintf( + "Package: %s\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSize: %d\nMD5sum: %s\nSHA256: %s\nDescription: Test package\n", + name, version, poolPath, len(data), MD5Hex(data), SHA256Hex(data)) +} + +// BuildDebRepo writes a minimal structured apt repository with main and +// universe components into dir and returns the pool file contents by +// archive-relative path. +func BuildDebRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pool := map[string][]byte{ + "pool/main/h/hello/hello_1.0_amd64.deb": bytes.Repeat([]byte("hello"), 300), + "pool/universe/e/extra/extra_2.0_amd64.deb": bytes.Repeat([]byte("extra"), 400), + } + for p, data := range pool { + WriteFile(t, filepath.Join(dir, filepath.FromSlash(p)), data) + } + + indexes := map[string][]byte{ + "main/binary-amd64/Packages": []byte(debPackagesStanza( + "hello", "1.0", "pool/main/h/hello/hello_1.0_amd64.deb", pool["pool/main/h/hello/hello_1.0_amd64.deb"])), + "universe/binary-amd64/Packages": []byte(debPackagesStanza( + "extra", "2.0", "pool/universe/e/extra/extra_2.0_amd64.deb", pool["pool/universe/e/extra/extra_2.0_amd64.deb"])), + } + + // The Release file lists both the uncompressed and gzip variants, but + // only the gzip variant is served, matching real archives. + release := "Origin: Test\nSuite: test\nCodename: test\nArchitectures: amd64\nComponents: main universe\nAcquire-By-Hash: yes\n" + listed := map[string][]byte{} + for p, data := range indexes { + gz := GzipBytes(t, data) + WriteFile(t, filepath.Join(dir, "dists", "test", filepath.FromSlash(p)+".gz"), gz) + listed[p] = data + listed[p+".gz"] = gz + } + for field, hasher := range map[string]func([]byte) string{"MD5Sum": MD5Hex, "SHA256": SHA256Hex} { + release += field + ":\n" + for p, data := range listed { + release += fmt.Sprintf(" %s %d %s\n", hasher(data), len(data), p) + } + } + WriteFile(t, filepath.Join(dir, "dists", "test", "Release"), []byte(release)) + return pool +} + +// BuildArchRepo writes a minimal pacman repository named name into dir and +// returns the package contents by file name. +func BuildArchRepo(t *testing.T, dir, name string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "zlib-1.3-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("zlib"), 300), + "jq-1.7-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("jq"), 400), + } + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for fname, data := range pkgs { + WriteFile(t, filepath.Join(dir, fname), data) + WriteFile(t, filepath.Join(dir, fname+".sig"), []byte("signature")) + desc := fmt.Sprintf("%%FILENAME%%\n%s\n\n%%NAME%%\n%s\n\n%%CSIZE%%\n%d\n\n%%SHA256SUM%%\n%s\n", + fname, strings.SplitN(fname, "-", 2)[0], len(data), SHA256Hex(data)) + hdr := &tar.Header{ + Name: strings.TrimSuffix(fname, "-x86_64.pkg.tar.zst") + "/desc", + Mode: 0644, + Size: int64(len(desc)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(desc)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + WriteFile(t, filepath.Join(dir, name+".db"), buf.Bytes()) + WriteFile(t, filepath.Join(dir, name+".files"), buf.Bytes()) + return pkgs +} + +// BuildApkRepo writes a minimal Alpine repository into dir and returns the +// package contents by file name. The index replicates apk's real layout: a +// signature tar segment with its end-of-archive blocks cut off and the +// index tar as separate gzip streams, concatenated. +func BuildApkRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + pkgs := map[string][]byte{ + "musl-1.2.5-r1.apk": bytes.Repeat([]byte("musl"), 300), + "zlib-1.3-r2.apk": bytes.Repeat([]byte("zlib"), 200), + } + index := "" + for fname, data := range pkgs { + name := strings.SplitN(fname, "-", 2)[0] + version := strings.TrimSuffix(strings.TrimPrefix(fname, name+"-"), ".apk") + index += fmt.Sprintf("C:Q1notarealpullchecksum=\nP:%s\nV:%s\nA:x86_64\nS:%d\n\n", name, version, len(data)) + WriteFile(t, filepath.Join(dir, fname), data) + } + + // tarBytes renders entries into an uncompressed tar archive. + tarBytes := func(entries map[string]string) []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for name, content := range entries { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() + } + + // The signature segment loses its two zero end-of-archive blocks so + // the tar stream continues into the index segment. + sig := tarBytes(map[string]string{".SIGN.RSA.test.rsa.pub": "signature"}) + sig = sig[:len(sig)-1024] + idx := tarBytes(map[string]string{"DESCRIPTION": "test repo", "APKINDEX": index}) + combined := append(GzipBytes(t, sig), GzipBytes(t, idx)...) + WriteFile(t, filepath.Join(dir, "APKINDEX.tar.gz"), combined) + return pkgs +} + +// BuildFlatDebRepo writes a minimal flat apt repository into dir and +// returns the package contents by file name. +func BuildFlatDebRepo(t *testing.T, dir string) map[string][]byte { + t.Helper() + data := bytes.Repeat([]byte("flat"), 250) + WriteFile(t, filepath.Join(dir, "hello_3.0_amd64.deb"), data) + packages := []byte(debPackagesStanza("hello", "3.0", "hello_3.0_amd64.deb", data)) + packagesGz := GzipBytes(t, packages) + WriteFile(t, filepath.Join(dir, "Packages.gz"), packagesGz) + release := "Architectures: amd64\nSHA256:\n" + for p, d := range map[string][]byte{"Packages": packages, "Packages.gz": packagesGz} { + release += fmt.Sprintf(" %s %d %s\n", SHA256Hex(d), len(d), p) + } + WriteFile(t, filepath.Join(dir, "Release"), []byte(release)) + return map[string][]byte{"hello_3.0_amd64.deb": data} +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..b8ad6e2 --- /dev/null +++ b/main.go @@ -0,0 +1,61 @@ +package main + +import ( + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// main parses the command line, installs the configuration every command +// shares, and dispatches to the selected command. +func main() { + ctx := ParseFlags() + if err := cfg.Init(flags.ConfigPath); err != nil { + log.Fatal(err) + } + applyGlobalFlags() + err := ctx.Run() + ctx.FatalIfErrorf(err) +} + +// applyGlobalFlags overrides the loaded configuration with the global +// command line flags, then reapplies the settings derived from it. It also +// runs after a SIGHUP reload so the flags keep winning over the file. +func applyGlobalFlags() { + if flags.LogLevel != "" { + cfg.C.Log.Level = flags.LogLevel + } + if flags.UserAgent != "" { + cfg.C.Crawler.UserAgent = flags.UserAgent + } + if flags.RequestTimeout > 0 { + cfg.C.Crawler.RequestTimeout = flags.RequestTimeout + } + applyTraceFlags() + cfg.SetupLogging(cfg.C.Log) + fetch.Reload() +} + +// applyTraceFlags overlays the trace command line flags onto the loaded +// configuration. Each flag is empty when unset so the configured value +// stands, matching how the other global overrides behave. +func applyTraceFlags() { + if flags.Trace != nil { + cfg.C.Trace.Enabled = *flags.Trace + } + for _, o := range []struct { + flag string + dst *string + }{ + {flags.TraceHost, &cfg.C.Trace.Host}, + {flags.TraceMaintainer, &cfg.C.Trace.Maintainer}, + {flags.TraceSponsor, &cfg.C.Trace.Sponsor}, + {flags.TraceCountry, &cfg.C.Trace.Country}, + {flags.TraceLocation, &cfg.C.Trace.Location}, + {flags.TraceThroughput, &cfg.C.Trace.Throughput}, + } { + if o.flag != "" { + *o.dst = cfg.TraceValue(o.flag) + } + } +} diff --git a/mirror/apk.go b/mirror/apk.go new file mode 100644 index 0000000..2107985 --- /dev/null +++ b/mirror/apk.go @@ -0,0 +1,179 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// APKIndexName is the fixed name of an Alpine repository index. +const APKIndexName = "APKINDEX.tar.gz" + +// apkPackage is one package entry parsed from an APKINDEX. Alpine's C: +// pull checksum only covers a package's control segment, so mirroring can +// verify downloads by size alone. +type apkPackage struct { + name string + version string + size int64 +} + +// filename returns the package file name as served beside the index. +func (p *apkPackage) filename() string { + return p.name + "-" + p.version + ".apk" +} + +// expect converts the package's metadata into a download expectation. +func (p *apkPackage) expect() *fetch.Expect { + e := &fetch.Expect{Size: p.size} + if p.size <= 0 { + e.Size = -1 + } + return e +} + +// syncApk synchronizes one Alpine repository from src into destDir. +func syncApk(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + miss := opts.newMissing(destDir) + tr := newTrace(opts) + ctx = tr.track(ctx) + + // Stage the index so a live tree keeps a consistent view until the + // packages it references are in place. + idxDst, err := fetch.LocalJoin(destDir, APKIndexName) + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, APKIndexName, idxDst, nil, true, false); err != nil { + return fmt.Errorf("fetch %s: %w", APKIndexName, err) + } + pkgs, err := readApkIndex(fetch.StagedOrFinal(idxDst)) + if err != nil { + return err + } + + // Download the packages, which live beside the index. + var jobs []fetch.Job + for _, pkg := range pkgs { + dst, err := fetch.LocalJoin(destDir, pkg.filename()) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename(), Dst: dst, Want: pkg.expect()}) + } + log.WithField("packages", len(pkgs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // Promote the index last so the published metadata chain is complete. + if err := fetch.PromoteOrDiscard(idxDst, opts.DryRun); err != nil { + return err + } + keep.Add(idxDst) + + // Publish the traces, upstream's included, before pruning so the keep + // set covers them. + tr.publish(ctx, destDir, src, keep, opts) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + + // The index is published either way; missing packages only decide + // whether the run reports itself as failed. + return miss.Finish() +} + +// readApkIndex parses an APKINDEX.tar.gz archive. The file is a signature +// segment and an index segment as concatenated gzip streams, which the +// multistream gzip reader and tar reader walk through transparently. +func readApkIndex(filename string) ([]apkPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + if hdr.Name != "APKINDEX" { + continue + } + pkgs, err := parseApkIndex(tr) + if err != nil { + return nil, fmt.Errorf("parse apk index %s: %w", filename, err) + } + return pkgs, nil + } + return nil, fmt.Errorf("no APKINDEX entry in %s", filename) +} + +// parseApkIndex reads APKINDEX stanzas, collecting each package's name, +// version, and file size from the P:, V:, and S: fields. +func parseApkIndex(r io.Reader) ([]apkPackage, error) { + var pkgs []apkPackage + var cur apkPackage + + // flush records the current stanza when it is complete. + flush := func() { + if cur.name != "" && cur.version != "" { + pkgs = append(pkgs, cur) + } + cur = apkPackage{} + } + + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 256*1024), 256*1024) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + flush() + continue + } + field, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + switch field { + case "P": + cur.name = value + case "V": + cur.version = value + case "S": + cur.size, _ = strconv.ParseInt(value, 10, 64) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + flush() + return pkgs, nil +} diff --git a/mirror/apk_test.go b/mirror/apk_test.go new file mode 100644 index 0000000..dfc8041 --- /dev/null +++ b/mirror/apk_test.go @@ -0,0 +1,100 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseApkIndex verifies stanza splitting and field extraction. +func TestParseApkIndex(t *testing.T) { + index := "C:Q1abc=\nP:musl\nV:1.2.5-r1\nA:x86_64\nS:1200\n\nC:Q1def=\nP:zlib\nV:1.3-r2\nS:800\n" + pkgs, err := parseApkIndex(strings.NewReader(index)) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Fatalf("got %d packages, want 2", len(pkgs)) + } + if pkgs[0].filename() != "musl-1.2.5-r1.apk" || pkgs[0].size != 1200 { + t.Errorf("first package = %+v", pkgs[0]) + } + if pkgs[1].filename() != "zlib-1.3-r2.apk" || pkgs[1].size != 800 { + t.Errorf("second package = %+v", pkgs[1]) + } +} + +// TestReadApkIndex verifies parsing through the signature segment of a +// multistream index archive. +func TestReadApkIndex(t *testing.T) { + dir := t.TempDir() + testrepos.BuildApkRepo(t, dir) + pkgs, err := readApkIndex(filepath.Join(dir, APKIndexName)) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Errorf("got %d packages, want 2", len(pkgs)) + } +} + +// TestSyncApk verifies a full synchronization mirrors packages and the +// index with no staged leftovers. +func TestSyncApk(t *testing.T) { + www := t.TempDir() + pkgs := testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/alpine/v3.24/community/x86_64", opts.Type, opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "alpine", "v3.24", "community", "x86_64") + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(local, APKIndexName)); err != nil { + t.Error("index not promoted:", err) + } + staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncApkPrune verifies stale packages are removed when pruning. +func TestSyncApkPrune(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "repo", "old-1.0-r0.apk") + if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale package survived pruning") + } +} diff --git a/mirror/arch.go b/mirror/arch.go new file mode 100644 index 0000000..c7e1148 --- /dev/null +++ b/mirror/arch.go @@ -0,0 +1,268 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path" + "sort" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// archPackage is one package entry parsed from a pacman database. +type archPackage struct { + filename string + size int64 + sums map[string]string +} + +// expect converts the package's metadata into a download expectation. +func (p *archPackage) expect() *fetch.Expect { + e := &fetch.Expect{Size: p.size, Sums: p.sums} + if p.size <= 0 { + e.Size = -1 + } + return e +} + +// archExtras lists the companion metadata files mirrored beside the +// database, as suffixes appended to the repository name. All are optional; +// which ones exist varies by repository. +var archExtras = []string{ + ".files", + ".db.tar.gz", + ".files.tar.gz", + ".links.tar.gz", + ".db.sig", + ".db.tar.gz.sig", + ".files.sig", + ".files.tar.gz.sig", + ".links.tar.gz.sig", +} + +// syncArch synchronizes one pacman repository from src into destDir. +func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + miss := opts.newMissing(destDir) + tr := newTrace(opts) + ctx = tr.track(ctx) + + // Determine the database name, which matches the repository rather + // than any fixed path. + name, err := archDBName(ctx, src, repoURL) + if err != nil { + return err + } + log.WithField("database", name+".db").Debug("Resolved pacman database.") + + // Stage the database so a live tree keeps a consistent view until the + // packages it references are in place. + dbDst, err := fetch.LocalJoin(destDir, name+".db") + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, name+".db", dbDst, nil, true, false); err != nil { + return fmt.Errorf("fetch %s.db: %w", name, err) + } + pkgs, err := readArchDB(fetch.StagedOrFinal(dbDst)) + if err != nil { + return err + } + + // Download packages with their detached signatures. Signatures have no + // published checksums, so an existing file is trusted as-is. + var jobs []fetch.Job + for _, pkg := range pkgs { + dst, err := fetch.LocalJoin(destDir, pkg.filename) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect()}) + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: &fetch.Expect{Size: -1}, Optional: true}) + } + log.WithField("packages", len(pkgs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // Stage the companion metadata files that exist upstream. A dry run + // only counts them, as none are needed for planning. + var extraJobs []fetch.Job + for _, suffix := range archExtras { + dst, err := fetch.LocalJoin(destDir, name+suffix) + if err != nil { + return err + } + extraJobs = append(extraJobs, fetch.Job{ReqPath: name + suffix, Dst: dst, Optional: true, Stage: true}) + } + if opts.DryRun { + fetch.PlanJobs(extraJobs, opts.Verify, keep) + } else { + states, err := fetch.Many(ctx, src, extraJobs, opts.Workers, opts.Verify, keep, nil) + if err != nil { + return fmt.Errorf("fetch repository metadata: %w", err) + } + for i, job := range extraJobs { + if states[i] == fetch.FileMissing { + // The upstream dropped the file; drop the local copy so a + // stale companion or signature is never served. + fetch.RemoveStale(job.Dst) + continue + } + if err := fetch.PromoteStaged(job.Dst); err != nil { + return err + } + } + } + + // Promote the database last so the published metadata chain is + // complete. + if err := fetch.PromoteOrDiscard(dbDst, opts.DryRun); err != nil { + return err + } + keep.Add(dbDst) + + // Publish the traces, upstream's included, before pruning so the keep + // set covers them. + tr.publish(ctx, destDir, src, keep, opts) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + + // The database is published either way; missing packages only decide + // whether the run reports itself as failed. + return miss.Finish() +} + +// archDBName finds the pacman database name for a repository, preferring +// the directory listing and falling back to probing names derived from the +// URL path, deepest segment first. +func archDBName(ctx context.Context, src *fetch.Source, repoURL string) (string, error) { + if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil { + var names []string + for f := range listing.files { + if strings.HasSuffix(f, ".db") { + names = append(names, strings.TrimSuffix(f, ".db")) + } + } + sort.Strings(names) + if len(names) > 1 { + log.WithField("databases", names).Warn("Multiple pacman databases found; using the first.") + } + if len(names) > 0 { + return names[0], nil + } + } + + // Listings can be disabled; a repository at .../core/os/x86_64 is + // probed as x86_64.db, os.db, then core.db. + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + var segs []string + for _, seg := range strings.Split(u.Path, "/") { + if seg != "" { + segs = append(segs, seg) + } + } + for i := len(segs) - 1; i >= 0; i-- { + resp, err := src.Get(ctx, segs[i]+".db", fetch.GetOptions{}) + if err == nil { + resp.Body.Close() + return segs[i], nil + } + if !errors.Is(err, fetch.ErrNotFound) { + return "", err + } + } + return "", fmt.Errorf("no pacman database found at %s", repoURL) +} + +// readArchDB parses a pacman database archive and collects each package's +// file name, size, and checksums from its desc entry. +func readArchDB(filename string) ([]archPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + // The database carries no compression extension, so the format is + // sniffed from its leading bytes. + r, closeFn, err := fetch.SniffDecompressor(f) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + tr := tar.NewReader(r) + var pkgs []archPackage + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse pacman database %s: %w", filename, err) + } + if hdr.FileInfo().IsDir() || path.Base(hdr.Name) != "desc" { + continue + } + data, err := io.ReadAll(io.LimitReader(tr, 1<<20)) + if err != nil { + return nil, fmt.Errorf("read desc entry %s: %w", hdr.Name, err) + } + fields := parseDesc(data) + if fields["FILENAME"] == "" { + continue + } + size, _ := strconv.ParseInt(fields["CSIZE"], 10, 64) + sums := map[string]string{} + if v := fields["SHA256SUM"]; v != "" { + sums["sha256"] = strings.ToLower(v) + } + if v := fields["MD5SUM"]; v != "" { + sums["md5"] = strings.ToLower(v) + } + pkgs = append(pkgs, archPackage{filename: fields["FILENAME"], size: size, sums: sums}) + } + return pkgs, nil +} + +// parseDesc extracts the first value of each %FIELD% block from a pacman +// desc entry. +func parseDesc(data []byte) map[string]string { + fields := map[string]string{} + field := "" + scanner := bufio.NewScanner(strings.NewReader(string(data))) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + switch { + case line == "": + field = "" + case strings.HasPrefix(line, "%") && strings.HasSuffix(line, "%"): + field = strings.Trim(line, "%") + case field != "": + if _, ok := fields[field]; !ok { + fields[field] = line + } + } + } + return fields +} diff --git a/mirror/arch_test.go b/mirror/arch_test.go new file mode 100644 index 0000000..e46b456 --- /dev/null +++ b/mirror/arch_test.go @@ -0,0 +1,166 @@ +package mirror + +import ( + "archive/tar" + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseDesc verifies field extraction from a pacman desc entry. +func TestParseDesc(t *testing.T) { + desc := "%FILENAME%\nzlib-1.3-1-x86_64.pkg.tar.zst\n\n%CSIZE%\n1200\n\n%SHA256SUM%\nabc123\n\n%DESC%\nFirst line\nSecond line\n" + fields := parseDesc([]byte(desc)) + if fields["FILENAME"] != "zlib-1.3-1-x86_64.pkg.tar.zst" { + t.Errorf("FILENAME = %q", fields["FILENAME"]) + } + if fields["CSIZE"] != "1200" { + t.Errorf("CSIZE = %q", fields["CSIZE"]) + } + if fields["SHA256SUM"] != "abc123" { + t.Errorf("SHA256SUM = %q", fields["SHA256SUM"]) + } + if fields["DESC"] != "First line" { + t.Errorf("DESC = %q, want first value only", fields["DESC"]) + } +} + +// TestReadArchDB verifies database parsing for both compressed and plain +// tar archives. +func TestReadArchDB(t *testing.T) { + dir := t.TempDir() + testrepos.BuildArchRepo(t, dir, "core") + + pkgs, err := readArchDB(filepath.Join(dir, "core.db")) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 2 { + t.Fatalf("got %d packages, want 2", len(pkgs)) + } + for _, pkg := range pkgs { + if pkg.filename == "" || pkg.size <= 0 || pkg.sums["sha256"] == "" { + t.Errorf("incomplete package entry: %+v", pkg) + } + } + + // A plain uncompressed tar database must parse through the sniffer. + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + desc := "%FILENAME%\nplain-1.0-1-any.pkg.tar.zst\n\n%CSIZE%\n10\n\n%SHA256SUM%\nabc\n" + if err := tw.WriteHeader(&tar.Header{Name: "plain-1.0-1/desc", Mode: 0644, Size: int64(len(desc))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(desc)); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + plain := filepath.Join(dir, "plain.db") + if err := os.WriteFile(plain, buf.Bytes(), 0644); err != nil { + t.Fatal(err) + } + pkgs, err = readArchDB(plain) + if err != nil { + t.Fatal(err) + } + if len(pkgs) != 1 || pkgs[0].filename != "plain-1.0-1-any.pkg.tar.zst" { + t.Errorf("plain tar parse = %+v", pkgs) + } +} + +// TestSyncArch verifies a full synchronization mirrors packages, +// signatures, and metadata with no staged leftovers. +func TestSyncArch(t *testing.T) { + www := t.TempDir() + pkgs := testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core") + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoArch, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/core/os/x86_64", opts.Type, opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "core", "os", "x86_64") + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + if _, err := os.Stat(filepath.Join(local, name+".sig")); err != nil { + t.Errorf("signature for %s missing: %v", name, err) + } + } + for _, name := range []string{"core.db", "core.files"} { + if _, err := os.Stat(filepath.Join(local, name)); err != nil { + t.Errorf("%s not promoted: %v", name, err) + } + } + staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncArchPrune verifies stale files are removed when pruning. +func TestSyncArchPrune(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "core"), "core") + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoArch, Destination: dest, Workers: 2, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/core", opts.Type, opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "core", "old-0.9-1-x86_64.pkg.tar.zst") + if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/core", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale package survived pruning") + } +} + +// TestArchDBNameFallback verifies the database name is probed from URL +// path segments when directory listings are unavailable. +func TestArchDBNameFallback(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "custom", "os", "x86_64"), "custom") + + // Serve files but refuse directory listings to force the fallback. + fs := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + http.NotFound(w, r) + return + } + fs.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + src := fetch.NewSource([]string{srv.URL + "/custom/os/x86_64"}) + name, err := archDBName(context.Background(), src, srv.URL+"/custom/os/x86_64") + if err != nil { + t.Fatal(err) + } + if name != "custom" { + t.Errorf("name = %q, want custom", name) + } +} diff --git a/mirror/control.go b/mirror/control.go new file mode 100644 index 0000000..d4416b2 --- /dev/null +++ b/mirror/control.go @@ -0,0 +1,100 @@ +package mirror + +import ( + "bufio" + "io" + "strings" +) + +// stanza is one RFC 822 style field block from an apt control file. +type stanza map[string]string + +// Get returns the first matching field, trying each name in order to cope +// with casing differences between repositories. +func (s stanza) Get(names ...string) string { + for _, name := range names { + if v, ok := s[name]; ok { + return v + } + } + return "" +} + +// controlReader reads stanzas from an apt control file. +type controlReader struct { + br *bufio.Reader +} + +// newControlReader wraps a reader for stanza-by-stanza parsing. +func newControlReader(r io.Reader) *controlReader { + return &controlReader{br: bufio.NewReaderSize(r, 64*1024)} +} + +// readStanza parses the next stanza, returning io.EOF when the input is +// exhausted. Continuation lines are joined with newlines so multi-line +// checksum fields keep one entry per line. +func (r *controlReader) readStanza() (stanza, error) { + st := stanza{} + var last string + for { + line, err := r.br.ReadString('\n') + if err != nil && err != io.EOF { + return nil, err + } + done := err == io.EOF && line == "" + line = strings.TrimRight(line, "\r\n") + switch { + case done || strings.TrimSpace(line) == "": + if len(st) > 0 { + return st, nil + } + if done { + return nil, io.EOF + } + case line[0] == ' ' || line[0] == '\t': + if last != "" { + st[last] += "\n" + strings.TrimSpace(line) + } + default: + name, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + last = name + st[name] = strings.TrimSpace(value) + } + } +} + +// stripClearsign extracts the signed payload from a clearsigned InRelease +// document, returning the input unchanged when it is not clearsigned. +func stripClearsign(data []byte) []byte { + const begin = "-----BEGIN PGP SIGNED MESSAGE-----" + const sig = "-----BEGIN PGP SIGNATURE-----" + text := strings.ReplaceAll(string(data), "\r\n", "\n") + idx := strings.Index(text, begin) + if idx < 0 { + return data + } + text = text[idx+len(begin):] + + // Skip the armor headers up to the first blank line. Without one the + // document is malformed, so treat it as not clearsigned rather than + // returning the armor headers as payload. + i := strings.Index(text, "\n\n") + if i < 0 { + return data + } + text = text[i+2:] + if i := strings.Index(text, sig); i >= 0 { + text = text[:i] + } + + // Reverse the dash escaping applied by the signer. + var b strings.Builder + for _, line := range strings.Split(text, "\n") { + b.WriteString(strings.TrimPrefix(line, "- ")) + b.WriteString("\n") + } + return []byte(b.String()) +} diff --git a/mirror/control_test.go b/mirror/control_test.go new file mode 100644 index 0000000..0b847d1 --- /dev/null +++ b/mirror/control_test.go @@ -0,0 +1,70 @@ +package mirror + +import ( + "errors" + "io" + "strings" + "testing" +) + +// TestReadStanza verifies stanza splitting, continuation joining, and EOF +// handling without a trailing newline. +func TestReadStanza(t *testing.T) { + input := "Package: hello\nSHA256:\n abc 10 path/one\n def 20 path/two\n\nPackage: world\nSize: 42" + cr := newControlReader(strings.NewReader(input)) + + st, err := cr.readStanza() + if err != nil { + t.Fatal(err) + } + if st.Get("Package") != "hello" { + t.Errorf("Package = %q, want hello", st.Get("Package")) + } + if want := "\nabc 10 path/one\ndef 20 path/two"; st.Get("SHA256") != want { + t.Errorf("SHA256 = %q, want %q", st.Get("SHA256"), want) + } + + st, err = cr.readStanza() + if err != nil { + t.Fatal(err) + } + if st.Get("Package") != "world" || st.Get("Size") != "42" { + t.Errorf("second stanza = %v", st) + } + + if _, err = cr.readStanza(); !errors.Is(err, io.EOF) { + t.Errorf("expected EOF, got %v", err) + } +} + +// TestStanzaGet verifies fallback field names cover casing differences. +func TestStanzaGet(t *testing.T) { + st := stanza{"MD5Sum": "abc"} + if got := st.Get("MD5sum", "MD5Sum"); got != "abc" { + t.Errorf("got %q, want abc", got) + } + if got := st.Get("Missing"); got != "" { + t.Errorf("got %q, want empty", got) + } +} + +// TestStripClearsign verifies payload extraction and dash unescaping from a +// clearsigned document, and passthrough of unsigned input. +func TestStripClearsign(t *testing.T) { + signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nOrigin: Test\n- Dashed: line\n-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n" + got := string(stripClearsign([]byte(signed))) + if !strings.Contains(got, "Origin: Test\n") { + t.Errorf("payload missing Origin: %q", got) + } + if !strings.Contains(got, "Dashed: line") || strings.Contains(got, "- Dashed") { + t.Errorf("dash escaping not reversed: %q", got) + } + if strings.Contains(got, "PGP") { + t.Errorf("armor not removed: %q", got) + } + + plain := "Origin: Test\n" + if string(stripClearsign([]byte(plain))) != plain { + t.Error("unsigned input was modified") + } +} diff --git a/mirror/deb.go b/mirror/deb.go new file mode 100644 index 0000000..bbe2ceb --- /dev/null +++ b/mirror/deb.go @@ -0,0 +1,542 @@ +package mirror + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// debFile is one file referenced by repository metadata with its expected +// size and checksums, keyed by lowercase algorithm name. +type debFile struct { + path string + size int64 + sums map[string]string +} + +// expect converts the file's metadata into a download expectation. +func (f *debFile) expect() *fetch.Expect { + e := &fetch.Expect{Size: f.size, Sums: f.sums} + if f.size <= 0 { + e.Size = -1 + } + return e +} + +// debRelease is the parsed subset of an apt Release file needed to mirror +// a suite. +type debRelease struct { + components []string + architectures []string + acquireByHash bool + files map[string]*debFile +} + +// releaseSumFields maps Release checksum block names to algorithm names, +// including the MD5sum casing some repositories use. +var releaseSumFields = map[string]string{ + "MD5Sum": "md5sum", + "MD5sum": "md5sum", + "SHA1": "sha1", + "SHA256": "sha256", + "SHA512": "sha512", +} + +// byHashDirs maps checksum algorithm names to the by-hash directory names +// apt uses. +var byHashDirs = map[string]string{ + "md5sum": "MD5Sum", + "sha1": "SHA1", + "sha256": "SHA256", + "sha512": "SHA512", +} + +// parseRelease extracts the fields the sync needs from a Release or +// InRelease document. +func parseRelease(data []byte) (*debRelease, error) { + st, err := newControlReader(bytes.NewReader(stripClearsign(data))).readStanza() + if err != nil { + return nil, fmt.Errorf("parse release file: %w", err) + } + rel := &debRelease{files: map[string]*debFile{}} + rel.components = strings.Fields(st.Get("Components")) + rel.architectures = strings.Fields(st.Get("Architectures")) + rel.acquireByHash = strings.EqualFold(st.Get("Acquire-By-Hash"), "yes") + + // Merge each checksum block into one record per file path. + for field, algo := range releaseSumFields { + for _, line := range strings.Split(st.Get(field), "\n") { + parts := strings.Fields(line) + if len(parts) != 3 { + continue + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + continue + } + f := rel.files[parts[2]] + if f == nil { + f = &debFile{path: parts[2], sums: map[string]string{}} + rel.files[parts[2]] = f + } + f.size = size + f.sums[algo] = strings.ToLower(parts[0]) + } + } + return rel, nil +} + +// syncDeb synchronizes one apt suite or flat repository from repoURL. +func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + tr := newTrace(opts) + ctx = tr.track(ctx) + + // Split the URL into the archive root and the suite path. Repositories + // without a dists directory are flat, with everything relative to the + // repository URL itself. + u, err := url.Parse(repoURL) + if err != nil { + return err + } + suiteRel := "" + rootURL := repoURL + if idx := strings.Index(u.Path, "/dists/"); idx >= 0 { + suiteRel = strings.Trim(u.Path[idx+1:], "/") + root := *u + root.Path = u.Path[:idx] + rootURL = root.String() + src.Rebase(suiteRel) + } + destRoot, err := opts.repoDest(rootURL) + if err != nil { + return err + } + destSuite := destRoot + suitePrefix := "" + if suiteRel != "" { + if destSuite, err = fetch.LocalJoin(destRoot, suiteRel); err != nil { + return err + } + suitePrefix = suiteRel + "/" + } + + // Track the pool files this suite lists but the upstream does not serve. + // The records live in the suite directory, as the pool is shared with + // the archive's other suites. + miss := opts.newMissing(destSuite) + + // Stage the release files; they are promoted last so a live tree keeps + // a consistent metadata chain. + releaseNames := []string{"InRelease", "Release", "Release.gpg"} + releaseStates := make([]fetch.FileState, len(releaseNames)) + for i, name := range releaseNames { + dst, err := fetch.LocalJoin(destSuite, name) + if err != nil { + return err + } + state, err := fetch.File(ctx, src, suitePrefix+name, dst, nil, true, false) + if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + // The upstream dropped this release file; drop the local + // copy so outdated metadata is never served. + if !opts.DryRun { + fetch.RemoveStale(dst) + } + continue + } + return fmt.Errorf("fetch %s: %w", name, err) + } + releaseStates[i] = state + } + + // Parse the strongest release document available, which may be the + // promoted copy when the upstream reported it unchanged. + var relData []byte + switch { + case releaseStates[0] != fetch.FileMissing: + relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "InRelease"))) + case releaseStates[1] != fetch.FileMissing: + relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "Release"))) + default: + return errors.New("repository serves neither InRelease nor Release") + } + if err != nil { + return err + } + rel, err := parseRelease(relData) + if err != nil { + return err + } + + // Stage every index file the release lists, applying component and + // architecture filters. Missing variants are normal; repositories list + // checksums for files they do not serve. A dry run only fetches the + // package and source indexes needed to plan the pool. + var indexJobs, plannedJobs []fetch.Job + for _, f := range sortedFiles(rel.files) { + // Releases commonly list their own release files; those are + // already staged above and must not be fetched twice. + if slices.Contains(releaseNames, f.path) { + continue + } + if !includeIndexFile(f.path, opts.Components, opts.Architectures) { + continue + } + dst, err := fetch.LocalJoin(destSuite, f.path) + if err != nil { + return err + } + job := fetch.Job{ + ReqPath: suitePrefix + f.path, + Dst: dst, + Want: f.expect(), + Optional: true, + Stage: true, + } + if opts.DryRun { + base, _ := splitCompressExt(f.path) + if name := path.Base(base); name != "Packages" && name != "Sources" { + plannedJobs = append(plannedJobs, job) + continue + } + } + indexJobs = append(indexJobs, job) + } + fetch.PlanJobs(plannedJobs, opts.Verify, keep) + states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, opts.Verify, keep, nil) + if err != nil { + return fmt.Errorf("fetch suite indexes: %w", err) + } + + // Pick one local compression variant of each package and source index + // to parse for pool file references. + variants := map[string]string{} + variantExt := map[string]string{} + prefOrder := map[string]int{".gz": 0, ".xz": 1, ".zst": 2, ".bz2": 3, "": 4} + for i, job := range indexJobs { + if states[i] == fetch.FileMissing { + continue + } + relPath := strings.TrimPrefix(job.ReqPath, suitePrefix) + base, ext := splitCompressExt(relPath) + if name := path.Base(base); name != "Packages" && name != "Sources" { + continue + } + local := job.Dst + if states[i] == fetch.FileStaged { + local = job.Dst + fetch.StagedSuffix + } + if cur, ok := variantExt[base]; !ok || prefOrder[ext] < prefOrder[cur] { + variantExt[base] = ext + variants[base] = local + } + } + + // Collect the pool files referenced by the chosen indexes. + pool := map[string]*debFile{} + for _, base := range sortedKeys(variants) { + files, err := parseIndexFile(variants[base], path.Base(base)) + if err != nil { + return fmt.Errorf("parse %s: %w", base, err) + } + for _, f := range files { + if _, ok := pool[f.path]; !ok { + pool[f.path] = f + } + } + } + + // Download pool files relative to the archive root. + poolJobs := make([]fetch.Job, 0, len(pool)) + for _, f := range sortedFiles(pool) { + dst, err := fetch.LocalJoin(destRoot, f.path) + if err != nil { + return err + } + poolJobs = append(poolJobs, fetch.Job{ReqPath: f.path, Dst: dst, Want: f.expect()}) + } + log.WithField("files", len(poolJobs)).Info("Synchronizing pool files.") + if opts.DryRun { + fetch.PlanJobs(poolJobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, opts.Verify, keep, miss); err != nil { + return fmt.Errorf("fetch pool files: %w", err) + } + + // Promote the staged indexes now the pool files they reference exist. + for i, job := range indexJobs { + if states[i] == fetch.FileMissing { + continue + } + if err := fetch.PromoteOrDiscard(job.Dst, opts.DryRun); err != nil { + return err + } + if rel.acquireByHash && !opts.DryRun { + addByHash(job.Dst, strings.TrimPrefix(job.ReqPath, suitePrefix), rel, keep) + } + } + + // Promote the release files last to complete the metadata chain. + for _, name := range []string{"Release.gpg", "Release", "InRelease"} { + dst, err := fetch.LocalJoin(destSuite, name) + if err != nil { + return err + } + if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { + return err + } + if _, err := os.Stat(dst); err == nil { + keep.Add(dst) + } + } + + // Publish the traces at the archive root, where the convention places + // them and where they describe the archive as a whole rather than one + // suite. Writing before pruning lets the keep set cover them, which + // matters for a flat repository, whose root is itself the pruned tree. + tr.publish(ctx, destRoot, src, keep, opts) + + // Prune only the suite directory for structured repositories; the pool + // can be shared with other suites of the same archive. + if opts.Prune { + if suiteRel == "" { + fetch.PruneTree(destRoot, keep, opts.PruneGrace, opts.DryRun) + } else { + log.Info("Pruning is limited to the suite directory; shared pool files are kept.") + fetch.PruneTree(destSuite, keep, opts.PruneGrace, opts.DryRun) + } + } + + // The suite is published either way; missing pool files only decide + // whether the run reports itself as failed. + return miss.Finish() +} + +// addByHash links an index file into its directory's by-hash tree for each +// checksum the release lists, matching apt's Acquire-By-Hash layout. +// Failures are logged and do not fail the sync. +func addByHash(dst, relPath string, rel *debRelease, keep *fetch.KeepSet) { + f := rel.files[relPath] + if f == nil || !strings.Contains(relPath, "/") { + return + } + dir := filepath.Dir(dst) + for algo, digest := range f.sums { + sub, ok := byHashDirs[algo] + if !ok { + continue + } + link := filepath.Join(dir, "by-hash", sub, digest) + if err := os.MkdirAll(filepath.Dir(link), 0755); err != nil { + log.WithError(err).WithField("path", link).Warn("Failed to create by-hash directory.") + return + } + if _, err := os.Stat(link); err == nil { + keep.Add(link) + continue + } + if err := os.Link(dst, link); err != nil { + // Fall back to copying when hard links are not supported. + if err := copyFile(dst, link); err != nil { + log.WithError(err).WithField("path", link).Warn("Failed to create by-hash entry.") + continue + } + } + keep.Add(link) + } +} + +// copyFile duplicates src to dst. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + _ = os.Remove(dst) + return err + } + return out.Close() +} + +// parseIndexFile reads a Packages or Sources index and returns the pool +// files it references, with paths relative to the archive root. +func parseIndexFile(filename, kind string) ([]*debFile, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + cr := newControlReader(r) + var out []*debFile + for { + st, err := cr.readStanza() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + if kind == "Sources" { + out = append(out, sourceFiles(st)...) + continue + } + name := st.Get("Filename") + if name == "" { + continue + } + size, _ := strconv.ParseInt(st.Get("Size"), 10, 64) + sums := map[string]string{} + if v := st.Get("MD5sum", "MD5Sum"); v != "" { + sums["md5sum"] = strings.ToLower(v) + } + for _, algo := range []string{"SHA1", "SHA256", "SHA512"} { + if v := st.Get(algo); v != "" { + sums[strings.ToLower(algo)] = strings.ToLower(v) + } + } + out = append(out, &debFile{path: path.Clean(name), size: size, sums: sums}) + } + return out, nil +} + +// sourceSumFields maps source stanza checksum list fields to algorithm +// names; Files is the historical MD5 list. +var sourceSumFields = map[string]string{ + "Files": "md5sum", + "Checksums-Sha1": "sha1", + "Checksums-Sha256": "sha256", + "Checksums-Sha512": "sha512", +} + +// sourceFiles extracts the files of one source package stanza, merging the +// per-algorithm checksum lists by file name. +func sourceFiles(st stanza) []*debFile { + dir := st.Get("Directory") + byName := map[string]*debFile{} + var order []string + for field, algo := range sourceSumFields { + for _, line := range strings.Split(st.Get(field), "\n") { + parts := strings.Fields(line) + if len(parts) != 3 { + continue + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + continue + } + name := parts[2] + f := byName[name] + if f == nil { + f = &debFile{path: path.Join(dir, name), sums: map[string]string{}} + byName[name] = f + order = append(order, name) + } + f.size = size + f.sums[algo] = strings.ToLower(parts[0]) + } + } + out := make([]*debFile, 0, len(byName)) + for _, name := range order { + out = append(out, byName[name]) + } + return out +} + +// includeIndexFile applies the component and architecture filters to a +// release-listed file path. +func includeIndexFile(p string, comps, archs []string) bool { + if len(comps) > 0 { + if segs := strings.Split(p, "/"); len(segs) > 1 && !slices.Contains(comps, segs[0]) { + return false + } + } + if len(archs) > 0 { + if arch := indexArch(p); arch != "" && !slices.Contains(archs, arch) { + return false + } + } + return true +} + +// indexArch extracts the architecture a release-listed file applies to, or +// an empty string when the file is architecture independent. +func indexArch(p string) string { + for _, seg := range strings.Split(p, "/") { + if arch, ok := strings.CutPrefix(seg, "binary-"); ok { + return arch + } + if arch, ok := strings.CutPrefix(seg, "installer-"); ok { + return arch + } + if seg == "source" { + return "source" + } + } + base, _ := splitCompressExt(path.Base(p)) + if arch, ok := strings.CutPrefix(base, "Contents-udeb-"); ok { + return arch + } + if arch, ok := strings.CutPrefix(base, "Contents-"); ok { + return arch + } + return "" +} + +// splitCompressExt splits a known compression extension off a path. +func splitCompressExt(p string) (string, string) { + for _, ext := range []string{".gz", ".xz", ".bz2", ".zst"} { + if strings.HasSuffix(p, ext) { + return strings.TrimSuffix(p, ext), ext + } + } + return p, "" +} + +// sortedFiles returns metadata file records in a stable path order. +func sortedFiles(files map[string]*debFile) []*debFile { + out := make([]*debFile, 0, len(files)) + for _, f := range files { + out = append(out, f) + } + sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path }) + return out +} + +// sortedKeys returns a map's keys in a stable order. +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/mirror/deb_test.go b/mirror/deb_test.go new file mode 100644 index 0000000..b0f9b6e --- /dev/null +++ b/mirror/deb_test.go @@ -0,0 +1,291 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestParseRelease verifies field extraction and checksum block merging. +func TestParseRelease(t *testing.T) { + release := "Origin: Test\nArchitectures: amd64 arm64\nComponents: main universe\nAcquire-By-Hash: yes\n" + + "MD5Sum:\n aaa 10 main/binary-amd64/Packages\n" + + "SHA256:\n bbb 10 main/binary-amd64/Packages\n ccc 20 main/binary-amd64/Packages.gz\n" + rel, err := parseRelease([]byte(release)) + if err != nil { + t.Fatal(err) + } + if len(rel.components) != 2 || rel.components[0] != "main" { + t.Errorf("components = %v", rel.components) + } + if len(rel.architectures) != 2 { + t.Errorf("architectures = %v", rel.architectures) + } + if !rel.acquireByHash { + t.Error("acquireByHash not detected") + } + f := rel.files["main/binary-amd64/Packages"] + if f == nil || f.size != 10 || f.sums["md5sum"] != "aaa" || f.sums["sha256"] != "bbb" { + t.Errorf("merged file = %+v", f) + } + if rel.files["main/binary-amd64/Packages.gz"] == nil { + t.Error("gz variant missing") + } +} + +// TestIndexArch verifies architecture extraction from release file paths. +func TestIndexArch(t *testing.T) { + cases := map[string]string{ + "main/binary-amd64/Packages.gz": "amd64", + "main/debian-installer/binary-i386/Packages": "i386", + "main/installer-armhf/current/images/SHA256SUMS": "armhf", + "main/source/Sources.gz": "source", + "Contents-s390x.gz": "s390x", + "main/Contents-udeb-riscv64.gz": "riscv64", + "main/i18n/Translation-en.gz": "", + "main/dep11/icons-64x64.tar.gz": "", + "Release": "", + } + for p, want := range cases { + if got := indexArch(p); got != want { + t.Errorf("indexArch(%q) = %q, want %q", p, got, want) + } + } +} + +// TestIncludeIndexFile verifies component and architecture filtering. +func TestIncludeIndexFile(t *testing.T) { + cases := []struct { + path string + comps []string + archs []string + want bool + }{ + {"main/binary-amd64/Packages.gz", nil, nil, true}, + {"universe/binary-amd64/Packages.gz", []string{"main"}, nil, false}, + {"main/binary-i386/Packages.gz", []string{"main"}, []string{"amd64"}, false}, + {"main/binary-amd64/Packages.gz", []string{"main"}, []string{"amd64"}, true}, + {"main/i18n/Translation-en.gz", []string{"main"}, []string{"amd64"}, true}, + {"Contents-arm64.gz", []string{"main"}, []string{"amd64"}, false}, + {"main/source/Sources.gz", nil, []string{"amd64"}, false}, + {"main/source/Sources.gz", nil, []string{"amd64", "source"}, true}, + } + for _, c := range cases { + if got := includeIndexFile(c.path, c.comps, c.archs); got != c.want { + t.Errorf("includeIndexFile(%q, %v, %v) = %v, want %v", c.path, c.comps, c.archs, got, c.want) + } + } +} + +// TestSourceFiles verifies checksum list merging and directory joining for +// source package stanzas. +func TestSourceFiles(t *testing.T) { + st := stanza{ + "Directory": "pool/main/h/hello", + "Files": "\naaa 100 hello_1.0.dsc\nbbb 200 hello_1.0.tar.gz", + "Checksums-Sha256": "\nccc 100 hello_1.0.dsc\nddd 200 hello_1.0.tar.gz", + } + files := sourceFiles(st) + if len(files) != 2 { + t.Fatalf("got %d files, want 2", len(files)) + } + byPath := map[string]*debFile{} + for _, f := range files { + byPath[f.path] = f + } + dsc := byPath["pool/main/h/hello/hello_1.0.dsc"] + if dsc == nil || dsc.size != 100 || dsc.sums["md5sum"] != "aaa" || dsc.sums["sha256"] != "ccc" { + t.Errorf("dsc = %+v", dsc) + } +} + +// TestSyncDebStructured verifies a dists-layout synchronization: pool files +// at the archive root, promoted indexes, by-hash entries, and no staged +// leftovers. +func TestSyncDebStructured(t *testing.T) { + www := t.TempDir() + pool := testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "debian") + for p, data := range pool { + got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(p))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("pool file %s content mismatch", p) + } + } + suite := filepath.Join(root, "dists", "test") + if _, err := os.Stat(filepath.Join(suite, "Release")); err != nil { + t.Error("Release not promoted:", err) + } + if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages.gz")); err != nil { + t.Error("Packages.gz missing:", err) + } + // The uncompressed variant is listed in the Release file but not + // served, so it must be skipped without failing the sync. + if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages")); err == nil { + t.Error("unserved uncompressed Packages unexpectedly present") + } + byHash, err := filepath.Glob(filepath.Join(suite, "main", "binary-amd64", "by-hash", "SHA256", "*")) + if err != nil || len(byHash) == 0 { + t.Error("by-hash entries missing") + } + staged, _ := filepath.Glob(filepath.Join(suite, "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncDebComponentFilter verifies filtered components are not mirrored. +func TestSyncDebComponentFilter(t *testing.T) { + www := t.TempDir() + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2, Components: []string{"main"}} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "debian") + if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil { + t.Error("main pool file missing:", err) + } + if _, err := os.Stat(filepath.Join(root, "pool", "universe")); err == nil { + t.Error("filtered universe pool files present") + } + if _, err := os.Stat(filepath.Join(root, "dists", "test", "universe")); err == nil { + t.Error("filtered universe indexes present") + } +} + +// TestSyncDebFlat verifies a repository without a dists directory mirrors +// relative to the repository URL itself. +func TestSyncDebFlat(t *testing.T) { + www := t.TempDir() + pool := testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/flat", opts.Type, opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "flat") + for name, data := range pool { + got, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(root, "Release")); err != nil { + t.Error("Release not promoted:", err) + } +} + +// TestSyncDebMissingPoolRetry verifies a pool file the upstream stopped +// serving does not cost the rest of the suite: the run mirrors everything +// else, reports the absence until its retry budget runs out, and then +// synchronizes cleanly with the absence recorded as known. +func TestSyncDebMissingPoolRetry(t *testing.T) { + www := t.TempDir() + archive := filepath.Join(www, "debian") + testrepos.BuildDebRepo(t, archive) + srv := testrepos.ServeDir(t, www) + + // Withdraw one pool file while its index keeps listing it, which is how + // an incomplete upstream presents itself. + const gone = "pool/universe/e/extra/extra_2.0_amd64.deb" + if err := os.Remove(filepath.Join(archive, filepath.FromSlash(gone))); err != nil { + t.Fatal(err) + } + + dest := t.TempDir() + opts := &Options{ + Type: RepoDeb, + Destination: dest, + Workers: 2, + Missing: fetch.MissingRetry, + MissingRetries: 2, + } + + // The first run still publishes the suite, but reports the absence. + err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts) + if err == nil { + t.Fatal("the first run hid a missing pool file") + } + root := filepath.Join(dest, "debian") + if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil { + t.Error("a missing pool file cost the rest of the pool:", err) + } + if _, err := os.Stat(filepath.Join(root, "dists", "test", "Release")); err != nil { + t.Error("a missing pool file cost the suite metadata:", err) + } + + // The second run exhausts the retry budget, so the absence is accepted + // and recorded in the suite directory rather than failing the run. + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil { + t.Fatal("the absence still failed the run after its retry budget:", err) + } + state := filepath.Join(root, "dists", "test", fetch.MissingStateName) + data, err := os.ReadFile(state) + if err != nil { + t.Fatal("the known absence was not recorded:", err) + } + if !bytes.Contains(data, []byte(gone)) { + t.Errorf("recorded state %s does not name the missing file", data) + } + + // Serving the file again clears the record on the next run. + testrepos.WriteFile(t, filepath.Join(archive, filepath.FromSlash(gone)), bytes.Repeat([]byte("extra"), 400)) + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(state); !os.IsNotExist(err) { + t.Error("the record survived the upstream serving the file again") + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(gone))); err != nil { + t.Error("the restored pool file was not fetched:", err) + } +} + +// TestSyncDebMissingPoolModes verifies the strict mode still fails on the +// first missing pool file and the ignore mode never fails at all. +func TestSyncDebMissingPoolModes(t *testing.T) { + www := t.TempDir() + archive := filepath.Join(www, "debian") + testrepos.BuildDebRepo(t, archive) + if err := os.Remove(filepath.Join(archive, filepath.FromSlash("pool/universe/e/extra/extra_2.0_amd64.deb"))); err != nil { + t.Fatal(err) + } + srv := testrepos.ServeDir(t, www) + + strict := &Options{Type: RepoDeb, Destination: t.TempDir(), Workers: 2, Missing: fetch.MissingFail} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", strict.Type, strict); err == nil { + t.Error("the strict mode tolerated a missing pool file") + } + + lenient := &Options{Type: RepoDeb, Destination: t.TempDir(), Workers: 2, Missing: fetch.MissingIgnore} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", lenient.Type, lenient); err != nil { + t.Error("the ignore mode failed on a missing pool file:", err) + } +} diff --git a/mirror/discover.go b/mirror/discover.go new file mode 100644 index 0000000..b5893b7 --- /dev/null +++ b/mirror/discover.go @@ -0,0 +1,304 @@ +package mirror + +import ( + "context" + "fmt" + "io" + "net/url" + "path" + "slices" + "sort" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" + "golang.org/x/net/html" +) + +// dirListing holds the entries parsed from an HTML directory index. +type dirListing struct { + dirs []string + files map[string]bool +} + +// repoTarget is one repository to synchronize paired with the format it is +// synchronized as. A single crawl can turn up several formats, so the type +// travels with the repository rather than with the run. +type repoTarget struct { + url string + typ RepoType +} + +// fileTarget is one loose file an include pattern matched during discovery, +// kept as the directory it was found in and its name so its destination +// follows the same path mapping a repository's does. +type fileTarget struct { + dirURL string + name string +} + +// discovery holds what a crawl found below a base URL. +type discovery struct { + repos []repoTarget + files []fileTarget + // failures counts the directories the crawl could not list. A crawl + // that missed part of the tree is not evidence that what it did not + // reach went away, so the cache does not withdraw anything from it. + failures int +} + +// discoverRepos crawls directory listings under baseURL looking for +// repositories of the formats the run accepts, descending at most +// opts.DiscoverDepth levels. Excluded entries are never crawled, and files +// matching the include patterns are collected along the way. +func discoverRepos(ctx context.Context, baseURL string, opts *Options) (*discovery, error) { + type node struct { + url string + rel string + depth int + } + types := opts.discoverTypes() + start := strings.TrimRight(baseURL, "/") + "/" + queue := []node{{url: start}} + visited := map[string]bool{start: true} + found := &discovery{} + + for len(queue) > 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + n := queue[0] + queue = queue[1:] + listing, err := listDir(ctx, n.url) + if err != nil { + log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.") + found.failures++ + continue + } + + // A directory containing repository metadata is synchronized as a + // whole; its children are not crawled further. + if typ, ok := repoTypeOf(listing, types); ok { + found.repos = append(found.repos, repoTarget{url: strings.TrimRight(n.url, "/"), typ: typ}) + continue + } + + // Collect the loose files the run asks for. Only directories that + // are not repositories are considered: a repository's own files come + // from its metadata, and pruning there would remove anything else. + found.files = append(found.files, matchedFiles(listing, n.url, n.rel, opts)...) + + if n.depth >= opts.DiscoverDepth { + continue + } + for _, dir := range listing.dirs { + rel := path.Join(n.rel, dir) + if matchAny(opts.Exclude, dir, rel) { + log.WithField("url", n.url+dir+"/").Debug("Skipped excluded directory.") + continue + } + child := n.url + url.PathEscape(dir) + "/" + if visited[child] { + continue + } + visited[child] = true + queue = append(queue, node{url: child, rel: rel, depth: n.depth + 1}) + } + } + sort.Slice(found.repos, func(i, j int) bool { return found.repos[i].url < found.repos[j].url }) + sort.Slice(found.files, func(i, j int) bool { + if found.files[i].dirURL != found.files[j].dirURL { + return found.files[i].dirURL < found.files[j].dirURL + } + return found.files[i].name < found.files[j].name + }) + return found, nil +} + +// repoTypeOf reports the format a directory listing identifies, considering +// only the formats the run accepts. The first match wins, so a directory +// carrying more than one format's markers is synchronized as the earlier +// one rather than twice. +func repoTypeOf(listing *dirListing, types []RepoType) (RepoType, bool) { + for _, typ := range types { + switch typ { + case RepoRPM: + if slices.Contains(listing.dirs, "repodata") { + return typ, true + } + case RepoDeb: + if listing.files["InRelease"] || listing.files["Release"] { + return typ, true + } + case RepoArch: + if hasArchDB(listing.files) { + return typ, true + } + case RepoApk: + if listing.files[APKIndexName] { + return typ, true + } + } + } + return "", false +} + +// matchedFiles returns the files in a listing the include patterns select, +// in name order so a run's work is deterministic. +func matchedFiles(listing *dirListing, dirURL, rel string, opts *Options) []fileTarget { + if len(opts.IncludeFiles) == 0 { + return nil + } + names := make([]string, 0, len(listing.files)) + for name := range listing.files { + names = append(names, name) + } + sort.Strings(names) + + dir := strings.TrimRight(dirURL, "/") + var files []fileTarget + for _, name := range names { + entry := path.Join(rel, name) + if matchAny(opts.Exclude, name, entry) { + continue + } + if !matchAny(opts.IncludeFiles, name, entry) { + continue + } + files = append(files, fileTarget{dirURL: dir, name: name}) + } + return files +} + +// matchAny reports whether any glob matches an entry. A pattern containing +// a slash is matched against the entry's path below the crawled URL, so it +// can pin a name to one place in the tree; a plain pattern is matched +// against the name alone, wherever in the tree it appears. Patterns are +// validated before a run starts, so a bad one cannot silently never match. +func matchAny(patterns []string, name, rel string) bool { + for _, pattern := range patterns { + pattern, subject := matchSubject(pattern, name, rel) + if ok, err := path.Match(pattern, subject); err == nil && ok { + return true + } + } + return false +} + +// matchSubject pairs a pattern with the string it is matched against. A +// leading slash anchors the pattern to the crawled URL, so it is stripped +// and the entry's path below that URL is matched; at the root that path is +// the bare name, which is what makes anchoring to the root expressible at +// all. Any other pattern carrying a slash is matched against the same path +// unanchored, and a plain pattern against the name alone. +func matchSubject(pattern, name, rel string) (string, string) { + if strings.HasPrefix(pattern, "/") { + return strings.TrimPrefix(pattern, "/"), rel + } + if strings.Contains(pattern, "/") { + return pattern, rel + } + return pattern, name +} + +// ValidPattern reports whether a glob is usable as an exclude or include +// pattern, so a run rejects a malformed one instead of never matching it. +// The anchoring slash is not part of the glob, so it is stripped before the +// syntax is checked. +func ValidPattern(pattern string) error { + _, err := path.Match(strings.TrimPrefix(pattern, "/"), "") + return err +} + +// hasArchDB reports whether a directory listing contains a pacman database. +func hasArchDB(files map[string]bool) bool { + for name := range files { + if strings.HasSuffix(name, ".db") { + return true + } + } + return false +} + +// listDir fetches an HTML directory index and extracts its immediate child +// directories and files. +func listDir(ctx context.Context, dirURL string) (*dirListing, error) { + resp, err := fetch.Get(ctx, dirURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode) + } + + base, err := url.Parse(dirURL) + if err != nil { + return nil, err + } + listing := &dirListing{files: map[string]bool{}} + seenDirs := map[string]bool{} + tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20)) + for { + tt := tok.Next() + if tt == html.ErrorToken { + break + } + if tt != html.StartTagToken && tt != html.SelfClosingTagToken { + continue + } + name, hasAttr := tok.TagName() + if string(name) != "a" || !hasAttr { + continue + } + var href string + for { + key, val, more := tok.TagAttr() + if string(key) == "href" { + href = string(val) + } + if !more { + break + } + } + entry, isDir := childEntry(base, href) + if entry == "" { + continue + } + if isDir { + if !seenDirs[entry] { + seenDirs[entry] = true + listing.dirs = append(listing.dirs, entry) + } + } else { + listing.files[entry] = true + } + } + return listing, nil +} + +// childEntry resolves an anchor href against the directory URL, returning +// the entry name when it is an immediate child of that directory. +func childEntry(base *url.URL, href string) (string, bool) { + if href == "" || strings.HasPrefix(href, "#") { + return "", false + } + ref, err := url.Parse(href) + if err != nil || ref.RawQuery != "" || ref.Fragment != "" { + return "", false + } + resolved := base.ResolveReference(ref) + if resolved.Scheme != base.Scheme || resolved.Host != base.Host { + return "", false + } + rel := strings.TrimPrefix(resolved.Path, base.Path) + if rel == "" || rel == resolved.Path { + return "", false + } + isDir := strings.HasSuffix(rel, "/") + rel = strings.TrimSuffix(rel, "/") + if rel == "" || strings.Contains(rel, "/") { + return "", false + } + return rel, isDir +} diff --git a/mirror/discover_cache.go b/mirror/discover_cache.go new file mode 100644 index 0000000..6450b87 --- /dev/null +++ b/mirror/discover_cache.go @@ -0,0 +1,341 @@ +package mirror + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// discoverState is the on-disk layout of the discovery cache, keyed by the +// base URL each crawl started from so one destination can hold several. +type discoverState struct { + Crawls map[string]*crawlRecord `json:"crawls"` +} + +// crawlRecord is one crawled base URL: when it was last scanned, the crawl +// options that scan ran with, and what it found. The options are recorded so +// a run that changes them scans again rather than replaying results the new +// patterns would not have produced. +type crawlRecord struct { + LastCrawled time.Time `json:"last_crawled"` + Options string `json:"options"` + Repos []*repoRecord `json:"repos,omitempty"` + Files []*fileRecord `json:"files,omitempty"` +} + +// repoRecord is one discovered repository. Gone marks when a scan stopped +// finding it upstream, and is what the prune grace is measured from. It is a +// pointer so a record the upstream still publishes carries no timestamp at +// all, rather than a zero one a reader has to interpret. +type repoRecord struct { + URL string `json:"url"` + Type string `json:"type"` + Gone *time.Time `json:"gone,omitempty"` +} + +// fileRecord is one discovered loose file, kept as the directory it was +// found in and its name, matching how a run mirrors it. +type fileRecord struct { + Dir string `json:"dir"` + Name string `json:"name"` + Gone *time.Time `json:"gone,omitempty"` +} + +// discoverCache is the discovery state for one destination tree. Crawling a +// vendor archive costs hundreds of directory listings, so the results are +// reused until they age out, and the record of what was found is also what +// tells a later run which repositories the upstream has since dropped. +type discoverCache struct { + path string + state discoverState +} + +// loadDiscoverCache reads the discovery state kept at the destination root. +// A missing or unreadable file starts empty: the cache only saves work and +// tracks what upstream has dropped, so losing it costs a crawl rather than +// correctness. +func loadDiscoverCache(dest string) *discoverCache { + c := &discoverCache{ + path: filepath.Join(dest, fetch.DiscoverStateName), + state: discoverState{Crawls: map[string]*crawlRecord{}}, + } + data, err := os.ReadFile(c.path) + if err != nil { + return c + } + var state discoverState + if err := json.Unmarshal(data, &state); err != nil { + log.WithError(err).WithField("path", c.path).Warn("Failed to parse the discovery state; scanning again.") + return c + } + for base, rec := range state.Crawls { + if rec != nil { + c.state.Crawls[base] = rec + } + } + return c +} + +// crawlKey normalizes a base URL into the key its crawl is recorded under, +// so the same tree named with and without a trailing slash is one record +// rather than two. +func crawlKey(base string) string { + return strings.TrimRight(base, "/") + "/" +} + +// reuse returns a recent crawl's results for a base URL, or reports that the +// tree has to be scanned again. A crawl ages out after ttl, and one run with +// different discovery options is never replayed under new ones. +func (c *discoverCache) reuse(base, options string, ttl time.Duration, now time.Time) (*discovery, bool) { + if ttl <= 0 { + return nil, false + } + rec := c.state.Crawls[crawlKey(base)] + if rec == nil || rec.Options != options || rec.LastCrawled.IsZero() { + return nil, false + } + age := now.Sub(rec.LastCrawled) + if age < 0 || age >= ttl { + return nil, false + } + + // Entries the last scan stopped finding are kept for their grace period + // but are no longer part of the tree, so they are not replayed. + found := &discovery{} + for _, repo := range rec.Repos { + if repo.Gone == nil { + found.repos = append(found.repos, repoTarget{url: repo.URL, typ: RepoType(repo.Type)}) + } + } + for _, file := range rec.Files { + if file.Gone == nil { + found.files = append(found.files, fileTarget{dirURL: file.Dir, name: file.Name}) + } + } + log.WithFields(log.Fields{ + "url": base, + "repositories": len(found.repos), + "files": len(found.files), + "age": age.Round(time.Second).String(), + }).Info("Reusing the cached discovery.") + return found, true +} + +// update records what a scan found and returns the targets the upstream no +// longer serves, which are the ones whose grace period has run out. A scan +// that could not list part of the tree is not evidence that anything went +// away, so it refreshes what it did find and leaves the rest alone. +// +// A withdrawal is only forgotten once the local copy goes with it, so a run +// that is not pruning keeps the record and leaves the removal to one that +// is, rather than losing track of a directory nothing will clean up. +func (c *discoverCache) update(base, options string, found *discovery, now time.Time, opts *Options) *discovery { + key := crawlKey(base) + rec := c.state.Crawls[key] + if rec == nil { + rec = &crawlRecord{} + c.state.Crawls[key] = rec + } + rec.LastCrawled = now + rec.Options = options + clean := found.failures == 0 + + seenRepos := map[string]bool{} + for _, repo := range found.repos { + seenRepos[repo.url] = true + } + seenFiles := map[string]bool{} + for _, file := range found.files { + seenFiles[file.dirURL+"/"+file.name] = true + } + + gone := &discovery{} + repos := rec.Repos[:0] + for _, repo := range rec.Repos { + if seenRepos[repo.URL] { + repo.Gone = nil + repos = append(repos, repo) + continue + } + if !clean { + repos = append(repos, repo) + continue + } + if repo.Gone == nil { + repo.Gone = &now + log.WithField("url", repo.URL).Info("Repository is no longer published upstream.") + } + if !opts.Prune || now.Sub(*repo.Gone) < opts.PruneGrace { + repos = append(repos, repo) + continue + } + gone.repos = append(gone.repos, repoTarget{url: repo.URL, typ: RepoType(repo.Type)}) + } + rec.Repos = repos + + files := rec.Files[:0] + for _, file := range rec.Files { + if seenFiles[file.Dir+"/"+file.Name] { + file.Gone = nil + files = append(files, file) + continue + } + if !clean { + files = append(files, file) + continue + } + if file.Gone == nil { + file.Gone = &now + log.WithFields(log.Fields{"url": file.Dir, "name": file.Name}).Info("File is no longer published upstream.") + } + if !opts.Prune || now.Sub(*file.Gone) < opts.PruneGrace { + files = append(files, file) + continue + } + gone.files = append(gone.files, fileTarget{dirURL: file.Dir, name: file.Name}) + } + rec.Files = files + + // Add what this scan turned up for the first time. + known := map[string]bool{} + for _, repo := range rec.Repos { + known[repo.URL] = true + } + for _, repo := range found.repos { + if !known[repo.url] { + rec.Repos = append(rec.Repos, &repoRecord{URL: repo.url, Type: string(repo.typ)}) + } + } + knownFiles := map[string]bool{} + for _, file := range rec.Files { + knownFiles[file.Dir+"/"+file.Name] = true + } + for _, file := range found.files { + if !knownFiles[file.dirURL+"/"+file.name] { + rec.Files = append(rec.Files, &fileRecord{Dir: file.dirURL, Name: file.name}) + } + } + + // Keep the file in a stable order so it reads as a listing rather than + // as whatever order the crawl happened to finish in. + sort.Slice(rec.Repos, func(i, j int) bool { return rec.Repos[i].URL < rec.Repos[j].URL }) + sort.Slice(rec.Files, func(i, j int) bool { + if rec.Files[i].Dir != rec.Files[j].Dir { + return rec.Files[i].Dir < rec.Files[j].Dir + } + return rec.Files[i].Name < rec.Files[j].Name + }) + return gone +} + +// save writes the discovery state, removing it once no crawl is tracked. A +// dry run leaves it as it found it, so a planning run never spends the +// cache's lifetime or its grace periods. +func (c *discoverCache) save(dryRun bool) { + if dryRun { + return + } + if len(c.state.Crawls) == 0 { + if err := os.Remove(c.path); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", c.path).Warn("Failed to remove the discovery state.") + } + return + } + data, err := json.MarshalIndent(c.state, "", " ") + if err == nil { + err = os.MkdirAll(filepath.Dir(c.path), 0755) + } + if err == nil { + err = os.WriteFile(c.path, data, 0644) + } + if err != nil { + log.WithError(err).WithField("path", c.path).Warn("Failed to save the discovery state.") + } +} + +// discoverOptions renders the crawl settings that decide what a scan finds, +// so a run that changes any of them scans the tree again instead of +// replaying results the new settings would not have produced. +func (o *Options) discoverOptions() string { + types := make([]string, 0, len(o.discoverTypes())) + for _, typ := range o.discoverTypes() { + types = append(types, string(typ)) + } + sort.Strings(types) + exclude := append([]string(nil), o.Exclude...) + sort.Strings(exclude) + include := append([]string(nil), o.IncludeFiles...) + sort.Strings(include) + return fmt.Sprintf("depth=%d types=%s exclude=%s include=%s", + o.DiscoverDepth, strings.Join(types, ","), strings.Join(exclude, ","), strings.Join(include, ",")) +} + +// evictDiscovered removes what a scan no longer finds upstream, so a +// repository a vendor withdrew does not linger in the mirror forever. It is +// as destructive as pruning and is gated on the same flag; a dry run reports +// the removals instead of making them. +func evictDiscovered(gone *discovery, opts *Options) { + for _, repo := range gone.repos { + dir, err := opts.repoDest(repo.url) + if err != nil { + log.WithError(err).WithField("url", repo.url).Warn("Failed to resolve a withdrawn repository's directory.") + continue + } + // A repository mapping onto the destination root itself, which is + // what --flat does, shares its directory with everything else in the + // run; removing that tree would take the whole mirror with it. + if !underDestination(dir, opts.Destination) { + log.WithFields(log.Fields{"url": repo.url, "path": dir}). + Warn("Not evicting a withdrawn repository that maps onto the destination root.") + continue + } + if opts.DryRun { + log.WithFields(log.Fields{"url": repo.url, "path": dir}).Info("Would evict a withdrawn repository.") + continue + } + if err := os.RemoveAll(dir); err != nil { + log.WithError(err).WithField("path", dir).Warn("Failed to evict a withdrawn repository.") + continue + } + log.WithFields(log.Fields{"url": repo.url, "path": dir}).Info("Evicted a withdrawn repository.") + } + + for _, file := range gone.files { + dir, err := opts.repoDest(file.dirURL) + if err != nil { + log.WithError(err).WithField("url", file.dirURL).Warn("Failed to resolve a withdrawn file's directory.") + continue + } + name, err := fetch.LocalJoin(dir, file.name) + if err != nil { + log.WithError(err).WithField("name", file.name).Warn("Failed to resolve a withdrawn file's path.") + continue + } + if opts.DryRun { + log.WithField("path", name).Info("Would evict a withdrawn file.") + continue + } + if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", name).Warn("Failed to evict a withdrawn file.") + continue + } + log.WithField("path", name).Info("Evicted a withdrawn file.") + } +} + +// underDestination reports whether a path sits strictly below the +// destination directory, which is what makes it safe to remove as a whole. +func underDestination(dir, dest string) bool { + dirClean := filepath.Clean(dir) + destClean := filepath.Clean(dest) + return dirClean != destClean && strings.HasPrefix(dirClean, destClean+string(os.PathSeparator)) +} diff --git a/mirror/discover_cache_test.go b/mirror/discover_cache_test.go new file mode 100644 index 0000000..38fe1cb --- /dev/null +++ b/mirror/discover_cache_test.go @@ -0,0 +1,336 @@ +package mirror + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// countingArchive serves a fixture tree, counting the directory listings a +// crawl requests so a test can tell a fresh scan from a reused one. +func countingArchive(t *testing.T, www string) (*httptest.Server, *atomic.Int64) { + t.Helper() + var listings atomic.Int64 + files := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + listings.Add(1) + } + files.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return srv, &listings +} + +// readDiscoverState reads the discovery cache a run left at a destination. +func readDiscoverState(t *testing.T, dest string) discoverState { + t.Helper() + data, err := os.ReadFile(filepath.Join(dest, fetch.DiscoverStateName)) + if err != nil { + t.Fatal(err) + } + var state discoverState + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + return state +} + +// TestDiscoverCacheReuse verifies a crawl's results are reused until they +// age out, so a repeat run costs no directory listings, and that scanning +// again picks up what the upstream has published since. +func TestDiscoverCacheReuse(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv, listings := countingArchive(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + DiscoverCache: time.Hour, + Workers: 2, + } + + // The first run scans the tree and records what it found. + summary, err := Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 1 { + t.Fatalf("repositories = %d, want 1", summary.Repositories) + } + scanned := listings.Load() + if scanned == 0 { + t.Fatal("the first run listed no directories") + } + state := readDiscoverState(t, dest) + rec := state.Crawls[crawlKey(srv.URL)] + if rec == nil || len(rec.Repos) != 1 || rec.Repos[0].Type != string(RepoRPM) { + t.Fatalf("recorded crawl = %+v", rec) + } + + // A repository published after that scan is not picked up while the + // cached results are still current, and the tree is not listed again. + testrepos.BuildRPMRepo(t, filepath.Join(www, "el10")) + summary, err = Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 1 { + t.Errorf("repositories = %d on the cached run, want 1", summary.Repositories) + } + if got := listings.Load(); got != scanned { + t.Errorf("the cached run listed %d directories", got-scanned) + } + if _, err := os.Stat(filepath.Join(dest, "el10")); err == nil { + t.Error("a repository published after the scan was mirrored from the cache") + } + + // Scanning on every run finds it. + opts.DiscoverCache = 0 + summary, err = Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 2 { + t.Errorf("repositories = %d after re-scanning, want 2", summary.Repositories) + } + if _, err := os.Stat(filepath.Join(dest, "el10", "repodata", "repomd.xml")); err != nil { + t.Error("the newly published repository was not mirrored:", err) + } +} + +// TestDiscoverCacheOptionsChange verifies a run that changes what a crawl +// would find scans again rather than replaying the previous results. +func TestDiscoverCacheOptionsChange(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "sles15")) + srv, _ := countingArchive(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + DiscoverCache: time.Hour, + Workers: 2, + } + summary, err := Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 2 { + t.Fatalf("repositories = %d, want 2", summary.Repositories) + } + + // The cache is still current, but the run no longer asks for the same + // tree, so the crawl has to happen again. + opts.Exclude = []string{"sles*"} + summary, err = Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 1 { + t.Errorf("repositories = %d after excluding one, want 1", summary.Repositories) + } +} + +// TestDiscoverCacheEviction verifies a repository the upstream withdrew is +// removed from the mirror when pruning is enabled, and kept otherwise. +func TestDiscoverCacheEviction(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "el10")) + srv, _ := countingArchive(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + Workers: 2, + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10", "repodata", "repomd.xml")); err != nil { + t.Fatal("second repository not mirrored:", err) + } + + // Withdraw it upstream. Without pruning the local copy stays, but the + // crawl records that it is gone. + if err := os.RemoveAll(filepath.Join(www, "el10")); err != nil { + t.Fatal(err) + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10")); err != nil { + t.Error("a withdrawn repository was removed without pruning:", err) + } + + // With pruning it is evicted, and the record goes with it. + opts.Prune = true + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10")); !os.IsNotExist(err) { + t.Error("a withdrawn repository survived a pruning run") + } + if _, err := os.Stat(filepath.Join(dest, "el9", "repodata", "repomd.xml")); err != nil { + t.Error("the surviving repository was evicted too:", err) + } + for _, repo := range readDiscoverState(t, dest).Crawls[crawlKey(srv.URL)].Repos { + if strings.HasSuffix(repo.URL, "/el10") { + t.Error("the evicted repository is still recorded") + } + } +} + +// TestDiscoverCacheEvictionGrace verifies a withdrawn repository waits out +// the prune grace before it is evicted, so one bad crawl does not take a +// tree with it. +func TestDiscoverCacheEvictionGrace(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "el10")) + srv, _ := countingArchive(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + Workers: 2, + Prune: true, + PruneGrace: time.Hour, + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(filepath.Join(www, "el10")); err != nil { + t.Fatal(err) + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10")); err != nil { + t.Error("a withdrawn repository was evicted inside its grace period:", err) + } + + // Age the recorded withdrawal past the grace period; the next run + // evicts it. + state := readDiscoverState(t, dest) + aged := time.Now().Add(-2 * time.Hour) + for _, repo := range state.Crawls[crawlKey(srv.URL)].Repos { + if strings.HasSuffix(repo.URL, "/el10") { + if repo.Gone == nil { + t.Fatal("the withdrawal was not recorded") + } + repo.Gone = &aged + } + } + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, fetch.DiscoverStateName), data, 0644); err != nil { + t.Fatal(err) + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10")); !os.IsNotExist(err) { + t.Error("a withdrawn repository survived its grace period") + } +} + +// TestDiscoverCacheEvictionNeedsCleanCrawl verifies a crawl that could not +// list part of the tree withdraws nothing: an upstream that failed to +// answer is not an upstream that dropped a repository. +func TestDiscoverCacheEvictionNeedsCleanCrawl(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "el10")) + + // The second run answers the el10 listing with an error, which is how a + // repository that is still published can go missing from a crawl. + var broken atomic.Bool + files := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if broken.Load() && strings.HasPrefix(r.URL.Path, "/el10") { + http.Error(w, "upstream is having a bad day", http.StatusInternalServerError) + return + } + files.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + Workers: 2, + Prune: true, + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + + broken.Store(true) + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "el10", "repodata", "repomd.xml")); err != nil { + t.Error("a failed listing evicted a repository that is still published:", err) + } + for _, repo := range readDiscoverState(t, dest).Crawls[crawlKey(srv.URL)].Repos { + if strings.HasSuffix(repo.URL, "/el10") && repo.Gone != nil { + t.Error("a failed listing recorded a repository as withdrawn") + } + } +} + +// TestDiscoverCacheDryRun verifies a planning run neither records a crawl +// nor spends the cache's lifetime. +func TestDiscoverCacheDryRun(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv, _ := countingArchive(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 3, + DiscoverCache: time.Hour, + Workers: 2, + DryRun: true, + } + if _, err := Sync(context.Background(), opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, fetch.DiscoverStateName)); !os.IsNotExist(err) { + t.Error("a dry run recorded a crawl") + } +} diff --git a/mirror/discover_test.go b/mirror/discover_test.go new file mode 100644 index 0000000..8d002ce --- /dev/null +++ b/mirror/discover_test.go @@ -0,0 +1,374 @@ +package mirror + +import ( + "context" + "net/url" + "path/filepath" + "testing" + + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestChildEntry verifies anchor resolution against a directory URL. +func TestChildEntry(t *testing.T) { + base, _ := url.Parse("http://example.com/repos/") + cases := []struct { + href string + want string + isDir bool + }{ + {"el9/", "el9", true}, + {"file.rpm", "file.rpm", false}, + {"/repos/el8/", "el8", true}, + {"../", "", false}, + {"?C=N;O=D", "", false}, + {"http://other.example.com/repos/x/", "", false}, + {"/outside/", "", false}, + {"deep/nested/", "", false}, + {"#anchor", "", false}, + } + for _, c := range cases { + got, isDir := childEntry(base, c.href) + if got != c.want || isDir != c.isDir { + t.Errorf("childEntry(%q) = %q,%v, want %q,%v", c.href, got, isDir, c.want, c.isDir) + } + } +} + +// discoverOpts builds the options a discovery test crawls with. +func discoverOpts(typ RepoType, depth int) *Options { + return &Options{Type: typ, DiscoverDepth: depth} +} + +// repoURLs reduces a crawl's repositories to their URLs, for comparison +// against what a fixture tree publishes. +func repoURLs(found *discovery) []string { + urls := make([]string, 0, len(found.repos)) + for _, repo := range found.repos { + urls = append(urls, repo.url) + } + return urls +} + +// checkURLs verifies a crawl found exactly the expected repositories, in +// order. +func checkURLs(t *testing.T, found *discovery, want []string) { + t.Helper() + got := repoURLs(found) + if len(got) != len(want) { + t.Fatalf("found %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("found[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +// TestDiscoverRepos verifies RPM repositories are found through directory +// listings and that the depth limit is honored. +func TestDiscoverRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "a", "repo1")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "b", "nested", "repo2")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "deep", "x", "y", "z", "repo3")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 3)) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/a/repo1", srv.URL + "/b/nested/repo2"}) + + // A deeper limit reaches the third repository. + found, err = discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 6)) + if err != nil { + t.Fatal(err) + } + if len(found.repos) != 3 { + t.Errorf("depth 6 found %v, want 3 repositories", repoURLs(found)) + } +} + +// TestDiscoverArchRepos verifies pacman repositories are recognized by +// their database files. +func TestDiscoverArchRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core") + testrepos.BuildArchRepo(t, filepath.Join(www, "extra", "os", "x86_64"), "extra") + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoArch, 4)) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/core/os/x86_64", srv.URL + "/extra/os/x86_64"}) +} + +// TestDiscoverApkRepos verifies Alpine repositories are recognized by +// their index files, matching the arch directories under a release tree. +func TestDiscoverApkRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "x86_64")) + testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "aarch64")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL+"/v3.24/community", discoverOpts(RepoApk, 1)) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/v3.24/community/aarch64", srv.URL + "/v3.24/community/x86_64"}) +} + +// TestDiscoverDebRepos verifies suite directories are recognized by their +// release files. +func TestDiscoverDebRepos(t *testing.T) { + www := t.TempDir() + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoDeb, 4)) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/debian/dists/test", srv.URL + "/flat"}) +} + +// TestDiscoverAllTypes verifies a crawl that names no type finds every +// format under one URL, as a vendor archive publishing both yum and apt +// repositories requires. +func TestDiscoverAllTypes(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9")) + testrepos.BuildDebRepo(t, filepath.Join(www, "apt")) + testrepos.BuildArchRepo(t, filepath.Join(www, "arch", "os", "x86_64"), "core") + testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "x86_64")) + srv := testrepos.ServeDir(t, www) + + found, err := discoverRepos(context.Background(), srv.URL, &Options{DiscoverDepth: 4}) + if err != nil { + t.Fatal(err) + } + want := map[string]RepoType{ + srv.URL + "/alpine/v3.24/x86_64": RepoApk, + srv.URL + "/apt/dists/test": RepoDeb, + srv.URL + "/arch/os/x86_64": RepoArch, + srv.URL + "/yum/el9": RepoRPM, + } + if len(found.repos) != len(want) { + t.Fatalf("found %v, want %v", found.repos, want) + } + for _, repo := range found.repos { + if want[repo.url] != repo.typ { + t.Errorf("%s discovered as %q, want %q", repo.url, repo.typ, want[repo.url]) + } + } + + // Limiting the crawl to one format leaves the rest of the tree alone. + found, err = discoverRepos(context.Background(), srv.URL, &Options{Types: []RepoType{RepoRPM}, DiscoverDepth: 4}) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/yum/el9"}) +} + +// TestDiscoverExclude verifies excluded directories are neither crawled nor +// reported, by name anywhere in the tree or by path below the crawled URL. +func TestDiscoverExclude(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el", "9")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles", "15")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "fc", "40")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "docker", "el", "9")) + srv := testrepos.ServeDir(t, www) + + opts := discoverOpts(RepoRPM, 4) + opts.Exclude = []string{"sles", "fc", "docker"} + found, err := discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/yum/el/9"}) + + // A pattern with a slash pins the exclusion to one place in the tree. + opts.Exclude = []string{"yum/*"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/docker/el/9"}) +} + +// TestDiscoverIncludeFiles verifies loose files outside repositories are +// matched by the include patterns, and that files inside a repository are +// left to that repository's own synchronization. +func TestDiscoverIncludeFiles(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9")) + testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-mysql-2023"), []byte("key")) + testrepos.WriteFile(t, filepath.Join(www, "README.txt"), []byte("notes")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "el9", "extra.rpm"), []byte("rpm")) + srv := testrepos.ServeDir(t, www) + + opts := discoverOpts(RepoRPM, 4) + opts.IncludeFiles = []string{"*.rpm", "RPM-GPG-KEY-*"} + found, err := discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, []string{srv.URL + "/yum/el9"}) + + want := []fileTarget{ + {dirURL: srv.URL, name: "RPM-GPG-KEY-mysql-2023"}, + {dirURL: srv.URL + "/yum", name: "loose-1.0.rpm"}, + } + if len(found.files) != len(want) { + t.Fatalf("matched %v, want %v", found.files, want) + } + for i := range want { + if found.files[i] != want[i] { + t.Errorf("files[%d] = %v, want %v", i, found.files[i], want[i]) + } + } + + // An exclusion wins over an include pattern. + opts.Exclude = []string{"RPM-GPG-KEY-*"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + if len(found.files) != 1 || found.files[0].name != "loose-1.0.rpm" { + t.Errorf("matched %v, want only the loose rpm", found.files) + } +} + +// TestDiscoverAnchoredPatterns verifies a leading slash limits a pattern to +// the crawled directory itself, for both include and exclude patterns. +func TestDiscoverAnchoredPatterns(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9")) + testrepos.WriteFile(t, filepath.Join(www, "root-1.0.rpm"), []byte("rpm")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "deep", "buried-1.0.rpm"), []byte("rpm")) + srv := testrepos.ServeDir(t, www) + + // Anchored, only the root directory's file is mirrored. + opts := discoverOpts(RepoRPM, 4) + opts.IncludeFiles = []string{"/*.rpm"} + found, err := discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + want := []fileTarget{{dirURL: srv.URL, name: "root-1.0.rpm"}} + if len(found.files) != len(want) || found.files[0] != want[0] { + t.Errorf("anchored include matched %v, want %v", found.files, want) + } + + // The same pattern unanchored matches at every depth, which is what + // makes the anchor worth having. + opts.IncludeFiles = []string{"*.rpm"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + if len(found.files) != 3 { + t.Errorf("unanchored include matched %v, want 3 files", found.files) + } + + // An anchored exclusion applies to the crawled directory only, leaving + // the same name deeper in the tree alone. + opts.Exclude = []string{"/root-1.0.rpm"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + if len(found.files) != 2 { + t.Errorf("anchored exclude left %v, want 2 files", found.files) + } + for _, file := range found.files { + if file.name == "root-1.0.rpm" { + t.Errorf("anchored exclude kept %v", file) + } + } + + // An anchored directory exclusion keeps the crawl out of a top-level + // directory without matching that name elsewhere. + opts.Exclude = []string{"/deep"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + if len(found.files) != 3 { + t.Errorf("top-level exclude of an unrelated name left %v, want 3 files", found.files) + } + opts.Exclude = []string{"/yum"} + found, err = discoverRepos(context.Background(), srv.URL, opts) + if err != nil { + t.Fatal(err) + } + checkURLs(t, found, nil) + if len(found.files) != 1 || found.files[0].name != "root-1.0.rpm" { + t.Errorf("anchored directory exclude left %v, want only the root rpm", found.files) + } +} + +// TestMatchSubject verifies the pairing of a pattern with the string it is +// matched against, which decides whether a pattern is anchored, pinned by +// path, or matched by name anywhere in the tree. +func TestMatchSubject(t *testing.T) { + cases := []struct { + pattern string + wantPattern string + wantSubject string + }{ + {"*.rpm", "*.rpm", "pkg.rpm"}, + {"/*.rpm", "*.rpm", "yum/deep/pkg.rpm"}, + {"yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"}, + {"/yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"}, + } + for _, c := range cases { + pattern, subject := matchSubject(c.pattern, "pkg.rpm", "yum/deep/pkg.rpm") + if pattern != c.wantPattern || subject != c.wantSubject { + t.Errorf("matchSubject(%q) = %q,%q, want %q,%q", + c.pattern, pattern, subject, c.wantPattern, c.wantSubject) + } + } +} + +// TestDetectType verifies a repository URL given without a type is +// identified by its metadata, and that an unidentifiable URL is reported. +func TestDetectType(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core") + testrepos.BuildApkRepo(t, filepath.Join(www, "alpine")) + srv := testrepos.ServeDir(t, www) + + cases := []struct { + path string + want RepoType + }{ + {"/el9", RepoRPM}, + {"/debian/dists/test", RepoDeb}, + {"/core/os/x86_64", RepoArch}, + {"/alpine", RepoApk}, + } + for _, c := range cases { + got, err := detectType(context.Background(), srv.URL+c.path, AllTypes) + if err != nil { + t.Errorf("detectType(%s): %v", c.path, err) + continue + } + if got != c.want { + t.Errorf("detectType(%s) = %q, want %q", c.path, got, c.want) + } + } + + // A URL holding no repository metadata cannot be identified. + if _, err := detectType(context.Background(), srv.URL+"/debian", AllTypes); err == nil { + t.Error("expected an error detecting the type of an archive root") + } +} diff --git a/mirror/mirror_test.go b/mirror/mirror_test.go new file mode 100644 index 0000000..6b5aa44 --- /dev/null +++ b/mirror/mirror_test.go @@ -0,0 +1,14 @@ +package mirror + +import ( + "os" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + fetch.Reload() + os.Exit(m.Run()) +} diff --git a/mirror/options.go b/mirror/options.go new file mode 100644 index 0000000..9a105ea --- /dev/null +++ b/mirror/options.go @@ -0,0 +1,160 @@ +// Package mirror synchronizes remote package repositories into a local +// directory tree, preserving the upstream layout so the result can be +// served directly to package managers. +package mirror + +import ( + "net/url" + "path" + "strings" + "time" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// RepoType identifies the repository format being synchronized. +type RepoType string + +const ( + // RepoRPM synchronizes yum/dnf style repositories. + RepoRPM RepoType = "rpm" + // RepoDeb synchronizes apt style repositories. + RepoDeb RepoType = "deb" + // RepoArch synchronizes pacman style repositories. + RepoArch RepoType = "arch" + // RepoApk synchronizes Alpine apk style repositories. + RepoApk RepoType = "apk" +) + +// AllTypes lists every supported repository format, in the order a +// directory's metadata is checked against them. RPM and deb come first as +// their markers are the most specific. +var AllTypes = []RepoType{RepoRPM, RepoDeb, RepoArch, RepoApk} + +// TypeNames lists the supported format names, for user facing messages. +func TypeNames() []string { + names := make([]string, 0, len(AllTypes)) + for _, typ := range AllTypes { + names = append(names, string(typ)) + } + return names +} + +// Options configures a synchronization run. +type Options struct { + // Type selects the repository format. It is empty for a run whose + // repositories are identified by their metadata rather than named on the + // command line, and is pinned to the repository's own format before that + // repository is synchronized. + Type RepoType + // Types limits the formats a run identifies repositories as. An empty + // list accepts every supported format. + Types []RepoType + // URLs lists the repository or mirrorlist URLs to synchronize. + URLs []string + // Destination is the local directory repositories are placed under. + Destination string + // Trim drops this many leading URL path components when building the + // destination path. + Trim int + // Flat places repositories directly in Destination without copying the + // URL path. + Flat bool + // Discover crawls each URL's directory listings for repositories. + Discover bool + // DiscoverDepth limits how deep discovery crawls below each URL. + DiscoverDepth int + // DiscoverCache is how long a crawl's results are reused before the + // tree is scanned again. Zero scans on every run. + DiscoverCache time.Duration + // Exclude lists globs matched against the names of entries found while + // crawling; a match is neither crawled nor mirrored. A pattern + // containing a slash is matched against the entry's path below the + // crawled URL instead of its name, and a leading slash anchors it to + // that URL. + Exclude []string + // IncludeFiles lists globs matched against the names of files found + // outside of any repository while crawling; a match is mirrored as a + // plain file. Patterns match by path the same way Exclude does, so a + // leading slash limits one to the crawled directory itself. Files + // inside a repository come from its metadata and are not matched here. + IncludeFiles []string + // Workers is the number of concurrent download workers. + Workers int + // Verify re-verifies checksums of existing local files. + Verify bool + // Prune deletes local files no longer part of the repository. + Prune bool + // PruneGrace defers pruning a file until it has been unreferenced for + // this long across runs. + PruneGrace time.Duration + // DryRun reports what would change without downloading packages or + // altering the published repository. + DryRun bool + // Missing selects what happens when the upstream does not serve a + // package the repository metadata lists. The zero value is the strict + // mode, failing the repository at the first such file. + Missing fetch.MissingMode + // MissingRetries is how many consecutive runs must miss a file before + // the retry mode accepts it as gone and stops failing the run over it. + MissingRetries int + // Trace writes a trace file describing this mirror into every + // repository synchronized, and mirrors the traces upstream publishes + // beside it. + Trace bool + // TraceInfo describes the mirror recorded in that trace file. + TraceInfo TraceInfo + // destBase, when set by the mirror server, pins the repository root + // directory directly instead of deriving it from the URL path. + destBase string + // Components limits deb synchronization to these components. + Components []string + // Architectures limits deb synchronization to these architectures. + Architectures []string +} + +// discoverTypes returns the formats a run identifies repositories as: the +// formats it was limited to, the single format it was given, or every +// supported format when it named none. +func (o *Options) discoverTypes() []RepoType { + if len(o.Types) > 0 { + return o.Types + } + if o.Type != "" { + return []RepoType{o.Type} + } + return AllTypes +} + +// newMissing starts tracking the packages an upstream does not serve for +// one repository, recording them in dir. That is the repository's own +// directory, and for apt the suite directory rather than the archive root, +// so suites sharing a pool keep their own records. +func (o *Options) newMissing(dir string) *fetch.Missing { + return fetch.NewMissing(dir, o.Missing, o.MissingRetries, o.DryRun) +} + +// repoDest maps a repository URL onto the destination directory according +// to the configured trimming rules. +func (o *Options) repoDest(repoURL string) (string, error) { + if o.destBase != "" { + return o.destBase, nil + } + if o.Flat { + return o.Destination, nil + } + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + var segs []string + for _, seg := range strings.Split(u.Path, "/") { + if seg != "" { + segs = append(segs, seg) + } + } + if o.Trim >= len(segs) { + return o.Destination, nil + } + return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...)) +} diff --git a/mirror/options_test.go b/mirror/options_test.go new file mode 100644 index 0000000..84004de --- /dev/null +++ b/mirror/options_test.go @@ -0,0 +1,31 @@ +package mirror + +import ( + "path/filepath" + "testing" +) + +// TestRepoDest verifies URL path copying, trimming, and flat placement. +func TestRepoDest(t *testing.T) { + dest := t.TempDir() + repoURL := "https://example.com/repos/CentOS/7/EA4/" + cases := []struct { + name string + opts Options + want string + }{ + {"default", Options{Destination: dest}, filepath.Join(dest, "repos", "CentOS", "7", "EA4")}, + {"trim2", Options{Destination: dest, Trim: 2}, filepath.Join(dest, "7", "EA4")}, + {"trim-all", Options{Destination: dest, Trim: 10}, dest}, + {"flat", Options{Destination: dest, Flat: true}, dest}, + } + for _, c := range cases { + got, err := c.opts.repoDest(repoURL) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got != c.want { + t.Errorf("%s: got %q, want %q", c.name, got, c.want) + } + } +} diff --git a/mirror/resolve.go b/mirror/resolve.go new file mode 100644 index 0000000..f3b6819 --- /dev/null +++ b/mirror/resolve.go @@ -0,0 +1,126 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// probePaths lists the metadata files that identify a repository of a type. +// Types without fixed-name metadata return nothing. +func probePaths(typ RepoType) []string { + switch typ { + case RepoRPM: + return []string{"repodata/repomd.xml"} + case RepoDeb: + return []string{"InRelease", "Release"} + case RepoApk: + return []string{APKIndexName} + } + return nil +} + +// detectType identifies the format of a repository URL for a run that did +// not name one. The directory listing is consulted first, as a pacman +// database is named after its repository and has no fixed path to probe; +// the formats that do have one are then probed directly, so a URL whose +// upstream serves no listing is still identified. Only the formats the run +// accepts are considered. +func detectType(ctx context.Context, repoURL string, types []RepoType) (RepoType, error) { + // A URL carrying a query or fragment is a mirrorlist or metalink + // endpoint rather than a directory, and nothing can be joined onto it. + if hasQueryOrFragment(repoURL) { + return "", fmt.Errorf("cannot determine the repository type of %s; name it with --type", repoURL) + } + + if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil { + if typ, ok := repoTypeOf(listing, types); ok { + return typ, nil + } + } + + src := fetch.NewSource([]string{repoURL}) + for _, typ := range types { + for _, probe := range probePaths(typ) { + resp, err := src.Get(ctx, probe, fetch.GetOptions{}) + if err == nil { + resp.Body.Close() + return typ, nil + } + if !errors.Is(err, fetch.ErrNotFound) { + return "", err + } + } + } + return "", fmt.Errorf("cannot determine the repository type of %s; name it with --type", repoURL) +} + +// hasQueryOrFragment reports whether a URL carries a query or fragment, +// which makes it unusable as a base for joining repository paths onto. +// Unparsable URLs are reported as plain so the caller still probes them. +func hasQueryOrFragment(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + return u.RawQuery != "" || u.Fragment != "" +} + +// resolveSource decides whether repoURL is a repository root or a mirror +// list and returns the source to fetch from, along with the URL whose path +// reflects the repository layout: repoURL itself for a direct repository, +// or the first mirror's URL when repoURL is a mirror list. RPM mirror lists +// may be plain text or metalink documents. +func resolveSource(ctx context.Context, repoURL string, typ RepoType) (*fetch.Source, string, error) { + // Pacman databases are named after the repository, so there is no + // fixed path to probe and pacman mirrorlists are config snippets + // rather than URL lists; the URL is used directly. + direct := fetch.NewSource([]string{repoURL}) + probes := probePaths(typ) + if len(probes) == 0 { + return direct, repoURL, nil + } + + // Probe for repository metadata directly under the URL. A URL carrying a + // query or fragment cannot have a repository path joined onto it, as the + // path would land inside the query, so skip the probe: such URLs are + // mirrorlist or metalink endpoints and are read as one below. + if !hasQueryOrFragment(repoURL) { + for _, probe := range probes { + resp, err := direct.Get(ctx, probe, fetch.GetOptions{}) + if err == nil { + resp.Body.Close() + return direct, repoURL, nil + } + if !errors.Is(err, fetch.ErrNotFound) { + return nil, "", err + } + } + } + + // Fall back to reading the URL itself as a mirror list. + body, err := fetch.ReadURL(ctx, repoURL) + if err != nil { + return nil, "", fmt.Errorf("read mirrorlist %s: %w", repoURL, err) + } + kind := "mirrorlist" + var bases []string + if typ == RepoRPM { + if bases = fetch.ParseMetalink(body); len(bases) > 0 { + kind = "metalink" + } + } + if len(bases) == 0 { + bases = fetch.ParseMirrorlist(body) + } + if len(bases) == 0 { + return nil, "", fmt.Errorf("%s is neither a repository nor a mirrorlist", repoURL) + } + log.WithFields(log.Fields{"url": repoURL, "mirrors": len(bases), "kind": kind}).Info("Using mirror list for repository.") + return fetch.NewSource(bases), bases[0], nil +} diff --git a/mirror/resolve_test.go b/mirror/resolve_test.go new file mode 100644 index 0000000..a3e5157 --- /dev/null +++ b/mirror/resolve_test.go @@ -0,0 +1,123 @@ +package mirror + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// TestResolveSource verifies direct repositories are used as-is and +// mirrorlist bodies expand into multi-mirror sources. +func TestResolveSource(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + // The mirrorlist needs the real server URL, which is only known now + // that the listener is up; files are read per-request. + testrepos.WriteFile(t, filepath.Join(www, "ml.txt"), + []byte("http://127.0.0.1:1/el9\n"+srv.URL+"/el9\n")) + + direct, layoutURL, err := resolveSource(context.Background(), srv.URL+"/el9", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(direct.Bases()) != 1 { + t.Errorf("direct bases = %v", direct.Bases()) + } + if layoutURL != srv.URL+"/el9" { + t.Errorf("direct layout URL = %q", layoutURL) + } + + ml, layoutURL, err := resolveSource(context.Background(), srv.URL+"/ml.txt", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(ml.Bases()) != 2 { + t.Errorf("mirrorlist bases = %v", ml.Bases()) + } + if layoutURL != "http://127.0.0.1:1/el9" { + t.Errorf("mirrorlist layout URL = %q", layoutURL) + } + + if _, _, err := resolveSource(context.Background(), srv.URL+"/el9/Packages", RepoRPM); err == nil { + t.Error("expected error for a non-repository URL") + } +} + +// TestResolveSourceQueryMirrorlist verifies a mirrorlist URL carrying a +// query string is read as a mirrorlist rather than probed as a repository. +// A repository path cannot be joined onto such a URL: it would land inside +// the query, and endpoints that answer 200 with an error page for an +// unrecognized parameter would then look like a repository. +func TestResolveSourceQueryMirrorlist(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + repo := testrepos.ServeDir(t, www) + + var probed []string + list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probed = append(probed, r.URL.String()) + if r.URL.Query().Get("repo") != "os" { + // Several real mirrorlist services answer 200 with a message. + w.Write([]byte("Invalid release/repo/arch combination\n")) + return + } + w.Write([]byte(repo.URL + "/el9\n")) + })) + t.Cleanup(list.Close) + + src, layoutURL, err := resolveSource(context.Background(), list.URL+"/?release=9&repo=os", RepoRPM) + if err != nil { + t.Fatal(err) + } + if bases := src.Bases(); len(bases) != 1 || bases[0] != repo.URL+"/el9" { + t.Errorf("bases = %v, want [%s]", bases, repo.URL+"/el9") + } + if layoutURL != repo.URL+"/el9" { + t.Errorf("layout URL = %q, want %q", layoutURL, repo.URL+"/el9") + } + for _, u := range probed { + if strings.Contains(u, "repodata") { + t.Errorf("probe folded a repository path into the query: %s", u) + } + } +} + +// TestResolveSourceMetalink verifies metalink documents expand into +// multi-mirror sources for RPM repositories. +func TestResolveSourceMetalink(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + metalink := `` + + `http://127.0.0.1:1/dead/repodata/repomd.xml` + + `` + srv.URL + `/el9/repodata/repomd.xml` + + `` + testrepos.WriteFile(t, filepath.Join(www, "metalink.xml"), []byte(metalink)) + + src, layoutURL, err := resolveSource(context.Background(), srv.URL+"/metalink.xml", RepoRPM) + if err != nil { + t.Fatal(err) + } + if len(src.Bases()) != 2 || src.Bases()[1] != srv.URL+"/el9" { + t.Errorf("bases = %v", src.Bases()) + } + if layoutURL != "http://127.0.0.1:1/dead" { + t.Errorf("layout URL = %q", layoutURL) + } + + // The dead first mirror must fail over to the live one. + resp, err := src.Get(context.Background(), "repodata/repomd.xml", fetch.GetOptions{}) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} diff --git a/mirror/rpm.go b/mirror/rpm.go new file mode 100644 index 0000000..6372293 --- /dev/null +++ b/mirror/rpm.go @@ -0,0 +1,338 @@ +package mirror + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// repoMD is the partial XML schema for a yum repomd.xml file. Only the +// fields the sync needs are decoded; unknown elements are ignored so +// upstream additions do not break parsing. +type repoMD struct { + Data []repoData `xml:"data"` +} + +// repoData is one metadata file referenced from repomd.xml. +type repoData struct { + Type string `xml:"type,attr"` + Checksum repoChecksum `xml:"checksum"` + Location repoLocation `xml:"location"` + Size int64 `xml:"size"` +} + +// repoChecksum is a checksum element with its algorithm type. +type repoChecksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +// repoLocation captures the relative href of a metadata or package file. +type repoLocation struct { + Href string `xml:"href,attr"` +} + +// rpmPackage is the minimal package element decoded from primary metadata, +// carrying the location, checksum, and size needed to mirror the file. +type rpmPackage struct { + Checksum repoChecksum `xml:"checksum"` + Size rpmSize `xml:"size"` + Location repoLocation `xml:"location"` +} + +// rpmSize carries the on-disk package size from primary metadata. +type rpmSize struct { + Package int64 `xml:"package,attr"` +} + +// syncRPM synchronizes one RPM repository from src into destDir. +func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error { + keep := fetch.NewKeepSet(opts.Prune) + miss := opts.newMissing(destDir) + tr := newTrace(opts) + ctx = tr.track(ctx) + + // Stage repomd.xml so clients of a live tree keep a consistent view of + // the old repository until everything else is in place. + repomdDst, err := fetch.LocalJoin(destDir, "repodata/repomd.xml") + if err != nil { + return err + } + if _, err := fetch.File(ctx, src, "repodata/repomd.xml", repomdDst, nil, true, false); err != nil { + return fmt.Errorf("fetch repomd.xml: %w", err) + } + md, err := readRepomd(fetch.StagedOrFinal(repomdDst)) + if err != nil { + return err + } + + // Download the metadata files referenced by repomd. Their names embed + // content hashes, so they can be written directly into place. A dry run + // only fetches the indexes that must be parsed, staged for discard. + var jobs, planned []fetch.Job + var primary *repoData + var deltaMDs []*repoData + var parseDsts []string + for i, data := range md.Data { + if data.Location.Href == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, data.Location.Href) + if err != nil { + return err + } + job := fetch.Job{ + ReqPath: data.Location.Href, + Dst: dst, + Want: fetch.MakeExpect(data.Size, data.Checksum.Type, data.Checksum.Value), + } + parsed := false + switch data.Type { + case "primary": + primary = &md.Data[i] + parsed = true + case "prestodelta", "deltainfo": + deltaMDs = append(deltaMDs, &md.Data[i]) + parsed = true + } + if opts.DryRun { + if !parsed { + planned = append(planned, job) + continue + } + job.Stage = true + parseDsts = append(parseDsts, dst) + } + jobs = append(jobs, job) + } + fetch.PlanJobs(planned, opts.Verify, keep) + if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, nil); err != nil { + return fmt.Errorf("fetch repository metadata: %w", err) + } + if primary == nil { + return errors.New("repository metadata lists no primary index") + } + + // Collect package locations from the primary index, plus any delta + // packages referenced by prestodelta metadata. + primaryPath, err := fetch.LocalJoin(destDir, primary.Location.Href) + if err != nil { + return err + } + pkgs, err := readPrimary(fetch.StagedOrFinal(primaryPath)) + if err != nil { + return err + } + jobs = jobs[:0] + for _, pkg := range pkgs { + if pkg.Location.Href == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, pkg.Location.Href) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ + ReqPath: pkg.Location.Href, + Dst: dst, + Want: fetch.MakeExpect(pkg.Size.Package, pkg.Checksum.Type, pkg.Checksum.Value), + }) + } + for _, deltaMD := range deltaMDs { + deltaPath, err := fetch.LocalJoin(destDir, deltaMD.Location.Href) + if err != nil { + return err + } + deltas, err := readDeltas(fetch.StagedOrFinal(deltaPath)) + if err != nil { + return err + } + for _, delta := range deltas { + if delta.Filename == "" { + continue + } + dst, err := fetch.LocalJoin(destDir, delta.Filename) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ + ReqPath: delta.Filename, + Dst: dst, + Want: fetch.MakeExpect(delta.Size, delta.Checksum.Type, delta.Checksum.Value), + }) + } + } + log.WithField("packages", len(jobs)).Info("Synchronizing packages.") + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + + // A dry run discards the indexes it staged for parsing. + for _, dst := range parseDsts { + _ = os.Remove(dst + fetch.StagedSuffix) + } + + // Stage the optional signature material so it is published together + // with the repomd.xml it signs, never beside the previous index. + var sigDsts []string + for _, extra := range []string{"repodata/repomd.xml.asc", "repodata/repomd.xml.key"} { + dst, err := fetch.LocalJoin(destDir, extra) + if err != nil { + return err + } + _, err = fetch.File(ctx, src, extra, dst, nil, true, false) + switch { + case err == nil: + sigDsts = append(sigDsts, dst) + case errors.Is(err, fetch.ErrNotFound): + // The upstream dropped the signature; drop the local copy so + // a stale signature is never served beside a new repomd.xml. + if !opts.DryRun { + fetch.RemoveStale(dst) + } + default: + return fmt.Errorf("fetch %s: %w", extra, err) + } + } + + // Promote the signatures and then repomd.xml so the published metadata + // chain is complete. + for _, dst := range sigDsts { + if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { + return err + } + if _, err := os.Stat(dst); err == nil { + keep.Add(dst) + } + } + if err := fetch.PromoteOrDiscard(repomdDst, opts.DryRun); err != nil { + return err + } + keep.Add(repomdDst) + + // Publish the traces, upstream's included, before pruning so the keep + // set covers them. + tr.publish(ctx, destDir, src, keep, opts) + + // Remove files that are no longer part of the repository. + if opts.Prune { + fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) + } + + // The metadata is published either way; missing packages only decide + // whether the run reports itself as failed. + return miss.Finish() +} + +// rpmDelta is one delta package referenced from prestodelta metadata. +type rpmDelta struct { + Filename string `xml:"filename"` + Size int64 `xml:"size"` + Checksum repoChecksum `xml:"checksum"` +} + +// readDeltas streams prestodelta metadata and collects each delta package +// reference so drpm files are mirrored alongside full packages. +func readDeltas(filename string) ([]rpmDelta, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + dec := xml.NewDecoder(r) + var deltas []rpmDelta + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse delta metadata %s: %w", filename, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "delta" { + continue + } + var delta rpmDelta + if err := dec.DecodeElement(&delta, &start); err != nil { + return nil, fmt.Errorf("parse delta entry %s: %w", filename, err) + } + if delta.Filename != "" { + deltas = append(deltas, delta) + } + } + return deltas, nil +} + +// readRepomd decodes a repomd.xml file. +func readRepomd(filename string) (*repoMD, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + var md repoMD + if err := xml.NewDecoder(f).Decode(&md); err != nil { + return nil, fmt.Errorf("parse repomd %s: %w", filename, err) + } + return &md, nil +} + +// readPrimary streams a primary metadata index and collects each package +// entry without loading the whole document into memory. +func readPrimary(filename string) ([]rpmPackage, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r, closeFn, err := fetch.Decompressor(f, filename) + if err != nil { + return nil, err + } + if closeFn != nil { + defer closeFn() + } + + dec := xml.NewDecoder(r) + var pkgs []rpmPackage + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse primary metadata %s: %w", filename, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg rpmPackage + if err := dec.DecodeElement(&pkg, &start); err != nil { + return nil, fmt.Errorf("parse package entry %s: %w", filename, err) + } + if pkg.Location.Href != "" { + pkgs = append(pkgs, pkg) + } + } + return pkgs, nil +} diff --git a/mirror/rpm_test.go b/mirror/rpm_test.go new file mode 100644 index 0000000..99c53d7 --- /dev/null +++ b/mirror/rpm_test.go @@ -0,0 +1,171 @@ +package mirror + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// syncRPMFixture builds a served RPM repository, synchronizes it, and +// returns the fixture directory, repository URL, and local repository path. +func syncRPMFixture(t *testing.T, opts *Options) (string, string, string, map[string][]byte) { + t.Helper() + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + pkgs := testrepos.BuildRPMRepo(t, repoDir) + srv := testrepos.ServeDir(t, www) + repoURL := srv.URL + "/repos/el9" + + opts.Type = RepoRPM + if opts.Destination == "" { + opts.Destination = t.TempDir() + } + if opts.Workers == 0 { + opts.Workers = 2 + } + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + return repoDir, repoURL, filepath.Join(opts.Destination, "repos", "el9"), pkgs +} + +// TestSyncRPM verifies a full synchronization mirrors packages and metadata +// with no staged leftovers. +func TestSyncRPM(t *testing.T) { + _, _, local, pkgs := syncRPMFixture(t, &Options{}) + for name, data := range pkgs { + got, err := os.ReadFile(filepath.Join(local, "Packages", name)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Errorf("package %s content mismatch", name) + } + } + if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml")); err != nil { + t.Error("repomd.xml not promoted:", err) + } + if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml.asc")); err != nil { + t.Error("repomd.xml.asc not promoted:", err) + } + drpm, err := os.ReadFile(filepath.Join(local, "drpms", "foo-0.9_1.0-1.x86_64.drpm")) + if err != nil { + t.Error("delta package not mirrored:", err) + } else if !bytes.Equal(drpm, testrepos.RPMDelta()) { + t.Error("delta package content mismatch") + } + staged, _ := filepath.Glob(filepath.Join(local, "repodata", "*"+fetch.StagedSuffix)) + if len(staged) != 0 { + t.Errorf("staged leftovers remain: %v", staged) + } +} + +// TestSyncRPMVerify verifies size-only skipping keeps same-size corruption +// while the verify option repairs it. +func TestSyncRPMVerify(t *testing.T) { + opts := &Options{} + repoDir, repoURL, local, _ := syncRPMFixture(t, opts) + _ = repoDir + + // Corrupt a local package without changing its size. + victim := filepath.Join(local, "Packages", "foo-1.0-1.x86_64.rpm") + corrupt := bytes.Repeat([]byte("oof"), 500) + if err := os.WriteFile(victim, corrupt, 0644); err != nil { + t.Fatal(err) + } + + // A plain re-run trusts the matching size and keeps the corruption. + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(victim) + if !bytes.Equal(got, corrupt) { + t.Error("size-only run unexpectedly rewrote the file") + } + + // A verify run re-hashes local files and repairs the corruption. + opts.Verify = true + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + got, _ = os.ReadFile(victim) + if !bytes.Equal(got, bytes.Repeat([]byte("foo"), 500)) { + t.Error("verify run did not repair the corruption") + } +} + +// TestSyncRPMPrune verifies stale files are removed only when pruning is +// enabled. +func TestSyncRPMPrune(t *testing.T) { + opts := &Options{} + _, repoURL, local, _ := syncRPMFixture(t, opts) + + stale := filepath.Join(local, "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Error("file pruned without the prune option") + } + + opts.Prune = true + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale file survived pruning") + } +} + +// TestSyncRPMDroppedSignature verifies a signature the upstream stops +// serving is removed locally instead of lingering beside a new repomd.xml. +func TestSyncRPMDroppedSignature(t *testing.T) { + opts := &Options{} + repoDir, repoURL, local, _ := syncRPMFixture(t, opts) + + ascLocal := filepath.Join(local, "repodata", "repomd.xml.asc") + if _, err := os.Stat(ascLocal); err != nil { + t.Fatal("signature not mirrored:", err) + } + if err := os.Remove(filepath.Join(repoDir, "repodata", "repomd.xml.asc")); err != nil { + t.Fatal(err) + } + + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ascLocal); !os.IsNotExist(err) { + t.Error("stale signature survived after the upstream dropped it") + } +} + +// TestSyncRPMChecksumMismatch verifies a package whose served content does +// not match the published checksum fails the synchronization. +func TestSyncRPMChecksumMismatch(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + testrepos.BuildRPMRepo(t, repoDir) + srv := testrepos.ServeDir(t, www) + + // Replace a served package with same-size different content after the + // metadata was generated. + victim := filepath.Join(repoDir, "Packages", "foo-1.0-1.x86_64.rpm") + if err := os.WriteFile(victim, bytes.Repeat([]byte("fox"), 500), 0644); err != nil { + t.Fatal(err) + } + + opts := &Options{Type: RepoRPM, Destination: t.TempDir(), Workers: 2} + err := syncOne(context.Background(), srv.URL+"/repos/el9", opts.Type, opts) + if err == nil || !strings.Contains(err.Error(), "checksum") { + t.Errorf("expected checksum mismatch error, got %v", err) + } +} diff --git a/mirror/sync.go b/mirror/sync.go new file mode 100644 index 0000000..4c22195 --- /dev/null +++ b/mirror/sync.go @@ -0,0 +1,402 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// destLock is the exclusive hold a run has on its destination directory. +type destLock struct { + file *os.File + path string +} + +// lockAttempts bounds the retries for a lock file removed between opening +// it and locking it. That window is a few instructions wide and only opens +// when another run finishes inside it, so a handful of attempts always +// resolves it. +const lockAttempts = 5 + +// lockDestination takes an exclusive lock under the destination directory +// so overlapping runs, such as from cron, cannot race on the same tree. +// The caller releases it with release. +func lockDestination(dest string) (*destLock, error) { + if err := os.MkdirAll(dest, 0755); err != nil { + return nil, err + } + name := filepath.Join(dest, fetch.LockFileName) + for attempt := 0; attempt < lockAttempts; attempt++ { + f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("destination %s is locked by another repo-sync run", dest) + } + + // The lock only guards the tree while the file holding it is still + // the one at the path. A run that finished between the open and the + // lock above took its file with it, so this lock is on a file the + // next run will never look at; open the replacement and lock that. + held, herr := f.Stat() + current, cerr := os.Stat(name) + if herr == nil && cerr == nil && os.SameFile(held, current) { + return &destLock{file: f, path: name}, nil + } + f.Close() + log.WithField("path", name).Debug("Destination lock file was replaced while locking; retrying.") + } + return nil, fmt.Errorf("failed to lock destination %s", dest) +} + +// release removes the lock file and then drops the lock, in that order, so +// the file is never removed out from under whichever run takes it next. It +// leaves nothing behind in the published tree, including when a run ends +// early on an interrupt. +func (l *destLock) release() { + if err := os.Remove(l.path); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.WithError(err).WithField("path", l.path).Warn("Failed to remove the destination lock file.") + } + l.file.Close() +} + +// Summary reports what a whole run transferred, for callers that act on the +// result rather than read the log. A repository that failed part way through +// still counts what it transferred before failing, as those transfers changed +// the destination all the same. +type Summary struct { + Repositories int + Failed int + DryRun bool + Fetched int64 + FetchedBytes int64 + Unchanged int64 + Pruned int64 + Planned int64 + PlannedBytes int64 + // Missing counts the files repository metadata listed that the upstream + // did not serve, whether or not the run tolerated their absence. + Missing int64 +} + +// Changed reports whether the run altered the destination. A dry run reports +// what it would have done in the same counters, so it never counts as a +// change. +func (s *Summary) Changed() bool { + if s.DryRun { + return false + } + return s.Fetched > 0 || s.Pruned > 0 +} + +// add folds one repository's counters into the run totals. +func (s *Summary) add(c *fetch.Counters) { + s.Fetched += c.Fetched.Load() + s.FetchedBytes += c.FetchedBytes.Load() + s.Unchanged += c.Unchanged.Load() + s.Pruned += c.Pruned.Load() + s.Planned += c.Planned.Load() + s.PlannedBytes += c.PlannedBytes.Load() + s.Missing += c.Missing.Load() +} + +// Sync synchronizes every configured repository URL into the destination, +// continuing after per-repository failures so one bad repository does not +// block the rest. The returned summary covers every repository reached, +// including on the error paths, so a caller can still tell whether a failed +// run changed anything. +func Sync(ctx context.Context, opts *Options) (*Summary, error) { + fetch.Reload() + summary := &Summary{DryRun: opts.DryRun} + + // Hold the destination for the whole run, releasing it however the run + // ends so no lock file is left in the published tree. + lock, err := lockDestination(opts.Destination) + if err != nil { + return summary, err + } + defer lock.release() + + // Resolve the configured URLs into the repositories, and the loose + // files, this run works on. + repos, files, err := resolveTargets(ctx, opts) + if err != nil { + return summary, err + } + if len(repos) == 0 && len(files) == 0 { + return summary, errors.New("no repositories to synchronize") + } + + // Synchronize each repository in turn. Every repository's counters are + // folded into the run totals before the next one resets them, whatever + // its outcome. + for _, repo := range repos { + if err := ctx.Err(); err != nil { + return summary, err + } + log.WithFields(log.Fields{"url": repo.url, "type": repo.typ}).Info("Synchronizing repository.") + fetch.Stats.Reset() + start := time.Now() + err := syncOne(ctx, repo.url, repo.typ, opts) + summary.Repositories++ + summary.add(fetch.Stats) + if err != nil { + summary.Failed++ + if errors.Is(err, context.Canceled) { + return summary, err + } + log.WithError(err).WithField("url", repo.url).Error("Repository synchronization failed.") + continue + } + logSummary(repo.url, opts, time.Since(start)) + } + + // Mirror the loose files last: a repository sharing a directory with + // them prunes against its own metadata, so files fetched before it ran + // would be removed as stale. + var fileErr error + if len(files) > 0 && ctx.Err() == nil { + log.WithField("files", len(files)).Info("Synchronizing files.") + fetch.Stats.Reset() + fileErr = syncFiles(ctx, files, opts) + summary.add(fetch.Stats) + if errors.Is(fileErr, context.Canceled) { + return summary, fileErr + } + if fileErr != nil { + log.WithError(fileErr).Error("File synchronization failed.") + } + } + + var errs []error + if summary.Failed > 0 { + errs = append(errs, fmt.Errorf("%d of %d repositories failed to synchronize", summary.Failed, len(repos))) + } + if fileErr != nil { + errs = append(errs, fileErr) + } + return summary, errors.Join(errs...) +} + +// resolveTargets expands the configured URLs into the repositories to +// synchronize and the loose files to mirror beside them. Discovery crawls +// for both; without it each URL is a repository, whose format is detected +// when the run did not name one. +func resolveTargets(ctx context.Context, opts *Options) ([]repoTarget, []fileTarget, error) { + if opts.Discover { + return discoverTargets(ctx, opts) + } + + repos := make([]repoTarget, 0, len(opts.URLs)) + for _, repoURL := range opts.URLs { + typ := opts.Type + if typ == "" { + detected, err := detectType(ctx, repoURL, opts.discoverTypes()) + if err != nil { + return nil, nil, err + } + log.WithFields(log.Fields{"url": repoURL, "type": detected}).Info("Detected repository type.") + typ = detected + } + repos = append(repos, repoTarget{url: repoURL, typ: typ}) + } + return repos, nil, nil +} + +// discoverTargets expands each configured URL into the repositories and +// loose files below it. Crawling a vendor archive costs hundreds of +// directory listings, so a recent crawl's results are reused instead, and +// the record of what was found is what tells this run which repositories +// the upstream has dropped since. +func discoverTargets(ctx context.Context, opts *Options) ([]repoTarget, []fileTarget, error) { + cache := loadDiscoverCache(opts.Destination) + settings := opts.discoverOptions() + now := time.Now() + + var repos []repoTarget + var files []fileTarget + for _, base := range opts.URLs { + found, cached := cache.reuse(base, settings, opts.DiscoverCache, now) + if !cached { + var err error + if found, err = discoverRepos(ctx, base, opts); err != nil { + return nil, nil, fmt.Errorf("discover repositories under %s: %w", base, err) + } + log.WithFields(log.Fields{ + "url": base, + "repositories": len(found.repos), + "files": len(found.files), + }).Info("Discovered repositories.") + + // Withdrawing content is as destructive as pruning, so it is + // gated on the same flag and waits out the same grace period. + evictDiscovered(cache.update(base, settings, found, now, opts), opts) + } + repos = append(repos, found.repos...) + files = append(files, found.files...) + } + cache.save(opts.DryRun) + return repos, files, nil +} + +// syncFiles mirrors the loose files a crawl matched. They sit outside any +// repository and carry no published checksum, so they revalidate against +// their modification time, and one withdrawn between the crawl and the +// fetch is not a failure. Nothing here prunes: the files are not described +// by any metadata that could say one no longer belongs. +func syncFiles(ctx context.Context, files []fileTarget, opts *Options) error { + // Group by directory so each group shares one source and its request + // paths stay relative to it, keeping the crawl's order. + var dirs []string + groups := map[string][]fileTarget{} + for _, file := range files { + if _, ok := groups[file.dirURL]; !ok { + dirs = append(dirs, file.dirURL) + } + groups[file.dirURL] = append(groups[file.dirURL], file) + } + + for _, dirURL := range dirs { + if err := ctx.Err(); err != nil { + return err + } + destDir, err := opts.repoDest(dirURL) + if err != nil { + return err + } + src := fetch.NewSource([]string{dirURL}) + var jobs []fetch.Job + for _, file := range groups[dirURL] { + dst, err := fetch.LocalJoin(destDir, file.name) + if err != nil { + return err + } + jobs = append(jobs, fetch.Job{ReqPath: url.PathEscape(file.name), Dst: dst, Optional: true}) + } + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, nil) + continue + } + if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, nil, nil); err != nil { + return fmt.Errorf("fetch files under %s: %w", dirURL, err) + } + log.WithFields(log.Fields{"url": dirURL, "files": len(jobs), "destination": destDir}).Debug("Mirrored files.") + } + return nil +} + +// logSummary reports one repository's transfer activity after a successful +// synchronization. +func logSummary(repoURL string, opts *Options, elapsed time.Duration) { + fields := log.Fields{ + "url": repoURL, + "fetched": fetch.Stats.Fetched.Load(), + "transferred": fetch.FormatBytes(fetch.Stats.FetchedBytes.Load()), + "unchanged": fetch.Stats.Unchanged.Load(), + "duration": elapsed.Round(time.Millisecond).String(), + } + // Report tolerated absences, so a repository that synchronized despite + // them does not look completely clean in the log. + if missing := fetch.Stats.Missing.Load(); missing > 0 { + fields["missing"] = missing + } + if opts.DryRun { + fields["would_fetch"] = fetch.Stats.Planned.Load() + fields["would_transfer"] = fetch.FormatBytes(fetch.Stats.PlannedBytes.Load()) + if opts.Prune { + fields["would_prune"] = fetch.Stats.Pruned.Load() + } + log.WithFields(fields).Info("Dry run complete.") + return + } + if opts.Prune { + fields["pruned"] = fetch.Stats.Pruned.Load() + } + log.WithFields(fields).Info("Repository synchronized.") +} + +// syncOne resolves the source for a single repository URL and runs the +// format-specific synchronization. Destination paths derive from the layout +// URL, so a mirror list maps to the first mirror's path rather than the +// list's own URL. +func syncOne(ctx context.Context, repoURL string, typ RepoType, opts *Options) error { + // Pin this repository's format on a copy, so a run covering several + // formats reports each repository as what it is and never mutates the + // caller's options. + repoOpts := *opts + repoOpts.Type = typ + opts = &repoOpts + + src, layoutURL, err := resolveSource(ctx, repoURL, typ) + if err != nil { + return err + } + switch typ { + case RepoRPM: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncRPM(ctx, src, destDir, opts) + case RepoDeb: + return syncDeb(ctx, src, layoutURL, opts) + case RepoArch: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncArch(ctx, src, layoutURL, destDir, opts) + case RepoApk: + destDir, err := opts.repoDest(layoutURL) + if err != nil { + return err + } + log.WithField("destination", destDir).Debug("Resolved repository destination.") + return syncApk(ctx, src, destDir, opts) + default: + return fmt.Errorf("unsupported repository type %q", typ) + } +} + +// SyncInto synchronizes one repository into an explicit destination +// directory, bypassing URL-derived destination mapping. It exists for the +// mirror server, whose destinations follow request paths rather than +// upstream URL paths. For deb the destination is the archive root +// directory; for other types it is the repository directory itself. +// Concurrent callers interleave into fetch.Stats, so the counters carry no +// per-repository meaning on this path and are neither reset nor reported. +// A trace enabled here still reports correct totals: it accumulates its own +// counters for the duration of the crawl rather than reading the shared +// collector. +func SyncInto(ctx context.Context, typ RepoType, src *fetch.Source, repoURL, dest string, opts *Options) error { + // Copy before the deb path pins destBase so the caller's Options is + // never mutated, keeping the struct safe to reuse or share. + serverOpts := *opts + opts = &serverOpts + switch typ { + case RepoRPM: + return syncRPM(ctx, src, dest, opts) + case RepoDeb: + opts.destBase = dest + return syncDeb(ctx, src, repoURL, opts) + case RepoArch: + return syncArch(ctx, src, repoURL, dest, opts) + case RepoApk: + return syncApk(ctx, src, dest, opts) + default: + return fmt.Errorf("unsupported repository type %q", typ) + } +} diff --git a/mirror/sync_test.go b/mirror/sync_test.go new file mode 100644 index 0000000..130b006 --- /dev/null +++ b/mirror/sync_test.go @@ -0,0 +1,416 @@ +package mirror + +import ( + "context" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// countFiles returns the number of regular files under root. +func countFiles(t *testing.T, root string) int { + t.Helper() + count := 0 + err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + count++ + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return count +} + +// TestDryRun verifies a dry run leaves the destination unchanged: no +// packages downloaded, no metadata published, and no files pruned. +func TestDryRun(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + // A dry run against an empty destination writes no files at all. + dest := t.TempDir() + opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, DryRun: true, Prune: true} + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + if n := countFiles(t, dest); n != 0 { + t.Errorf("dry run left %d files in the destination", n) + } + + // A real run works, and a later dry run with pruning keeps stale files. + opts.DryRun = false + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "el9", "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + opts.DryRun = true + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Error("dry run pruned a file:", err) + } +} + +// TestPruneGrace verifies stale files wait out the grace period across +// runs before being removed. +func TestPruneGrace(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, Prune: true, PruneGrace: time.Hour} + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + + local := filepath.Join(dest, "el9") + stale := filepath.Join(local, "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + + // The first run only starts the grace period. + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Fatal("file pruned before its grace period expired:", err) + } + statePath := filepath.Join(local, fetch.PruneStateName) + if _, err := os.Stat(statePath); err != nil { + t.Fatal("prune state not recorded:", err) + } + + // A second run within the grace period still keeps the file. + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Fatal("file pruned within its grace period:", err) + } + + // Age the recorded timestamp past the grace period; the next run + // prunes the file and clears the bookkeeping. + data, err := os.ReadFile(statePath) + if err != nil { + t.Fatal(err) + } + state := map[string]time.Time{} + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + for rel := range state { + state[rel] = time.Now().Add(-2 * time.Hour) + } + data, err = json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(statePath, data, 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale file survived past its grace period") + } + if _, err := os.Stat(statePath); err == nil { + t.Error("prune state lingered after all grace periods resolved") + } +} + +// TestLockDestination verifies a held destination lock blocks a second run, +// is released on release, and leaves no lock file behind in the tree. +func TestLockDestination(t *testing.T) { + dest := t.TempDir() + lock, err := lockDestination(dest) + if err != nil { + t.Fatal(err) + } + name := filepath.Join(dest, fetch.LockFileName) + if _, err := os.Stat(name); err != nil { + t.Error("no lock file while the lock is held:", err) + } + if _, err := lockDestination(dest); err == nil { + t.Error("second lock unexpectedly succeeded") + } + + lock.release() + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Error("lock file survived the run that took it") + } + lock, err = lockDestination(dest) + if err != nil { + t.Error("lock not released:", err) + } else { + lock.release() + } +} + +// TestLockDestinationReplaced verifies a lock taken on a file that is no +// longer the one at the path is not accepted: a run that released between +// the open and the lock takes its file with it, and the lock left over +// guards nothing the next run would consult. +func TestLockDestinationReplaced(t *testing.T) { + dest := t.TempDir() + name := filepath.Join(dest, fetch.LockFileName) + + // Stand in for the released run by replacing the lock file with a new + // one, which is the state a racing run would find. + stale, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + defer stale.Close() + if err := os.Remove(name); err != nil { + t.Fatal(err) + } + replacement, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + replacement.Close() + + lock, err := lockDestination(dest) + if err != nil { + t.Fatal(err) + } + defer lock.release() + held, err := lock.file.Stat() + if err != nil { + t.Fatal(err) + } + current, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(held, current) { + t.Error("locked a file that is no longer the destination's lock file") + } +} + +// TestSyncSummary verifies a run reports what it changed: the first run +// fetches, a repeat run finds everything unchanged, and a removed upstream +// file is counted as pruned. +func TestSyncSummary(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{ + Type: RepoRPM, + URLs: []string{srv.URL + "/el9"}, + Destination: dest, + Workers: 2, + Prune: true, + } + + // The first run populates the destination, so everything it fetched is a + // change. + summary, err := Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 1 || summary.Failed != 0 { + t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed) + } + if summary.Fetched == 0 { + t.Error("first run reported nothing fetched") + } + if summary.FetchedBytes == 0 { + t.Error("first run reported no bytes fetched") + } + if !summary.Changed() { + t.Error("first run did not report a change") + } + + // Repeating the run transfers nothing, which is what a hook keys off to + // stay idle. + summary, err = Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Fetched != 0 || summary.Pruned != 0 { + t.Errorf("repeat run fetched %d and pruned %d, want 0 and 0", summary.Fetched, summary.Pruned) + } + if summary.Unchanged == 0 { + t.Error("repeat run reported nothing unchanged") + } + if summary.Changed() { + t.Error("repeat run reported a change") + } + + // A local file the upstream never served is pruned, which counts as a + // change even though nothing was fetched. + stale := filepath.Join(dest, "el9", "Packages", "stale.rpm") + if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil { + t.Fatal(err) + } + summary, err = Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Pruned != 1 { + t.Errorf("pruned = %d, want 1", summary.Pruned) + } + if !summary.Changed() { + t.Error("pruning run did not report a change") + } +} + +// TestSyncSummaryDryRun verifies a dry run reports its planned work without +// ever claiming the mirror changed. +func TestSyncSummaryDryRun(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + opts := &Options{ + Type: RepoRPM, + URLs: []string{srv.URL + "/el9"}, + Destination: t.TempDir(), + Workers: 2, + DryRun: true, + } + summary, err := Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Planned == 0 { + t.Error("dry run planned nothing") + } + if summary.PlannedBytes == 0 { + t.Error("dry run planned no bytes") + } + // A dry run still fetches metadata to plan against, but none of it + // reaches the destination, so the run must never read as a change. + if summary.Changed() { + t.Error("dry run reported a change") + } +} + +// TestSyncSummaryFailure verifies a run that fails still reports the totals +// it accumulated, so a caller can tell a failed run apart from an idle one. +func TestSyncSummaryFailure(t *testing.T) { + www := t.TempDir() + srv := testrepos.ServeDir(t, www) + + opts := &Options{ + Type: RepoRPM, + URLs: []string{srv.URL + "/missing"}, + Destination: t.TempDir(), + Workers: 2, + } + summary, err := Sync(context.Background(), opts) + if err == nil { + t.Fatal("expected a failure syncing a repository that does not exist") + } + if summary == nil { + t.Fatal("no summary returned for a failed run") + } + if summary.Repositories != 1 || summary.Failed != 1 { + t.Errorf("repositories = %d, failed = %d, want 1 and 1", summary.Repositories, summary.Failed) + } + if summary.Changed() { + t.Error("a run that transferred nothing reported a change") + } +} + +// TestSyncDiscoverMixed verifies one run covers an upstream publishing more +// than one repository format, skips the trees it was told to exclude, and +// mirrors the loose files it was told to include beside the repositories. +func TestSyncDiscoverMixed(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9")) + testrepos.BuildDebRepo(t, filepath.Join(www, "apt")) + testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles15")) + testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-test"), []byte("key")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("loose")) + testrepos.WriteFile(t, filepath.Join(www, "yum", "notes.txt"), []byte("ignored")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{ + URLs: []string{srv.URL}, + Destination: dest, + Discover: true, + DiscoverDepth: 4, + Exclude: []string{"sles*"}, + IncludeFiles: []string{"*.rpm", "RPM-GPG-KEY-*"}, + Workers: 2, + } + summary, err := Sync(context.Background(), opts) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 2 || summary.Failed != 0 { + t.Errorf("repositories = %d, failed = %d, want 2 and 0", summary.Repositories, summary.Failed) + } + + // Both formats landed, each under its own upstream path. + for _, rel := range []string{ + filepath.Join("yum", "el9", "repodata", "repomd.xml"), + filepath.Join("apt", "dists", "test", "Release"), + filepath.Join("apt", "pool", "main", "h", "hello", "hello_1.0_amd64.deb"), + "RPM-GPG-KEY-test", + filepath.Join("yum", "loose-1.0.rpm"), + } { + if _, err := os.Stat(filepath.Join(dest, rel)); err != nil { + t.Errorf("missing %s: %v", rel, err) + } + } + + // The excluded tree was never crawled, and a file no pattern matched + // was never fetched. + for _, rel := range []string{ + filepath.Join("yum", "sles15"), + filepath.Join("yum", "notes.txt"), + } { + if _, err := os.Stat(filepath.Join(dest, rel)); err == nil { + t.Errorf("%s was synchronized despite not being asked for", rel) + } + } +} + +// TestSyncDetectType verifies a repository URL given without a type is +// synchronized as whatever its metadata identifies it to be. +func TestSyncDetectType(t *testing.T) { + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "el9")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + summary, err := Sync(context.Background(), &Options{ + URLs: []string{srv.URL + "/el9"}, + Destination: dest, + Workers: 2, + }) + if err != nil { + t.Fatal(err) + } + if summary.Repositories != 1 || summary.Failed != 0 { + t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed) + } + if _, err := os.Stat(filepath.Join(dest, "el9", "repodata", "repomd.xml")); err != nil { + t.Error("repository not synchronized:", err) + } +} diff --git a/mirror/trace.go b/mirror/trace.go new file mode 100644 index 0000000..90b62c7 --- /dev/null +++ b/mirror/trace.go @@ -0,0 +1,275 @@ +package mirror + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + log "github.com/sirupsen/logrus" +) + +// TraceDir is the directory, relative to a repository, that trace files are +// published in. Debian archives established the location and downstream +// mirror checkers look for it there. +const TraceDir = "project/trace" + +// TraceInfo describes the mirror a trace file is written for. It carries +// the same fields as the configuration's trace section, kept as a separate +// type so callers populate it from wherever they hold those details rather +// than this package taking a configuration struct as part of its API. +type TraceInfo struct { + Host string + Maintainer string + Sponsor string + Country string + Location string + Throughput string +} + +// traceRun records when a repository's synchronization began, and the +// transfer it accounted for, so the trace can report both. A nil run writes +// nothing, which is how tracing stays off without a condition at every call +// site. +type traceRun struct { + started time.Time + stats *fetch.Counters +} + +// newTrace starts timing a repository sync when tracing is enabled. +func newTrace(opts *Options) *traceRun { + if !opts.Trace { + return nil + } + return &traceRun{started: time.Now(), stats: &fetch.Counters{}} +} + +// track installs the run's own transfer counters on ctx so the trace +// reports what this synchronization moved rather than what the shared +// collector holds. Without it the mirror server's concurrent crawls would +// report each other's bytes. +func (t *traceRun) track(ctx context.Context) context.Context { + if t == nil { + return ctx + } + return fetch.WithStats(ctx, t.stats) +} + +// publish brings a repository's trace directory up to date: upstream's own +// traces are mirrored first, then this run's trace is written, so a name +// collision resolves in favour of the file describing this mirror. +func (t *traceRun) publish(ctx context.Context, root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) { + if t == nil { + return + } + t.mirrorUpstream(ctx, root, src, keep, opts) + t.write(root, src, keep, opts) +} + +// mirrorUpstream copies the traces upstream publishes into this mirror's +// trace directory, so a reader can follow the chain back through every +// mirror to the archive the content came from. Upstream may publish no +// traces at all or may not serve directory indexes, and neither is a +// failure of the synchronization, so problems here are logged and the run +// continues. +func (t *traceRun) mirrorUpstream(ctx context.Context, root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) { + base := src.Active() + if base == "" { + return + } + dirURL := base + "/" + TraceDir + "/" + listing, err := listDir(ctx, dirURL) + if err != nil { + log.WithError(err).WithField("url", dirURL).Debug("Upstream serves no trace directory.") + return + } + + // Skip the name this mirror traces under; write publishes that file + // itself, and upstream's copy of it describes a different mirror. + own, err := traceName(opts.TraceInfo.Host) + if err != nil { + own = "" + } + names := make([]string, 0, len(listing.files)) + for name := range listing.files { + if name == own || name == "." || name == ".." { + continue + } + names = append(names, name) + } + sort.Strings(names) + + dir := filepath.Join(root, filepath.FromSlash(TraceDir)) + var jobs []fetch.Job + for _, name := range names { + dst, err := fetch.LocalJoin(dir, name) + if err != nil { + log.WithError(err).WithField("name", name).Debug("Skipping upstream trace with an unusable name.") + continue + } + // Traces carry no checksums, so they revalidate against their + // modification time rather than being re-fetched every run, and a + // trace withdrawn between the listing and the fetch is not an error. + jobs = append(jobs, fetch.Job{ReqPath: TraceDir + "/" + name, Dst: dst, Optional: true}) + } + if len(jobs) == 0 { + return + } + if opts.DryRun { + fetch.PlanJobs(jobs, opts.Verify, keep) + return + } + if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, nil); err != nil { + log.WithError(err).WithField("url", dirURL).Warn("Failed to mirror upstream trace files.") + return + } + log.WithFields(log.Fields{"url": dirURL, "traces": len(jobs)}).Debug("Mirrored upstream trace files.") +} + +// traceName returns the file name a mirror is traced under. The value +// becomes a file name, so it is reduced to a single path element and +// rejected if nothing usable is left. +func traceName(host string) (string, error) { + name := filepath.Base(strings.TrimSpace(strings.ReplaceAll(host, "\\", "/"))) + if name == "." || name == ".." || name == "/" || name == "" { + return "", fmt.Errorf("invalid trace host %q", host) + } + return name, nil +} + +// write publishes the trace for a finished repository synchronization into +// root, and records it in keep so pruning does not treat the file as stale +// and remove it. On a dry run the file is left untouched but still kept, +// so the run does not report the existing trace as about to be pruned. +// +// A trace describes the mirror rather than the packages, so a failure to +// write one is logged and does not fail the synchronization that produced +// it. +func (t *traceRun) write(root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) { + if t == nil { + return + } + name, err := traceName(opts.TraceInfo.Host) + if err != nil { + log.WithError(err).Warn("Skipping trace file.") + return + } + dir := filepath.Join(root, filepath.FromSlash(TraceDir)) + file := filepath.Join(dir, name) + + // Keep the trace regardless of whether it is written now, so a dry run + // does not report an existing trace as stale. + keep.Add(file) + if opts.DryRun { + return + } + + if err := os.MkdirAll(dir, 0755); err != nil { + log.WithError(err).WithField("path", dir).Warn("Failed to create trace directory.") + return + } + content := t.render(src, opts) + // Write through a temporary file so a reader never sees a partial + // trace, and so an interrupted write cannot truncate the previous one. + tmp, err := os.CreateTemp(dir, ".trace-*") + if err != nil { + log.WithError(err).WithField("path", dir).Warn("Failed to create trace file.") + return + } + tmpName := tmp.Name() + if _, err := tmp.WriteString(content); err != nil { + tmp.Close() + os.Remove(tmpName) + log.WithError(err).WithField("path", tmpName).Warn("Failed to write trace file.") + return + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + log.WithError(err).WithField("path", tmpName).Warn("Failed to write trace file.") + return + } + if err := os.Chmod(tmpName, 0644); err != nil { + log.WithError(err).WithField("path", tmpName).Debug("Failed to set trace file mode.") + } + if err := os.Rename(tmpName, file); err != nil { + os.Remove(tmpName) + log.WithError(err).WithField("path", file).Warn("Failed to publish trace file.") + return + } + log.WithField("path", file).Debug("Wrote trace file.") +} + +// render builds the trace file's contents. The first line is the plain +// completion date, as the convention expects, followed by one field per +// line. Fields with no configured value are omitted rather than written +// empty. +func (t *traceRun) render(src *fetch.Source, opts *Options) string { + done := time.Now().UTC() + var b strings.Builder + + // The opening line matches date(1)'s default UTC output, which is what + // the convention's readers expect; _2 space-pads the day as it does. + fmt.Fprintf(&b, "%s\n", done.Format("Mon Jan _2 15:04:05 UTC 2006")) + fmt.Fprintf(&b, "Date: %s\n", done.Format(time.RFC1123Z)) + fmt.Fprintf(&b, "Date-Started: %s\n", t.started.UTC().Format(time.RFC1123Z)) + fmt.Fprintf(&b, "Creator: %s %s\n", cfg.Name, cfg.Version) + fmt.Fprintf(&b, "Running on host: %s\n", opts.TraceInfo.Host) + + for _, f := range []struct{ key, value string }{ + {"Maintainer", opts.TraceInfo.Maintainer}, + {"Sponsor", opts.TraceInfo.Sponsor}, + {"Country", opts.TraceInfo.Country}, + {"Location", opts.TraceInfo.Location}, + {"Throughput", opts.TraceInfo.Throughput}, + } { + if f.value != "" { + fmt.Fprintf(&b, "%s: %s\n", f.key, f.value) + } + } + + fmt.Fprintf(&b, "Repository type: %s\n", opts.Type) + if arch := traceArchitectures(opts); arch != "" { + fmt.Fprintf(&b, "Architectures: %s\n", arch) + } + + // Report the mirror actually fetched from. For a mirrorlist or metalink + // that is the mirror that answered, which is the useful fact and is not + // derivable from the configured URL. + if src != nil { + if active := src.Active(); active != "" { + fmt.Fprintf(&b, "Upstream-mirror: %s\n", active) + } + } + + elapsed := done.Sub(t.started.UTC()) + if elapsed < 0 { + elapsed = 0 + } + seconds := int64(elapsed.Round(time.Second) / time.Second) + // The run's own counters, not the shared collector, so a trace written + // by one of the server's concurrent crawls reports that crawl alone. + bytes := t.stats.FetchedBytes.Load() + fmt.Fprintf(&b, "Total bytes received: %d\n", bytes) + fmt.Fprintf(&b, "Total time spent syncing: %d\n", seconds) + if seconds > 0 { + fmt.Fprintf(&b, "Average rate: %d B/s\n", bytes/seconds) + } + return b.String() +} + +// traceArchitectures reports the architectures a run was restricted to, +// which only apt repositories can express. An unrestricted run mirrors +// whatever upstream publishes, so no claim is made. +func traceArchitectures(opts *Options) string { + if opts.Type != RepoDeb || len(opts.Architectures) == 0 { + return "" + } + arch := append([]string(nil), opts.Architectures...) + sort.Strings(arch) + return strings.Join(arch, " ") +} diff --git a/mirror/trace_test.go b/mirror/trace_test.go new file mode 100644 index 0000000..1b36e9a --- /dev/null +++ b/mirror/trace_test.go @@ -0,0 +1,453 @@ +package mirror + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" +) + +// traceOptions builds sync options with tracing enabled and a fixed set of +// mirror details, so tests assert on known values. +func traceOptions(typ RepoType, dest string) *Options { + return &Options{ + Type: typ, + Destination: dest, + Workers: 2, + Trace: true, + TraceInfo: TraceInfo{ + Host: "mirror.example.com", + Maintainer: "Test Maintainer ", + Sponsor: "Example Inc", + Country: "US", + Location: "Texas", + Throughput: "10Gb", + }, + } +} + +// traceField returns the value following a field prefix in a trace. +func traceField(t *testing.T, trace, prefix string) string { + t.Helper() + for _, line := range strings.Split(trace, "\n") { + if strings.HasPrefix(line, prefix) { + return strings.TrimPrefix(line, prefix) + } + } + t.Fatalf("trace has no %q field\ngot:\n%s", prefix, trace) + return "" +} + +// readTrace returns the trace file published under root. +func readTrace(t *testing.T, root string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(TraceDir), "mirror.example.com")) + if err != nil { + t.Fatal("reading trace:", err) + } + return string(data) +} + +// TestTraceName verifies a trace host is reduced to a single path element, +// so a configured value cannot direct the file outside the repository. +func TestTraceName(t *testing.T) { + cases := []struct { + host string + want string + ok bool + }{ + {"mirror.example.com", "mirror.example.com", true}, + {" mirror.example.com ", "mirror.example.com", true}, + {"../../etc/cron.d/evil", "evil", true}, + {"/etc/passwd", "passwd", true}, + {`..\..\windows`, "windows", true}, + {"sub/dir/host", "host", true}, + {"", "", false}, + {"..", "", false}, + {"/", "", false}, + } + for _, c := range cases { + got, err := traceName(c.host) + if c.ok && err != nil { + t.Errorf("traceName(%q) unexpected error: %v", c.host, err) + continue + } + if !c.ok { + if err == nil { + t.Errorf("traceName(%q) = %q, want error", c.host, got) + } + continue + } + if got != c.want { + t.Errorf("traceName(%q) = %q, want %q", c.host, got, c.want) + } + } +} + +// TestSyncTraceWritten verifies a trace lands in the repository directory +// carrying the configured mirror details and the upstream actually used. +func TestSyncTraceWritten(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + repoURL := srv.URL + "/alpine/v3.24/community/x86_64" + if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil { + t.Fatal(err) + } + + trace := readTrace(t, filepath.Join(dest, "alpine", "v3.24", "community", "x86_64")) + for _, want := range []string{ + "Date: ", + "Date-Started: ", + "Creator: repo-sync ", + "Running on host: mirror.example.com", + "Maintainer: Test Maintainer ", + "Sponsor: Example Inc", + "Country: US", + "Location: Texas", + "Throughput: 10Gb", + "Repository type: apk", + "Upstream-mirror: " + repoURL, + "Total bytes received: ", + "Total time spent syncing: ", + } { + if !strings.Contains(trace, want) { + t.Errorf("trace missing %q\ngot:\n%s", want, trace) + } + } + + // The first line is the plain completion date the convention expects, + // carrying no field name of its own and matching date(1)'s UTC output. + first, _, _ := strings.Cut(trace, "\n") + if strings.HasPrefix(first, "Date:") { + t.Errorf("first line should be the bare date, got %q", first) + } + const dateLayout = "Mon Jan _2 15:04:05 UTC 2006" + parsed, err := time.Parse(dateLayout, first) + if err != nil { + t.Fatalf("first line %q is not a date(1)-style UTC date: %v", first, err) + } + // Parsing alone is too lenient about spacing to catch a malformed day + // field, so require the line to survive a round trip unchanged. + if canonical := parsed.Format(dateLayout); canonical != first { + t.Errorf("first line %q is not canonically formatted, want %q", first, canonical) + } +} + +// TestSyncTraceOmitsUnsetFields verifies fields with no configured value +// are left out rather than written empty. +func TestSyncTraceOmitsUnsetFields(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{ + Type: RepoApk, Destination: dest, Workers: 2, Trace: true, + TraceInfo: TraceInfo{Host: "mirror.example.com"}, + } + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + + trace := readTrace(t, filepath.Join(dest, "repo")) + for _, absent := range []string{"Maintainer:", "Sponsor:", "Country:", "Location:", "Throughput:"} { + if strings.Contains(trace, absent) { + t.Errorf("trace contains unset field %q\ngot:\n%s", absent, trace) + } + } + if !strings.Contains(trace, "Running on host: mirror.example.com") { + t.Errorf("trace missing host\ngot:\n%s", trace) + } +} + +// TestSyncTraceDisabled verifies no trace directory appears when tracing +// is off, which is the default. +func TestSyncTraceDisabled(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "repo", "project")); err == nil { + t.Error("trace written while tracing is disabled") + } +} + +// TestSyncTraceSurvivesPrune verifies a trace is not treated as a stale +// file. A flat layout puts the trace inside the pruned tree itself, which +// is the case that would otherwise delete it on the following sync. +func TestSyncTraceSurvivesPrune(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + opts.Flat = true + opts.Prune = true + + // Two syncs: the first writes the trace, the second prunes with that + // trace already on disk. + for i := 0; i < 2; i++ { + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatalf("sync %d: %v", i+1, err) + } + if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { + t.Fatalf("trace missing after sync %d: %v", i+1, err) + } + } + + // A stale file beside it is still pruned, confirming pruning ran. + stale := filepath.Join(dest, "old-1.0-r0.apk") + if err := os.WriteFile(stale, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("stale package survived pruning, so pruning did not run") + } + if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { + t.Error("trace pruned:", err) + } +} + +// TestSyncTraceDryRun verifies a dry run neither writes a trace nor +// reports an existing one as a file it would prune. +func TestSyncTraceDryRun(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + opts.Flat = true + opts.DryRun = true + opts.Prune = true + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "project")); err == nil { + t.Error("dry run wrote a trace") + } + + // With a trace already on disk, a dry run must not count it as + // prunable. The file is planted directly so this stands on its own + // rather than depending on a preceding sync having written one. + traceFile := filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com") + if err := os.MkdirAll(filepath.Dir(traceFile), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(traceFile, []byte("previous trace\n"), 0644); err != nil { + t.Fatal(err) + } + fetch.Stats.Reset() + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if n := fetch.Stats.Pruned.Load(); n != 0 { + t.Errorf("dry run would prune %d files, want 0 (the trace must be kept)", n) + } + if _, err := os.Stat(traceFile); err != nil { + t.Error("dry run removed the existing trace:", err) + } +} + +// TestSyncTraceDebAtArchiveRoot verifies an apt trace is published at the +// archive root, where the convention places it, rather than in the suite. +func TestSyncTraceDebAtArchiveRoot(t *testing.T) { + www := t.TempDir() + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoDeb, dest) + opts.Architectures = []string{"amd64", "source"} + if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil { + t.Fatal(err) + } + + root := filepath.Join(dest, "debian") + trace := readTrace(t, root) + if !strings.Contains(trace, "Repository type: deb") { + t.Errorf("trace missing repository type\ngot:\n%s", trace) + } + if !strings.Contains(trace, "Architectures: amd64 source") { + t.Errorf("trace missing architectures\ngot:\n%s", trace) + } + if _, err := os.Stat(filepath.Join(root, "dists", "test", "project")); err == nil { + t.Error("trace written into the suite directory instead of the archive root") + } +} + +// TestSyncIntoWritesTrace verifies the mirror server's entry point +// publishes a trace like the sync commands do, and that it still leaves the +// caller's options untouched. +func TestSyncIntoWritesTrace(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + src := fetch.NewSource([]string{srv.URL + "/repo"}) + if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil { + t.Fatal(err) + } + trace := readTrace(t, dest) + if !strings.Contains(trace, "Running on host: mirror.example.com") { + t.Errorf("server trace missing host\ngot:\n%s", trace) + } + if !opts.Trace { + t.Error("SyncInto mutated the caller's options") + } +} + +// TestSyncIntoTraceDisabled verifies the server path honors tracing being +// off, which is the default. +func TestSyncIntoTraceDisabled(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{Type: RepoApk, Destination: dest, Workers: 2} + src := fetch.NewSource([]string{srv.URL + "/repo"}) + if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "project")); err == nil { + t.Error("trace written while tracing is disabled") + } +} + +// TestSyncTraceReportsOwnTransfer verifies a trace reports the bytes its own +// synchronization moved rather than whatever the shared collector holds. +// That isolation is what makes a trace meaningful on the server's path, +// where crawls run concurrently and all report into the same collector. +func TestSyncTraceReportsOwnTransfer(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + // Stand in for a concurrent crawl's transfer, which must not be + // attributed to this run. + const foreign = int64(1) << 40 + fetch.Stats.Reset() + fetch.Stats.FetchedBytes.Add(foreign) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + + got := traceField(t, readTrace(t, filepath.Join(dest, "repo")), "Total bytes received: ") + bytes, err := strconv.ParseInt(got, 10, 64) + if err != nil { + t.Fatalf("byte count %q: %v", got, err) + } + if bytes <= 0 { + t.Errorf("trace reports %d bytes received, want the run's own transfer", bytes) + } + if bytes >= foreign { + t.Errorf("trace reports %d bytes received, which includes another run's transfer", bytes) + } +} + +// TestSyncTraceMirrorsUpstream verifies upstream's own traces are copied +// into this mirror's trace directory, so the chain back to the origin +// archive is preserved, while the file naming this mirror stays the one +// this run wrote. +func TestSyncTraceMirrorsUpstream(t *testing.T) { + www := t.TempDir() + repo := filepath.Join(www, "repo") + testrepos.BuildApkRepo(t, repo) + upstreamTrace := filepath.Join(repo, filepath.FromSlash(TraceDir)) + testrepos.WriteFile(t, filepath.Join(upstreamTrace, "origin.example.net"), []byte("origin trace\n")) + testrepos.WriteFile(t, filepath.Join(upstreamTrace, "middle.example.net"), []byte("middle trace\n")) + // Upstream also carries a trace under this mirror's own name, which + // describes a different mirror and must not replace ours. + testrepos.WriteFile(t, filepath.Join(upstreamTrace, "mirror.example.com"), []byte("someone else\n")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + opts.Prune = true + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + + traceDir := filepath.Join(dest, "repo", filepath.FromSlash(TraceDir)) + for name, want := range map[string]string{ + "origin.example.net": "origin trace\n", + "middle.example.net": "middle trace\n", + } { + data, err := os.ReadFile(filepath.Join(traceDir, name)) + if err != nil { + t.Errorf("upstream trace %s not mirrored: %v", name, err) + continue + } + if string(data) != want { + t.Errorf("upstream trace %s = %q, want %q", name, data, want) + } + } + if trace := readTrace(t, filepath.Join(dest, "repo")); !strings.Contains(trace, "Running on host: mirror.example.com") { + t.Errorf("upstream trace replaced this mirror's own\ngot:\n%s", trace) + } + + // A second sync prunes with the mirrored traces already on disk; they + // belong to the repository and must survive. + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(traceDir, "origin.example.net")); err != nil { + t.Error("mirrored upstream trace pruned:", err) + } +} + +// TestSyncTraceUpstreamAbsent verifies an upstream that publishes no traces +// is not a synchronization failure. +func TestSyncTraceUpstreamAbsent(t *testing.T) { + www := t.TempDir() + testrepos.BuildApkRepo(t, filepath.Join(www, "repo")) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := traceOptions(RepoApk, dest) + if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, "repo", filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil { + t.Error("trace missing:", err) + } +} + +// TestTraceArchitectures verifies architectures are only claimed for apt +// runs that were actually restricted to them. +func TestTraceArchitectures(t *testing.T) { + if got := traceArchitectures(&Options{Type: RepoDeb, Architectures: []string{"source", "amd64"}}); got != "amd64 source" { + t.Errorf("deb architectures = %q, want %q", got, "amd64 source") + } + if got := traceArchitectures(&Options{Type: RepoDeb}); got != "" { + t.Errorf("unrestricted deb architectures = %q, want empty", got) + } + if got := traceArchitectures(&Options{Type: RepoRPM, Architectures: []string{"x86_64"}}); got != "" { + t.Errorf("rpm architectures = %q, want empty", got) + } +} diff --git a/server/classify.go b/server/classify.go new file mode 100644 index 0000000..eac42c1 --- /dev/null +++ b/server/classify.go @@ -0,0 +1,98 @@ +package server + +import ( + "path" + "strings" + + "github.com/grmrgecko/repo-sync/mirror" +) + +// kindGeneric marks request paths that are not repository entry points. +const kindGeneric = "generic" + +// resource is a crawlable unit derived from a request path. +type resource struct { + // Kind is a RepoType string, or kindGeneric. + Kind string + // Key uniquely identifies the resource in state and the lock map. + Key string + // Path is the repository request path (the suite directory for deb). + Path string + // Root is the request path prefix covering every file the resource + // owns; deb uses the archive root so pool requests match. + Root string + // ReqPath preserves the original request path. + ReqPath string +} + +// classifyRequest maps a request path onto a crawlable resource. Only +// repository entry-point metadata registers a repository; every other path +// is generic and served on demand. +func classifyRequest(reqPath string) resource { + clean := path.Clean("/" + strings.TrimPrefix(reqPath, "/")) + base := path.Base(clean) + dir := path.Dir(clean) + + // RPM repositories are identified by any repodata request. + if idx := strings.Index(clean, "/repodata/"); idx >= 0 { + root := clean[:idx] + if root == "" { + root = "/" + } + return resource{ + Kind: string(mirror.RepoRPM), + Key: string(mirror.RepoRPM) + ":" + root, + Path: root, + Root: root, + ReqPath: clean, + } + } + + // Apt suites are identified by their release files; the archive root + // above dists also covers the pool. + if base == "InRelease" || base == "Release" || base == "Release.gpg" { + root := dir + if idx := strings.Index(clean, "/dists/"); idx >= 0 { + root = clean[:idx] + if root == "" { + root = "/" + } + } + return resource{ + Kind: string(mirror.RepoDeb), + Key: string(mirror.RepoDeb) + ":" + dir, + Path: dir, + Root: root, + ReqPath: clean, + } + } + + // Pacman repositories are identified by their database file. + if strings.HasSuffix(base, ".db") { + return resource{ + Kind: string(mirror.RepoArch), + Key: string(mirror.RepoArch) + ":" + dir, + Path: dir, + Root: dir, + ReqPath: clean, + } + } + + // Alpine repositories are identified by their index archive. + if base == mirror.APKIndexName { + return resource{ + Kind: string(mirror.RepoApk), + Key: string(mirror.RepoApk) + ":" + dir, + Path: dir, + Root: dir, + ReqPath: clean, + } + } + + return resource{ + Kind: kindGeneric, + Key: kindGeneric + ":" + clean, + Path: clean, + ReqPath: clean, + } +} diff --git a/server/classify_test.go b/server/classify_test.go new file mode 100644 index 0000000..202e8c4 --- /dev/null +++ b/server/classify_test.go @@ -0,0 +1,68 @@ +package server + +import ( + "testing" + "time" + + "github.com/grmrgecko/repo-sync/state" +) + +// TestClassifyRequest verifies request paths map to the right resource +// kinds, crawl paths, and keep-alive roots. +func TestClassifyRequest(t *testing.T) { + cases := []struct { + reqPath string + kind string + path string + root string + }{ + {"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"}, + {"/almalinux/9/BaseOS/x86_64/os/repodata/abc-primary.xml.gz", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"}, + {"/debian/dists/bookworm/InRelease", "deb", "/debian/dists/bookworm", "/debian"}, + {"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"}, + {"/flat/Release", "deb", "/flat", "/flat"}, + {"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"}, + {"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"}, + {"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""}, + {"/notes/README.txt", kindGeneric, "/notes/README.txt", ""}, + } + for _, c := range cases { + res := classifyRequest(c.reqPath) + if res.Kind != c.kind || res.Path != c.path || res.Root != c.root { + t.Errorf("classifyRequest(%q) = {%s %s %s}, want {%s %s %s}", + c.reqPath, res.Kind, res.Path, res.Root, c.kind, c.path, c.root) + } + } +} + +// TestRefreshTier verifies the tier walk and staleness budget. +func TestRefreshTier(t *testing.T) { + schedule := []time.Duration{4 * time.Hour, 12 * time.Hour, 24 * time.Hour} + if interval, stale := refreshTier(time.Hour, schedule); stale || interval != 4*time.Hour { + t.Errorf("fresh resource tier = %v, %v", interval, stale) + } + if interval, stale := refreshTier(10*time.Hour, schedule); stale || interval != 12*time.Hour { + t.Errorf("aging resource tier = %v, %v", interval, stale) + } + if _, stale := refreshTier(41*time.Hour, schedule); !stale { + t.Error("resource past the budget not stale") + } +} + +// TestNextCrawlAt verifies crawl gating for fresh, failed, and deferred +// entries. +func TestNextCrawlAt(t *testing.T) { + now := time.Now() + if !nextCrawlAt(state.Entry{}, time.Hour).IsZero() { + t.Error("never-crawled entry not due immediately") + } + failedAt := now.Add(30 * time.Minute) + failed := state.Entry{LastError: "boom", LastCrawled: now, NextCrawl: failedAt} + if got := nextCrawlAt(failed, time.Hour); !got.Equal(failedAt) { + t.Errorf("failed entry gate = %v, want %v", got, failedAt) + } + ok := state.Entry{LastCrawled: now, NextCrawl: now.Add(10 * time.Minute)} + if got := nextCrawlAt(ok, 2*time.Hour); !got.Equal(now.Add(2 * time.Hour)) { + t.Errorf("tier gate = %v, want %v", got, now.Add(2*time.Hour)) + } +} diff --git a/server/crawl_loop.go b/server/crawl_loop.go new file mode 100644 index 0000000..ae06d40 --- /dev/null +++ b/server/crawl_loop.go @@ -0,0 +1,459 @@ +package server + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/mirror" + "github.com/grmrgecko/repo-sync/state" + log "github.com/sirupsen/logrus" +) + +// lockMu guards the per-resource lock table and the in-flight crawl set. +var lockMu sync.Mutex + +// resourceLock is one entry in the lock table; refs counts holders and +// waiters so the entry can be dropped once nobody references it. +type resourceLock struct { + mu sync.Mutex + refs int +} + +// locks serializes crawls of the same resource across request handlers and +// the scheduled loop. +var locks = map[string]*resourceLock{} + +// lockResource acquires the lock for a resource key and returns its unlock +// function. Entries are reference counted and removed when the last holder +// or waiter releases, so evicted resources do not accumulate in the table. +func lockResource(key string) func() { + lockMu.Lock() + l := locks[key] + if l == nil { + l = &resourceLock{} + locks[key] = l + } + l.refs++ + lockMu.Unlock() + + l.mu.Lock() + return func() { + l.mu.Unlock() + lockMu.Lock() + l.refs-- + if l.refs == 0 { + delete(locks, key) + } + lockMu.Unlock() + } +} + +// inventoryEvictionGrace shields resources still advertised by a mount's +// configuration from eviction after requests stop. +const inventoryEvictionGrace = 72 * time.Hour + +// inflight records resources with a crawl queued or running so concurrent +// dispatches from the request handlers and the scheduler collapse into one +// crawl. Guarded by lockMu. +var inflight = map[string]bool{} + +// crawlWG tracks dispatched crawl goroutines so shutdown can wait for them +// before the final state flush records their outcomes. +var crawlWG sync.WaitGroup + +// crawlCtx is the context dispatched crawls run under. CrawlLoop installs +// its own so shutdown cancels crawls; the background default covers tests +// and requests arriving before the loop starts. +// A pointer is stored rather than the interface itself: atomic.Value +// rejects a second store of a different concrete type, and the background +// default and the loop's cancelable context are different types. +var crawlCtx atomic.Pointer[context.Context] + +func init() { + ctx := context.Background() + crawlCtx.Store(&ctx) +} + +// Crawl slots bound how many crawls run at once; slotCond re-reads the +// configured limit on every wake so a reload takes effect. +var ( + slotMu sync.Mutex + slotCond = sync.NewCond(&slotMu) + slotsInUse int +) + +// acquireCrawlSlot blocks until a crawl slot is free under the configured +// concurrency limit. +func acquireCrawlSlot() { + slotMu.Lock() + for slotsInUse >= cfg.C.Crawler.ConcurrentCrawls { + slotCond.Wait() + } + slotsInUse++ + slotMu.Unlock() +} + +// releaseCrawlSlot frees a crawl slot and wakes the waiters. +func releaseCrawlSlot() { + slotMu.Lock() + slotsInUse-- + slotMu.Unlock() + slotCond.Broadcast() +} + +// startCrawl dispatches an asynchronous crawl of one resource, collapsing +// duplicate dispatches and bounding concurrency. Failures are logged; the +// retry gating lives in the schedulers. +func startCrawl(res resource) { + lockMu.Lock() + if inflight[res.Key] { + lockMu.Unlock() + return + } + inflight[res.Key] = true + lockMu.Unlock() + + crawlWG.Add(1) + go func() { + defer crawlWG.Done() + defer func() { + lockMu.Lock() + delete(inflight, res.Key) + lockMu.Unlock() + }() + acquireCrawlSlot() + defer releaseCrawlSlot() + ctx := *crawlCtx.Load() + if ctx.Err() != nil { + return + } + if err := crawlResource(ctx, res); err != nil { + log.WithError(err).WithField("key", res.Key).Error("Repository crawl failed.") + } + }() +} + +// WaitCrawls blocks until every dispatched crawl has finished. It is called +// at shutdown after the crawl context is canceled and before the final +// state flush. +func WaitCrawls() { + crawlWG.Wait() +} + +// crawlResource synchronizes one classified resource under its lock. Deb +// resources also hold an archive-root lock so suites sharing a pool never +// download the same file concurrently. +func crawlResource(ctx context.Context, res resource) error { + if res.Kind == string(mirror.RepoDeb) && res.Root != res.Path { + defer lockResource("deb-root:" + res.Root)() + } + defer lockResource(res.Key)() + + err := dispatchCrawl(ctx, res) + state.S.MarkCrawled(res.Key, time.Now(), err) + return err +} + +// pathBelow reports whether a request path sits strictly below a base path. +func pathBelow(p, base string) bool { + if p == base { + return false + } + if base == "/" { + return true + } + return strings.HasPrefix(p, base+"/") +} + +// prunableTree reports whether a crawl may prune its destination tree. Every +// sync prunes the repository directory, which for deb is the suite directory +// and otherwise the repository's own path. A crawl's keep set covers only +// that one repository, so a tree that also holds another mount's content or +// another tracked repository must not be pruned: the neighbour's files are +// unreferenced there and would be deleted. This matters most for a +// repository sitting at the mirror root, whose tree is the whole cache. +func prunableTree(conf *cfg.Config, res resource) bool { + for _, mount := range conf.Mounts { + if pathBelow(mount.Path, res.Path) { + log.WithFields(log.Fields{"key": res.Key, "mount": mount.Path}). + Debug("Pruning disabled; the repository tree contains another mount.") + return false + } + } + for key, entry := range state.S.Snapshot() { + if key == res.Key || entry.Kind == kindGeneric { + continue + } + if pathBelow(entry.Path, res.Path) { + log.WithFields(log.Fields{"key": res.Key, "nested": key}). + Debug("Pruning disabled; the repository tree contains another repository.") + return false + } + } + return true +} + +// traceInfo maps the configured trace section onto the mirror package's +// description of this mirror. The mapping is repeated rather than shared +// with the sync commands so the mirror package keeps its own trace type +// instead of taking the configuration as part of its API. +func traceInfo(t cfg.TraceConfig) mirror.TraceInfo { + return mirror.TraceInfo{ + Host: t.HostName(), + Maintainer: t.Maintainer, + Sponsor: t.Sponsor, + Country: t.Country, + Location: t.Location, + Throughput: t.Throughput, + } +} + +// dispatchCrawl runs the format-specific synchronization for a resource +// into the online tree. +func dispatchCrawl(ctx context.Context, res resource) error { + online := cfg.C.OnlineDomain() + mount, ok := cfg.C.MountFor(res.Path) + if !ok { + return fmt.Errorf("no mount configured for %s", res.Path) + } + + // upstreamFor maps a request path onto the mount's upstream URL. + upstreamFor := func(p string) string { + rel := p + if mount.Path != "/" { + rel = strings.TrimPrefix(p, mount.Path) + } + return mount.Upstream + rel + } + + missing, err := fetch.ParseMissingMode(cfg.C.Crawler.MissingMode) + if err != nil { + return err + } + + // Generic files never reach the crawl path: they refresh on demand at + // serve time and expire through the failure cache and state eviction. + typ := mirror.RepoType(res.Kind) + opts := &mirror.Options{ + Type: typ, + Workers: cfg.C.Crawler.Workers, + Prune: prunableTree(cfg.C, res), + PruneGrace: cfg.C.Crawler.PruneGrace, + Missing: missing, + MissingRetries: cfg.C.Crawler.MissingRetries, + Trace: cfg.C.Trace.Enabled, + TraceInfo: traceInfo(cfg.C.Trace), + } + + // Deb destinations pin the archive root so pool files land beside + // their suites; every other type syncs into its own directory. + destPath := res.Path + if typ == mirror.RepoDeb { + destPath = res.Root + } + dest, err := fetch.LocalJoin(online.Root, destPath) + if err != nil { + return err + } + repoURL := upstreamFor(res.Path) + return mirror.SyncInto(ctx, typ, fetch.NewSource([]string{repoURL}), repoURL, dest, opts) +} + +// CrawlLoop keeps configured and discovered resources fresh until the +// context is canceled, starting with an immediate pass. Its context also +// governs request-dispatched crawls so shutdown cancels them. +func CrawlLoop(ctx context.Context) { + crawlCtx.Store(&ctx) + runScheduledCrawls(ctx) + t := time.NewTicker(cfg.C.CrawlCheckInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + fetchFailures.Sweep(time.Now()) + runScheduledCrawls(ctx) + // Re-arm from the active config so a reload takes effect. + t.Reset(cfg.C.CrawlCheckInterval) + } + } +} + +// runScheduledCrawls runs one pass of configured and request-discovered +// crawls. +func runScheduledCrawls(ctx context.Context) { + crawlConfigured(ctx) + crawlDue(ctx) +} + +// configuredResource builds the resource for a mount's configured +// repository. +func configuredResource(repo cfg.MountRepoConfig) resource { + root := repo.Path + if repo.Type == string(mirror.RepoDeb) { + if idx := strings.Index(repo.Path, "/dists/"); idx >= 0 { + root = repo.Path[:idx] + if root == "" { + root = "/" + } + } + } + return resource{ + Kind: repo.Type, + Key: repo.Type + ":" + repo.Path, + Path: repo.Path, + Root: root, + ReqPath: repo.Path, + } +} + +// crawlConfigured keeps every mount-configured repository synchronized +// regardless of client traffic. Crawls are dispatched asynchronously so a +// slow repository cannot stall the scheduler loop. +func crawlConfigured(ctx context.Context) { + now := time.Now() + for _, mount := range cfg.C.Mounts { + for _, repo := range mount.Repos { + if ctx.Err() != nil { + return + } + res := configuredResource(repo) + state.S.MarkSeenInInventory(res.Kind, res.Key, res.Path, res.Root, now) + entry, _ := state.S.Entry(res.Key) + if !dueForInventory(entry, now) { + continue + } + startCrawl(res) + } + } +} + +// dueForInventory gates configured crawls on the standard interval with +// faster retries after failures. +func dueForInventory(entry state.Entry, now time.Time) bool { + if entry.LastCrawled.IsZero() { + return true + } + if !entry.NextCrawl.IsZero() { + return !now.Before(entry.NextCrawl) + } + return now.Sub(entry.LastCrawled) >= cfg.C.RepoCrawlInterval +} + +// refreshTier walks the cumulative schedule, returning the refresh +// interval matching how long ago the resource was requested, or stale once +// the whole budget has elapsed. +func refreshTier(sinceRequested time.Duration, schedule []time.Duration) (time.Duration, bool) { + var cumulative time.Duration + for _, step := range schedule { + if step <= 0 { + continue + } + cumulative += step + if sinceRequested < cumulative { + return step, false + } + } + return 0, true +} + +// refreshSchedule is the tier list: the active crawl interval followed by +// the configured aging backoff steps. +func refreshSchedule() []time.Duration { + schedule := make([]time.Duration, 0, 1+len(cfg.C.Crawler.RefreshSchedule)) + schedule = append(schedule, cfg.C.RepoCrawlInterval) + return append(schedule, cfg.C.Crawler.RefreshSchedule...) +} + +// activeInInventory reports whether a mount still advertised the resource +// recently enough to shield it from eviction. +func activeInInventory(entry state.Entry, now time.Time) bool { + if entry.LastSeenInInventory.IsZero() { + return false + } + return now.Sub(entry.LastSeenInInventory) < inventoryEvictionGrace +} + +// nextCrawlAt returns when a resource should next crawl: immediately when +// never crawled, at the failure retry gate after errors, else at the tier +// interval. +func nextCrawlAt(entry state.Entry, tierInterval time.Duration) time.Time { + if entry.LastCrawled.IsZero() { + return time.Time{} + } + if entry.LastError != "" && !entry.NextCrawl.IsZero() { + return entry.NextCrawl + } + tierGate := entry.LastCrawled.Add(tierInterval) + if !entry.NextCrawl.IsZero() && entry.NextCrawl.After(tierGate) { + return entry.NextCrawl + } + return tierGate +} + +// crawlDue refreshes request-discovered resources on the tier schedule and +// evicts those whose traffic stopped past the whole budget. +func crawlDue(ctx context.Context) { + now := time.Now() + schedule := refreshSchedule() + for key, entry := range state.S.Snapshot() { + if ctx.Err() != nil { + return + } + // Inventory-only entries are managed by crawlConfigured. + if entry.LastRequested.IsZero() { + continue + } + interval, stale := refreshTier(now.Sub(entry.LastRequested), schedule) + if stale { + if activeInInventory(entry, now) { + continue + } + evictStale(key, entry) + continue + } + // Generic files refresh on demand at serve time. + if entry.Kind == kindGeneric { + continue + } + next := nextCrawlAt(entry, interval) + if !next.IsZero() && now.Before(next) { + continue + } + startCrawl(resource{Kind: entry.Kind, Key: key, Path: entry.Path, Root: entry.Root, ReqPath: entry.Path}) + } +} + +// evictStale removes a resource whose requests stopped long enough ago, +// re-checking under the lock so a racing request wins. +func evictStale(key string, entry state.Entry) { + // Deb resources share an archive root across sibling suites; hold that + // lock too, matching crawlResource, so eviction never races an + // in-flight sibling-suite crawl touching the shared pool. + if entry.Kind == string(mirror.RepoDeb) && entry.Root != entry.Path { + defer lockResource("deb-root:" + entry.Root)() + } + defer lockResource(key)() + + current, ok := state.S.Entry(key) + if ok && current.LastRequested.After(entry.LastRequested) { + return + } + target, err := fetch.LocalJoin(cfg.C.OnlineDomain().Root, entry.Path) + if err == nil && entry.Path != "/" && entry.Path != "" { + if err := os.RemoveAll(target); err != nil { + log.WithError(err).WithField("path", target).Warn("Failed to evict stale resource.") + return + } + log.WithFields(log.Fields{"key": key, "path": target}).Info("Evicted stale resource.") + } + state.S.Delete(key) +} diff --git a/server/failure_cache.go b/server/failure_cache.go new file mode 100644 index 0000000..1f854fb --- /dev/null +++ b/server/failure_cache.go @@ -0,0 +1,56 @@ +package server + +import ( + "sync" + "time" +) + +// failureEntry is one remembered upstream failure with its expiry. +type failureEntry struct { + status int + expires time.Time +} + +// failureCache remembers recent upstream failures for generic paths so a +// missing or erroring file does not trigger an upstream request per client. +// It is memory-only; nothing here reaches the state file. +type failureCache struct { + mu sync.Mutex + entries map[string]failureEntry +} + +// fetchFailures is the server's shared failure cache. +var fetchFailures = &failureCache{entries: map[string]failureEntry{}} + +// Hit returns the remembered status for a key, pruning expired entries. +func (c *failureCache) Hit(key string, now time.Time) (int, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok { + return 0, false + } + if !now.Before(e.expires) { + delete(c.entries, key) + return 0, false + } + return e.status, true +} + +// Add remembers a failure until now+ttl. +func (c *failureCache) Add(key string, status int, now time.Time, ttl time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.entries[key] = failureEntry{status: status, expires: now.Add(ttl)} +} + +// Sweep drops expired entries that nobody re-requested. +func (c *failureCache) Sweep(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, key) + } + } +} diff --git a/server/http.go b/server/http.go new file mode 100644 index 0000000..4da9c43 --- /dev/null +++ b/server/http.go @@ -0,0 +1,81 @@ +package server + +import ( + "context" + "net" + "net/http" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + log "github.com/sirupsen/logrus" +) + +// shutdownTimeout bounds the graceful shutdown of the listener, after which +// in-flight requests are abandoned. +const shutdownTimeout = 30 * time.Second + +// Server owns the mirror's HTTP listener. It binds from the active +// configuration and rebinds on reload when the listen address moves. +type Server struct { + server *http.Server + addr string +} + +// NewServer builds a mirror HTTP server from the active configuration. +func NewServer() *Server { + s := new(Server) + s.configure() + return s +} + +// configure rebuilds the underlying server from the active configuration. +// A running server is never reconfigured in place; Reload binds a new one. +func (s *Server) configure() { + s.addr = cfg.C.HTTP.ListenAddr() + s.server = &http.Server{ + Addr: s.addr, + Handler: Handler(), + ReadHeaderTimeout: cfg.C.HTTP.ReadHeaderTimeout, + ReadTimeout: cfg.C.HTTP.ReadTimeout, + WriteTimeout: cfg.C.HTTP.WriteTimeout, + IdleTimeout: cfg.C.HTTP.IdleTimeout, + } +} + +// Start binds the listen address and serves in the background. A bind +// failure is returned so the caller can abort startup rather than run +// without a listener. +func (s *Server) Start() error { + ln, err := net.Listen("tcp", s.addr) + if err != nil { + return err + } + log.WithField("addr", s.addr).Info("Mirror server listening.") + go func() { + if err := s.server.Serve(ln); err != nil && err != http.ErrServerClosed { + log.WithError(err).Error("HTTP server failed.") + } + }() + return nil +} + +// Reload rebinds the listener when the active configuration moved the +// listen address, and is a no-op otherwise. The old listener is shut down +// before the new one binds so the address is free. Other HTTP settings are +// read when the server is built, so they take effect on the next rebind. +func (s *Server) Reload() error { + if cfg.C.HTTP.ListenAddr() == s.addr { + return nil + } + s.Shutdown() + s.configure() + return s.Start() +} + +// Shutdown stops accepting connections and waits for in-flight requests, +// giving up after shutdownTimeout. +func (s *Server) Shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + _ = s.server.Shutdown(ctx) +} diff --git a/server/main_test.go b/server/main_test.go new file mode 100644 index 0000000..d7ec33b --- /dev/null +++ b/server/main_test.go @@ -0,0 +1,14 @@ +package server + +import ( + "os" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" +) + +// TestMain initializes the shared HTTP client all fetch paths depend on. +func TestMain(m *testing.M) { + fetch.Reload() + os.Exit(m.Run()) +} diff --git a/server/serve.go b/server/serve.go new file mode 100644 index 0000000..68e7cd7 --- /dev/null +++ b/server/serve.go @@ -0,0 +1,317 @@ +// Package server runs the caching mirror: it answers requests from the +// local tree, fetches misses through the configured upstream mounts, and +// crawls the repositories it discovers in the background so they stay +// current without holding a request open. +package server + +import ( + "context" + "errors" + "fmt" + "html" + "io" + "net/http" + "os" + "path" + "sort" + "strings" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/state" + log "github.com/sirupsen/logrus" +) + +// Handler returns the mirror server's HTTP handler. +func Handler() http.Handler { + return http.HandlerFunc(serveMirror) +} + +// regularFile stats a path, reporting only regular files as present. +func regularFile(name string) (os.FileInfo, bool) { + info, err := os.Stat(name) + if err != nil || !info.Mode().IsRegular() { + return nil, false + } + return info, true +} + +// serveLocal resolves a request path under a domain root, mapping +// directory requests to a cached index.html. +func serveLocal(root, reqPath string, isDir bool) (string, error) { + p := reqPath + if isDir || p == "/" { + p = strings.TrimSuffix(p, "/") + "/index.html" + } + return fetch.LocalJoin(root, p) +} + +// internalFile reports paths that belong to the mirror's own bookkeeping +// and must never be served. +func internalFile(reqPath string) bool { + base := path.Base(reqPath) + return base == fetch.LockFileName || base == fetch.PruneStateName || + base == fetch.MissingStateName || base == fetch.DiscoverStateName || + strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix) +} + +// fetchUpstream retrieves one file through the mount covering the request +// path and stores it under root. +func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error { + mount, ok := cfg.C.MountFor(reqPath) + if !ok { + return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound) + } + rel := reqPath + if mount.Path != "/" { + rel = strings.TrimPrefix(reqPath, mount.Path) + } + if isDir { + rel = strings.TrimSuffix(rel, "/") + "/" + } + dst, err := serveLocal(root, reqPath, isDir) + if err != nil { + return err + } + // Concurrent on-demand fetches and background crawls can target the + // same destination; fetch.File serializes them by path. + src := fetch.NewSource([]string{mount.Upstream}) + _, err = fetch.File(ctx, src, rel, dst, nil, false, false) + return err +} + +// writeFetchError reports an upstream failure to the client. +func writeFetchError(w http.ResponseWriter, err error) { + if errors.Is(err, fetch.ErrNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + http.Error(w, "upstream fetch failed", http.StatusBadGateway) +} + +// writeCachedFailure reports a remembered upstream failure. +func writeCachedFailure(w http.ResponseWriter, status int) { + if status == http.StatusNotFound { + http.Error(w, "not found", http.StatusNotFound) + return + } + http.Error(w, "upstream fetch failed", http.StatusBadGateway) +} + +// writeIndexesDisabled answers a directory request when directory indexes +// are turned off, confirming the mirror is alive without listing content. +func writeIndexesDisabled(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, "\n\nrepo-sync mirror\n\n"+ + "

repo-sync mirror

\n

This mirror is running. Directory indexes are disabled.

\n"+ + "\n\n") +} + +// writeMountIndex answers the root directory request with a generated index +// of every mount and configured repository, giving clients a navigable +// entry point when no catch-all mount covers the root. +func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) { + seen := map[string]bool{} + var paths []string + add := func(p string) { + if p == "/" || seen[p] { + return + } + seen[p] = true + paths = append(paths, p) + } + for _, mount := range conf.Mounts { + add(mount.Path) + for _, repo := range mount.Repos { + add(repo.Path) + } + } + sort.Strings(paths) + + var b strings.Builder + b.WriteString("\n\nIndex of /\n\n

Index of /

\n
    \n") + for _, p := range paths { + href := html.EscapeString(p + "/") + fmt.Fprintf(&b, "
  • %s
    • \n", href, href) + } + b.WriteString("
\n\n\n") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, b.String()) +} + +// serveMirror handles every mirror request: domain lookup, classification, +// and dispatch to the role-specific handler. +func serveMirror(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + domain, ok := cfg.C.Domain(r.Host) + if !ok { + http.Error(w, "unknown mirror domain", http.StatusNotFound) + return + } + + reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/")) + isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/" + if internalFile(reqPath) { + http.Error(w, "not found", http.StatusNotFound) + return + } + + // Directory requests are answered before role dispatch: a notice page + // when indexes are disabled, and a generated index of the mounts for a + // root that no catch-all mount covers. + if isDir { + if !cfg.C.HTTP.DirectoryIndexes { + writeIndexesDisabled(w) + return + } + if reqPath == "/" { + if _, ok := cfg.C.MountFor("/"); !ok { + writeMountIndex(w, cfg.C) + return + } + } + } + res := classifyRequest(reqPath) + + if domain.Role == cfg.RoleOnline { + handleOnline(w, r, domain, reqPath, isDir, res) + return + } + handleOffline(w, r, domain, reqPath, isDir, res) +} + +// handleOnline serves from the writable cache, fetching and crawling on +// demand. +func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) { + local, err := serveLocal(domain.Root, reqPath, isDir) + if err != nil { + http.Error(w, "bad request path", http.StatusBadRequest) + return + } + now := time.Now() + + // Directory listings are cached pass-through pages with no tracking. + if isDir { + if _, exists := regularFile(local); !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil { + writeFetchError(w, err) + return + } + } + http.ServeFile(w, r, local) + return + } + + if res.Kind == kindGeneric { + // Files under a registered repository keep it alive and are + // fetched on demand until its crawl completes. + if state.S.TouchRepoMembers(reqPath, now) { + if _, exists := regularFile(local); !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + writeFetchError(w, err) + return + } + } + http.ServeFile(w, r, local) + return + } + handleGeneric(w, r, domain, reqPath, res, local, now) + return + } + + // A repository entry point registers the repository and dispatches a + // background crawl on first sight so the whole repository fills in + // behind this response without holding it open. The requested file + // itself is fetched on demand so the client is answered immediately. + // Repositories already crawled only re-dispatch here to retry a failed + // crawl; missing optional files must not trigger a full crawl per + // request. + entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now) + _, exists := regularFile(local) + needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "") + if !exists { + if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + // A path whose first contact failed was never shown to be a + // repository, so the registration this request created is + // dropped and the scheduler never crawls it. Registrations an + // earlier request established are kept: every entry point of one + // repository shares a key, so a repository serving Release but + // not InRelease would otherwise deregister itself on the miss. + if !tracked { + state.S.Delete(res.Key) + } + writeFetchError(w, err) + return + } + } + if needCrawl { + startCrawl(res) + } + http.ServeFile(w, r, local) +} + +// handleGeneric serves a plain cached file, refreshing stale copies and +// remembering upstream failures. +func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) { + if status, hit := fetchFailures.Hit(res.Key, now); hit { + if _, exists := regularFile(local); !exists { + writeCachedFailure(w, status) + return + } + } else { + // GenericMaxAge bounds how often the upstream is revalidated, + // measured from the last upstream check recorded in state. The + // local file's modification time carries the upstream's + // Last-Modified, so it does not record when the mirror last checked. + entry, tracked := state.S.Entry(res.Key) + _, exists := regularFile(local) + if !exists || !tracked || now.Sub(entry.LastCrawled) > cfg.C.GenericMaxAge { + err := fetchUpstream(r.Context(), domain.Root, reqPath, false) + switch { + case errors.Is(err, fetch.ErrNotFound): + fetchFailures.Add(res.Key, http.StatusNotFound, now, cfg.C.NegativeCacheTTL) + state.S.Delete(res.Key) + if !exists { + writeFetchError(w, err) + return + } + log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.") + case err != nil: + fetchFailures.Add(res.Key, http.StatusBadGateway, now, cfg.C.TransientErrorTTL) + if !exists { + writeFetchError(w, err) + return + } + log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.") + default: + state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now) + // Record the upstream check so GenericMaxAge gates the next + // request. The scheduler skips generic entries, so the retry + // time this also sets is unused. + state.S.MarkCrawled(res.Key, now, nil) + } + } else { + state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now) + } + } + http.ServeFile(w, r, local) +} + +// handleOffline serves the read-only published tree, reading through the +// online cache for anything it does not hold. +func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) { + local, err := serveLocal(domain.Root, reqPath, isDir) + if err != nil { + http.Error(w, "bad request path", http.StatusBadRequest) + return + } + if _, exists := regularFile(local); exists { + http.ServeFile(w, r, local) + return + } + handleOnline(w, r, cfg.C.OnlineDomain(), reqPath, isDir, res) +} diff --git a/server/serve_test.go b/server/serve_test.go new file mode 100644 index 0000000..30c56aa --- /dev/null +++ b/server/serve_test.go @@ -0,0 +1,487 @@ +package server + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/grmrgecko/repo-sync/mirror" + "github.com/grmrgecko/repo-sync/state" +) + +// serverFixture builds an upstream with one repository of each type plus a +// generic file, configures the mirror server against it, and returns the +// mirror's test server and the online root. +func serverFixture(t *testing.T) (*httptest.Server, string, string) { + t.Helper() + + // Upstream content behind a single catch-all mount. + www := t.TempDir() + testrepos.BuildRPMRepo(t, filepath.Join(www, "almalinux", "9", "BaseOS", "x86_64", "os")) + testrepos.BuildDebRepo(t, filepath.Join(www, "debian")) + testrepos.BuildArchRepo(t, filepath.Join(www, "archlinux", "core", "os", "x86_64"), "core") + testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "main", "x86_64")) + testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file")) + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + offlineRoot := t.TempDir() + confDir := t.TempDir() + conf := fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s + - domain: offline.test + role: offline + root: %s +mounts: + - path: / + upstream: %s +`, confDir, onlineRoot, offlineRoot, upstream.URL) + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(conf)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + // Crawls dispatched by requests run in the background; drain them so + // they do not race the next test's state reload. + t.Cleanup(WaitCrawls) + return srv, onlineRoot, offlineRoot +} + +// waitFor polls a condition until it holds or the deadline passes; crawls +// dispatched by requests run asynchronously. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// get fetches a mirror URL with an optional Host override. +func get(t *testing.T, srv *httptest.Server, host, path string) (*http.Response, []byte) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil) + if err != nil { + t.Fatal(err) + } + if host != "" { + req.Host = host + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + return resp, body +} + +// TestServeRPMDiscovery verifies a repomd.xml request registers and crawls +// the repository, filling the local tree behind the response. +func TestServeRPMDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + repo := "/almalinux/9/BaseOS/x86_64/os" + resp, body := get(t, srv, "", repo+"/repodata/repomd.xml") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if len(body) == 0 { + t.Fatal("empty repomd.xml response") + } + + // The repository was registered and the background crawl mirrors its + // packages behind the response. + if _, ok := state.S.Entry("rpm:" + repo); !ok { + t.Error("repository not registered in state") + } + var pkgs []string + waitFor(t, "rpm crawl", func() bool { + pkgs, _ = filepath.Glob(filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), "Packages", "*.rpm")) + return len(pkgs) > 0 + }) + + // Package requests are served from the crawled tree. + resp, _ = get(t, srv, "", repo+"/Packages/"+filepath.Base(pkgs[0])) + if resp.StatusCode != http.StatusOK { + t.Errorf("package request status = %d", resp.StatusCode) + } +} + +// TestServeDebDiscovery verifies a Release request crawls the suite with +// pool files under the archive root. +func TestServeDebDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, _ := get(t, srv, "", "/debian/dists/test/Release") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if _, ok := state.S.Entry("deb:/debian/dists/test"); !ok { + t.Error("suite not registered in state") + } + waitFor(t, "deb crawl", func() bool { + _, err := os.Stat(filepath.Join(onlineRoot, "debian", "pool", "main", "h", "hello", "hello_1.0_amd64.deb")) + return err == nil + }) + + // A pool request under the archive root is a repository member. + resp, _ = get(t, srv, "", "/debian/pool/main/h/hello/hello_1.0_amd64.deb") + if resp.StatusCode != http.StatusOK { + t.Errorf("pool request status = %d", resp.StatusCode) + } +} + +// TestServeArchAndApkDiscovery verifies database and index requests crawl +// their repositories. +func TestServeArchAndApkDiscovery(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, _ := get(t, srv, "", "/archlinux/core/os/x86_64/core.db") + if resp.StatusCode != http.StatusOK { + t.Fatalf("arch status = %d", resp.StatusCode) + } + waitFor(t, "arch crawl", func() bool { + pkgs, _ := filepath.Glob(filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "*.pkg.tar.zst")) + return len(pkgs) > 0 + }) + + resp, _ = get(t, srv, "", "/alpine/v3.24/main/x86_64/APKINDEX.tar.gz") + if resp.StatusCode != http.StatusOK { + t.Fatalf("apk status = %d", resp.StatusCode) + } + waitFor(t, "apk crawl", func() bool { + apks, _ := filepath.Glob(filepath.Join(onlineRoot, "alpine", "v3.24", "main", "x86_64", "*.apk")) + return len(apks) > 0 + }) +} + +// TestServeGeneric verifies plain files proxy through with caching and +// missing files are answered from the negative cache. +func TestServeGeneric(t *testing.T) { + srv, onlineRoot, _ := serverFixture(t) + + resp, body := get(t, srv, "", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("generic = %d %q", resp.StatusCode, body) + } + if _, err := os.Stat(filepath.Join(onlineRoot, "notes", "README.txt")); err != nil { + t.Error("generic file not cached:", err) + } + + resp, _ = get(t, srv, "", "/notes/missing.txt") + if resp.StatusCode != http.StatusNotFound { + t.Errorf("missing file status = %d", resp.StatusCode) + } + if status, hit := fetchFailures.Hit("generic:/notes/missing.txt", time.Now()); !hit || status != http.StatusNotFound { + t.Error("missing file not negative-cached") + } +} + +// TestServeGenericRevalidation verifies GenericMaxAge gates how often a +// cached plain file is revalidated. The upstream file carries an old +// Last-Modified that the cached copy inherits, so the gate is measured from +// the recorded upstream check rather than the local modification time. +func TestServeGenericRevalidation(t *testing.T) { + www := t.TempDir() + name := filepath.Join(www, "notes", "README.txt") + testrepos.WriteFile(t, name, []byte("generic file")) + old := time.Now().Add(-30 * 24 * time.Hour) + if err := os.Chtimes(name, old, old); err != nil { + t.Fatal(err) + } + + var upstreamHits atomic.Int64 + files := http.FileServer(http.Dir(www)) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamHits.Add(1) + files.ServeHTTP(w, r) + })) + t.Cleanup(upstream.Close) + + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +generic_max_age: 6h +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: / + upstream: %s +`, confDir, t.TempDir(), upstream.URL))) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + for i := 0; i < 3; i++ { + resp, body := get(t, srv, "", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("request %d = %d %q", i, resp.StatusCode, body) + } + } + if got := upstreamHits.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 within generic_max_age", got) + } +} + +// TestServeOffline verifies offline domains serve their own tree first and +// read through the online cache otherwise. +func TestServeOffline(t *testing.T) { + srv, _, offlineRoot := serverFixture(t) + + // Not present offline: delegated to online, which proxies upstream. + resp, body := get(t, srv, "offline.test", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "generic file" { + t.Fatalf("offline read-through = %d %q", resp.StatusCode, body) + } + + // Present offline: served from the offline tree. + testrepos.WriteFile(t, filepath.Join(offlineRoot, "notes", "README.txt"), []byte("published copy")) + resp, body = get(t, srv, "offline.test", "/notes/README.txt") + if resp.StatusCode != http.StatusOK || string(body) != "published copy" { + t.Errorf("offline copy = %d %q", resp.StatusCode, body) + } +} + +// TestServeDirectoryIndexes verifies the generated root index without a +// catch-all mount and the notice page when indexes are disabled. +func TestServeDirectoryIndexes(t *testing.T) { + www := t.TempDir() + testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file")) + upstream := testrepos.ServeDir(t, www) + + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + conf := fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: /notes + upstream: %s/notes + - path: /almalinux + upstream: %s/almalinux + repos: + - path: /almalinux/9/BaseOS/x86_64/os + type: rpm +`, confDir, t.TempDir(), upstream.URL, upstream.URL) + testrepos.WriteFile(t, confPath, []byte(conf)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + // Without a catch-all mount the root serves a generated index linking + // every mount and configured repository. + resp, body := get(t, srv, "", "/") + if resp.StatusCode != http.StatusOK { + t.Fatalf("index status = %d", resp.StatusCode) + } + for _, want := range []string{`"/notes/"`, `"/almalinux/"`, `"/almalinux/9/BaseOS/x86_64/os/"`} { + if !strings.Contains(string(body), want) { + t.Errorf("index missing %s in %q", want, body) + } + } + + // Disabling indexes answers every directory request with the notice. + disabled := strings.Replace(conf, "mounts:", "http:\n directory_indexes: false\nmounts:", 1) + testrepos.WriteFile(t, confPath, []byte(disabled)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + resp, body = get(t, srv, "", "/") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") { + t.Errorf("disabled root = %d %q", resp.StatusCode, body) + } + resp, body = get(t, srv, "", "/notes/") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") { + t.Errorf("disabled subdirectory = %d %q", resp.StatusCode, body) + } +} + +// TestServeRejections verifies unknown domains and internal bookkeeping +// files are refused. +func TestServeRejections(t *testing.T) { + srv, _, _ := serverFixture(t) + + resp, _ := get(t, srv, "unknown.test", "/notes/README.txt") + if resp.StatusCode != http.StatusNotFound { + t.Errorf("unknown domain status = %d", resp.StatusCode) + } + resp, _ = get(t, srv, "", path.Join("/almalinux", fetch.LockFileName)) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("internal file status = %d", resp.StatusCode) + } +} + +// TestPrunableTree verifies a crawl only prunes a tree it wholly owns, so a +// repository at the mirror root cannot delete another mount's cached files. +func TestPrunableTree(t *testing.T) { + serverFixture(t) + + conf := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}, {Path: "/other"}}} + if prunableTree(conf, resource{Key: "rpm:/", Path: "/"}) { + t.Error("a root repository may not prune a tree holding another mount") + } + if !prunableTree(conf, resource{Key: "rpm:/other/el9", Path: "/other/el9"}) { + t.Error("a repository owning its own tree must prune") + } + + // A repository nested under another one blocks the outer one's prune. + single := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}}} + state.S.MarkRequested("rpm", "rpm:/vendor/el9", "/vendor/el9", "/vendor/el9", time.Now()) + if prunableTree(single, resource{Key: "rpm:/vendor", Path: "/vendor"}) { + t.Error("an outer repository may not prune a tree holding a nested one") + } + if !prunableTree(single, resource{Key: "rpm:/vendor/el9", Path: "/vendor/el9"}) { + t.Error("a repository must not be blocked by its own registration") + } +} + +// TestServeRootRepositoryKeepsOtherMounts verifies a repository discovered at +// the mirror root does not prune content another mount cached beside it. +func TestServeRootRepositoryKeepsOtherMounts(t *testing.T) { + // Two upstreams: a repository at its own root, and unrelated content. + repoWWW := t.TempDir() + testrepos.BuildRPMRepo(t, repoWWW) + repoUpstream := testrepos.ServeDir(t, repoWWW) + otherWWW := t.TempDir() + testrepos.WriteFile(t, filepath.Join(otherWWW, "notes.txt"), []byte("other mount content")) + otherUpstream := testrepos.ServeDir(t, otherWWW) + + onlineRoot := t.TempDir() + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: / + upstream: %s + - path: /other + upstream: %s +`, confDir, onlineRoot, repoUpstream.URL, otherUpstream.URL))) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + t.Cleanup(WaitCrawls) + + // Cache a file from the unrelated mount, then crawl the root repository. + if resp, _ := get(t, srv, "", "/other/notes.txt"); resp.StatusCode != http.StatusOK { + t.Fatalf("notes status = %d", resp.StatusCode) + } + cached := filepath.Join(onlineRoot, "other", "notes.txt") + if _, err := os.Stat(cached); err != nil { + t.Fatal("unrelated file not cached:", err) + } + if resp, _ := get(t, srv, "", "/repodata/repomd.xml"); resp.StatusCode != http.StatusOK { + t.Fatalf("repomd status = %d", resp.StatusCode) + } + WaitCrawls() + + if _, err := os.Stat(cached); err != nil { + t.Error("root repository crawl pruned another mount's cached file:", err) + } +} + +// TestServeSiblingEntryPointMiss verifies a repository keeps its +// registration when a sibling entry-point path the upstream does not serve +// is requested. Every entry point of one repository shares a state key, and +// clients probe all of them: apt asks for InRelease, Release, and +// Release.gpg, and dnf asks for signature material beside repomd.xml. +func TestServeSiblingEntryPointMiss(t *testing.T) { + srv, _, _ := serverFixture(t) + + for _, tc := range []struct{ found, missing, key string }{ + {"/debian/dists/test/Release", "/debian/dists/test/InRelease", "deb:/debian/dists/test"}, + { + "/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", + "/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml.key", + "rpm:/almalinux/9/BaseOS/x86_64/os", + }, + } { + if resp, _ := get(t, srv, "", tc.found); resp.StatusCode != http.StatusOK { + t.Fatalf("%s status = %d", tc.found, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); !ok { + t.Fatalf("%s did not register %s", tc.found, tc.key) + } + if resp, _ := get(t, srv, "", tc.missing); resp.StatusCode != http.StatusNotFound { + t.Fatalf("%s status = %d, want 404", tc.missing, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); !ok { + t.Errorf("%s deregistered %s", tc.missing, tc.key) + } + } +} + +// TestServeUnresolvedEntryPoint verifies a path that only looks like a +// repository entry point is not left registered for the scheduler to crawl. +func TestServeUnresolvedEntryPoint(t *testing.T) { + srv, _, _ := serverFixture(t) + + for _, tc := range []struct{ path, key string }{ + {"/nope/repodata/repomd.xml", "rpm:/nope"}, + {"/nope/dists/sid/InRelease", "deb:/nope/dists/sid"}, + {"/nope/database.db", "arch:/nope"}, + {"/nope/" + mirror.APKIndexName, "apk:/nope"}, + } { + resp, _ := get(t, srv, "", tc.path) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s status = %d", tc.path, resp.StatusCode) + } + if _, ok := state.S.Entry(tc.key); ok { + t.Errorf("%s left %s registered", tc.path, tc.key) + } + } +} diff --git a/server/trace_test.go b/server/trace_test.go new file mode 100644 index 0000000..646db0c --- /dev/null +++ b/server/trace_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/grmrgecko/repo-sync/mirror" + "github.com/grmrgecko/repo-sync/state" +) + +// traceFixture builds an upstream apk repository that publishes a trace of +// its own, configures the mirror server against it with tracing enabled, +// and returns the mirror's test server and the online root. +func traceFixture(t *testing.T) (*httptest.Server, string) { + t.Helper() + + www := t.TempDir() + repo := filepath.Join(www, "alpine", "v3.24", "main", "x86_64") + testrepos.BuildApkRepo(t, repo) + testrepos.WriteFile(t, filepath.Join(repo, filepath.FromSlash(mirror.TraceDir), "origin.example.net"), + []byte("origin trace\n")) + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + confDir := t.TempDir() + conf := fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - domain: 127.0.0.1 + role: online + root: %s +mounts: + - path: / + upstream: %s +trace: + enabled: true + host: mirror.example.com + maintainer: Test Maintainer +`, confDir, onlineRoot, upstream.URL) + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(conf)) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + t.Cleanup(WaitCrawls) + return srv, onlineRoot +} + +// TestServeCrawlWritesTrace verifies a server crawl publishes this mirror's +// trace and mirrors upstream's, and that both are then served like any +// other repository file. +func TestServeCrawlWritesTrace(t *testing.T) { + srv, onlineRoot := traceFixture(t) + + repo := "/alpine/v3.24/main/x86_64" + resp, _ := get(t, srv, "", repo+"/APKINDEX.tar.gz") + if resp.StatusCode != http.StatusOK { + t.Fatalf("index status = %d", resp.StatusCode) + } + + traceDir := filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), filepath.FromSlash(mirror.TraceDir)) + own := filepath.Join(traceDir, "mirror.example.com") + waitFor(t, "crawl trace", func() bool { + _, err := os.Stat(own) + return err == nil + }) + + data, err := os.ReadFile(own) + if err != nil { + t.Fatal("reading trace:", err) + } + trace := string(data) + for _, want := range []string{ + "Running on host: mirror.example.com", + "Maintainer: Test Maintainer ", + "Repository type: apk", + "Total bytes received: ", + } { + if !strings.Contains(trace, want) { + t.Errorf("trace missing %q\ngot:\n%s", want, trace) + } + } + if upstreamTrace, err := os.ReadFile(filepath.Join(traceDir, "origin.example.net")); err != nil { + t.Error("upstream trace not mirrored:", err) + } else if string(upstreamTrace) != "origin trace\n" { + t.Errorf("upstream trace = %q, want %q", upstreamTrace, "origin trace\n") + } + + // The published traces are served from the crawled tree. + resp, body := get(t, srv, "", repo+"/"+mirror.TraceDir+"/mirror.example.com") + if resp.StatusCode != http.StatusOK { + t.Fatalf("trace request status = %d", resp.StatusCode) + } + if string(body) != trace { + t.Errorf("served trace does not match the published file\ngot:\n%s", body) + } +} + +// TestTraceInfo verifies the configured trace section maps onto the mirror +// package's description of this mirror, including the host fallback. +func TestTraceInfo(t *testing.T) { + got := traceInfo(cfg.TraceConfig{ + Host: "mirror.example.com", + Maintainer: "Jane ", + Sponsor: "Example Inc", + Country: "US", + Location: "Texas", + Throughput: "10Gb", + }) + want := mirror.TraceInfo{ + Host: "mirror.example.com", + Maintainer: "Jane ", + Sponsor: "Example Inc", + Country: "US", + Location: "Texas", + Throughput: "10Gb", + } + if got != want { + t.Errorf("traceInfo = %+v, want %+v", got, want) + } + if host := traceInfo(cfg.TraceConfig{}).Host; host == "" { + t.Error("traceInfo left the host empty; the system hostname should stand in") + } +} diff --git a/server_cmd.go b/server_cmd.go new file mode 100644 index 0000000..495cad5 --- /dev/null +++ b/server_cmd.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/coreos/go-systemd/v22/daemon" + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/server" + "github.com/grmrgecko/repo-sync/state" + "github.com/kardianos/service" + log "github.com/sirupsen/logrus" +) + +// stopChan carries a shutdown request from the service manager. It is +// buffered so a stop delivered after the signal loop exits cannot block the +// service supervisor. +var stopChan = make(chan struct{}, 1) + +// ServerCmd runs the caching mirror server. +type ServerCmd struct { + Bind string `name:"http.bind" help:"Override the configured bind address." optional:""` + Port uint `name:"http.port" help:"Override the configured port." optional:""` +} + +// updateCFG applies the command line overrides onto the active +// configuration. The overridden fields are read only when the listener is +// built, which happens on this goroutine, so a reload can amend them in +// place after Init has published the new configuration. +func (c *ServerCmd) updateCFG() { + if c.Bind != "" { + cfg.C.HTTP.BindAddr = c.Bind + } + if c.Port != 0 { + cfg.C.HTTP.Port = c.Port + } +} + +// Run starts the mirror server: configuration, state, crawl loops, and the +// HTTP listener, then handles signals until shutdown. +func (c *ServerCmd) Run() error { + // main installed the configuration; the server additionally needs the + // domains and mounts the sync commands do without. + c.updateCFG() + if err := cfg.C.RequireServer(); err != nil { + return err + } + + // Load persisted crawl state. + if err := state.Load(); err != nil { + return err + } + + // Start the background loops and the listener. + loopCtx, loopCancel := context.WithCancel(context.Background()) + defer loopCancel() + go state.S.FlushLoop(loopCtx) + crawlDone := make(chan struct{}) + go func() { + defer close(crawlDone) + server.CrawlLoop(loopCtx) + }() + + // stopCrawls cancels the loops and waits for in-flight crawls so their + // outcomes are recorded before the final state flush. The crawl loop + // must return first: a scheduler pass in flight can still dispatch + // crawls, and a WaitGroup Add racing Wait is a misuse panic. + stopCrawls := func() { + loopCancel() + <-crawlDone + server.WaitCrawls() + } + + httpServer := server.NewServer() + if err := httpServer.Start(); err != nil { + return err + } + + // Attach to the service manager when not run interactively, so a stop + // request arrives on stopChan, then report readiness now that the + // listener is bound. + if !service.Interactive() { + svc, err := new(ServiceCmd).service() + if err != nil { + return err + } + go svc.Run() + } + _, _ = daemon.SdNotify(false, daemon.SdNotifyReady) + + // Handle signals: reload on SIGHUP, shut down on SIGINT/SIGTERM or on a + // stop from the service manager. + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) +SigLoop: + for { + select { + case <-stopChan: + break SigLoop + case sig := <-signals: + if sig != syscall.SIGHUP { + break SigLoop + } + log.Info("Reloading configuration.") + if err := cfg.InitServer(flags.ConfigPath); err != nil { + log.WithError(err).Error("Unable to reload configuration; keeping previous values.") + continue + } + applyGlobalFlags() + c.updateCFG() + if err := httpServer.Reload(); err != nil { + // Exiting on a failed rebind must still flush crawl state. + stopCrawls() + if saveErr := state.S.Save(); saveErr != nil { + log.WithError(saveErr).Error("Failed to save state while shutting down.") + } + return err + } + } + } + + // Shut down: stop the listener, stop the loops, then flush state. + log.Info("Shutting down.") + httpServer.Shutdown() + stopCrawls() + return state.S.Save() +} diff --git a/service_cmd.go b/service_cmd.go new file mode 100644 index 0000000..bb30789 --- /dev/null +++ b/service_cmd.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/kardianos/service" +) + +// systemdScript is the unit template used when installing the service. It +// replaces the library default to run as a notify service with automatic +// restart, so systemd only considers the mirror started once the listener +// is bound. +const systemdScript = `[Unit] +Description={{Description}} +ConditionFileIsExecutable={{Path | cmdEscape}} +{{range Dependencies}}{{.}} +{{end}}StartLimitIntervalSec=500 +StartLimitBurst=5 + +[Service] +Type=notify +ExecStart={{Path | cmdEscape}}{{range Arguments}} {{. | cmd}}{{end}} +{{if ChRoot}}RootDirectory={{ChRoot | cmd}} +{{end}}{{if WorkingDirectory}}WorkingDirectory={{WorkingDirectory | cmdEscape}} +{{end}}{{if UserName}}User={{UserName}} +{{end}}{{if ReloadSignal}}ExecReload=/bin/kill -{{ReloadSignal}} "$MAINPID" +{{end}}{{if PIDFile}}PIDFile={{PIDFile | cmd}} +{{end}}{{if OutputFileSupport}}StandardOutput=file:{{LogDirectory}}/{{Name}}.out +StandardError=file:{{LogDirectory}}/{{Name}}.err +{{end}}{{if LimitNOFILE}}LimitNOFILE={{LimitNOFILE}} +{{end}}{{if Restart}}Restart={{Restart}} +{{end}}{{if SuccessExitStatus}}SuccessExitStatus={{SuccessExitStatus}} +{{end}}RestartSec=5 +EnvironmentFile=-/etc/sysconfig/{{Name}} + +{{range EnvVars}}{{.}} +{{end}}[Install] +WantedBy=multi-user.target +` + +// ServiceAction lists the accepted service actions, in the order the Run +// switch handles them. +var ServiceAction = []string{"start", "stop", "status", "restart", "install", "uninstall"} + +// ServiceCmd manages the mirror server as a system service. +type ServiceCmd struct { + Action string `arg:"" enum:"${serviceActions}" help:"${serviceActions}" required:""` +} + +// action returns the requested service action. +func (s *ServiceCmd) action() string { + return s.Action +} + +// Run performs the requested action against the installed service. +func (s *ServiceCmd) Run() (err error) { + svc, err := s.service() + if err != nil { + return err + } + switch s.action() { + case ServiceAction[0]: + err = svc.Start() + case ServiceAction[1]: + err = svc.Stop() + case ServiceAction[2]: + var status service.Status + status, err = svc.Status() + if err == nil { + switch status { + case service.StatusRunning: + fmt.Println("Service is running.") + case service.StatusStopped: + fmt.Println("Service is stopped.") + default: + fmt.Println("Service is in an unknown state.") + } + } + case ServiceAction[3]: + err = svc.Restart() + case ServiceAction[4]: + err = svc.Install() + case ServiceAction[5]: + err = svc.Uninstall() + } + if err != nil { + return err + } + // Status already printed its own result. + if s.action() != ServiceAction[2] { + fmt.Println("Command executed successfully.") + } + return +} + +// service builds the service definition shared by the management actions +// and by the server when it is started by the service manager. +func (s *ServiceCmd) service() (service.Service, error) { + svcConfig := &service.Config{ + Name: cfg.Name, + DisplayName: cfg.DisplayName, + Description: cfg.Description, + Arguments: []string{"server"}, + Dependencies: []string{"After=network.target"}, + Option: service.KeyValue{ + "SystemdScript": systemdScript, + "ReloadSignal": "HUP", + "Restart": "always", + }, + } + return service.New(s, svcConfig) +} + +// Start satisfies service.Interface. The server is already running in the +// foreground by the time the supervisor attaches. +func (s *ServiceCmd) Start(svc service.Service) error { + return nil +} + +// Stop satisfies service.Interface, signalling the server's shutdown. +func (s *ServiceCmd) Stop(svc service.Service) error { + stopChan <- struct{}{} + return nil +} diff --git a/state/state.go b/state/state.go new file mode 100644 index 0000000..010c614 --- /dev/null +++ b/state/state.go @@ -0,0 +1,236 @@ +// Package state persists the mirror server's per-resource crawl tracking +// so discovered repositories survive restarts. +package state + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "gopkg.in/yaml.v3" +) + +// Entry tracks one resource: a discovered or configured repository, or a +// generic cached file. +type Entry struct { + // Kind is the repository format, or "generic" for plain files. + Kind string `yaml:"kind"` + // Path is the request path of the resource (a deb entry uses the + // suite directory). + Path string `yaml:"path"` + // Root is the request path prefix covering every file the resource + // owns; requests below it keep the resource alive. + Root string `yaml:"root,omitempty"` + LastRequested time.Time `yaml:"last_requested,omitempty"` + LastCrawled time.Time `yaml:"last_crawled,omitempty"` + NextCrawl time.Time `yaml:"next_crawl,omitempty"` + LastSeenInInventory time.Time `yaml:"last_seen_in_inventory,omitempty"` + LastError string `yaml:"last_error,omitempty"` +} + +// file is the on-disk YAML layout. +type file struct { + Entries map[string]*Entry `yaml:"entries"` +} + +// Store is the in-memory state with dirty tracking for periodic flushes. +type Store struct { + mu sync.Mutex + file file + dirty bool +} + +// S is the active store, installed by Load. +var S *Store + +// Load reads the state file, treating a missing or empty file as a fresh +// store. Other read failures abort startup so a state file the process +// cannot read is never silently replaced. +func Load() error { + s := &Store{file: file{Entries: map[string]*Entry{}}} + name := cfg.C.StatePath + data, err := os.ReadFile(name) + switch { + case err == nil && len(data) > 0: + if err := yaml.Unmarshal(data, &s.file); err != nil { + return err + } + // A null or absent entries mapping unmarshals over the initialized + // map with a nil one, so the map is restored before any write. + if s.file.Entries == nil { + s.file.Entries = map[string]*Entry{} + } + // Drop null entries a hand-edited file may carry, as lookups + // dereference the stored pointer. + for key, e := range s.file.Entries { + if e == nil { + delete(s.file.Entries, key) + } + } + case err != nil && !errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("read state %s: %w", name, err) + } + S = s + return nil +} + +// MarkRequested records a client request for a resource, creating it if +// needed. It returns a copy of the entry and whether the resource was +// already tracked, so a caller can tell a registration this request created +// from one an earlier request established. +func (s *Store) MarkRequested(kind, key, path, root string, now time.Time) (Entry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, tracked := s.file.Entries[key] + if !tracked { + e = &Entry{} + s.file.Entries[key] = e + } + e.Kind = kind + e.Path = path + e.Root = root + e.LastRequested = now + s.dirty = true + return *e, tracked +} + +// MarkSeenInInventory records that a configured mount still advertises the +// resource, shielding it from eviction. +func (s *Store) MarkSeenInInventory(kind, key, path, root string, now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.file.Entries[key] + if e == nil { + e = &Entry{} + s.file.Entries[key] = e + } + e.Kind = kind + e.Path = path + e.Root = root + e.LastSeenInInventory = now + s.dirty = true +} + +// MarkCrawled records a crawl outcome, scheduling the next attempt sooner +// after failures. +func (s *Store) MarkCrawled(key string, now time.Time, crawlErr error) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.file.Entries[key] + if e == nil { + return + } + e.LastCrawled = now + if crawlErr != nil { + e.LastError = crawlErr.Error() + e.NextCrawl = now.Add(cfg.C.FailureRetryInterval) + } else { + e.LastError = "" + e.NextCrawl = now.Add(cfg.C.RepoCrawlInterval) + } + s.dirty = true +} + +// Delete removes a resource from tracking. +func (s *Store) Delete(key string) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.file.Entries[key]; ok { + delete(s.file.Entries, key) + s.dirty = true + } +} + +// Entry returns a copy of a tracked resource. +func (s *Store) Entry(key string) (Entry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.file.Entries[key] + if !ok { + return Entry{}, false + } + return *e, true +} + +// Snapshot returns a copy of every tracked resource. +func (s *Store) Snapshot() map[string]Entry { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]Entry, len(s.file.Entries)) + for key, e := range s.file.Entries { + out[key] = *e + } + return out +} + +// TouchRepoMembers refreshes the last-requested time of every repository +// whose root covers a request path, keeping repositories alive while their +// files are fetched. It reports whether any repository matched. +func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + matched := false + for _, e := range s.file.Entries { + if e.Kind == "generic" || e.Root == "" { + continue + } + under := reqPath == e.Root || e.Root == "/" || + (len(reqPath) > len(e.Root) && reqPath[:len(e.Root)] == e.Root && reqPath[len(e.Root)] == '/') + if !under { + continue + } + e.LastRequested = now + matched = true + s.dirty = true + } + return matched +} + +// Save writes the state atomically when it has changed. +func (s *Store) Save() error { + s.mu.Lock() + defer s.mu.Unlock() + if !s.dirty { + return nil + } + data, err := yaml.Marshal(s.file) + if err != nil { + return err + } + name := cfg.C.StatePath + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + return err + } + tmp := name + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + return err + } + if err := os.Rename(tmp, name); err != nil { + return err + } + s.dirty = false + return nil +} + +// FlushLoop saves periodically and once more at shutdown. +func (s *Store) FlushLoop(ctx context.Context) { + t := time.NewTicker(cfg.C.StateFlushInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + _ = s.Save() + return + case <-t.C: + _ = s.Save() + // Re-arm from the active config so a reload takes effect. + t.Reset(cfg.C.StateFlushInterval) + } + } +} diff --git a/state/state_test.go b/state/state_test.go new file mode 100644 index 0000000..1d36882 --- /dev/null +++ b/state/state_test.go @@ -0,0 +1,93 @@ +package state + +import ( + "os" + "path/filepath" + "testing" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" +) + +// initConfig installs a minimal configuration with the given state path. +func initConfig(t *testing.T, statePath string) { + t.Helper() + conf := ` +state_path: ` + statePath + ` +domains: + - {domain: m.test, role: online, root: /tmp/online} +mounts: + - {path: /, upstream: "https://example.com"} +` + name := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(name, []byte(conf), 0644); err != nil { + t.Fatal(err) + } + if err := cfg.Init(name); err != nil { + t.Fatal(err) + } +} + +// TestLoadMissing verifies a missing state file yields a fresh store. +func TestLoadMissing(t *testing.T) { + initConfig(t, filepath.Join(t.TempDir(), "state.yaml")) + if err := Load(); err != nil { + t.Fatal(err) + } + if len(S.Snapshot()) != 0 { + t.Error("fresh store is not empty") + } +} + +// TestLoadUnreadable verifies read failures other than a missing file are +// reported to the caller. +func TestLoadUnreadable(t *testing.T) { + initConfig(t, t.TempDir()) + if err := Load(); err == nil { + t.Fatal("expected an error for an unreadable state path") + } +} + +// TestLoadNullEntryMap verifies a state file whose entries mapping is null +// yields a writable store. +func TestLoadNullEntryMap(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.yaml") + if err := os.WriteFile(statePath, []byte("entries:\n"), 0644); err != nil { + t.Fatal(err) + } + initConfig(t, statePath) + if err := Load(); err != nil { + t.Fatal(err) + } + S.MarkRequested("rpm", "rpm:/repo", "/repo", "/repo", time.Now()) + if _, ok := S.Entry("rpm:/repo"); !ok { + t.Error("entry not recorded after loading a null entries mapping") + } +} + +// TestLoadNullEntries verifies hand-edited null entries are dropped at load +// so later lookups only see valid entries. +func TestLoadNullEntries(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.yaml") + content := ` +entries: + rpm:/repo: null + deb:/suite: + kind: deb + path: /suite +` + if err := os.WriteFile(statePath, []byte(content), 0644); err != nil { + t.Fatal(err) + } + initConfig(t, statePath) + if err := Load(); err != nil { + t.Fatal(err) + } + if _, ok := S.Entry("rpm:/repo"); ok { + t.Error("null entry survived load") + } + if _, ok := S.Entry("deb:/suite"); !ok { + t.Error("valid entry lost on load") + } + S.TouchRepoMembers("/repo/repodata/repomd.xml", time.Now()) +} diff --git a/sync_cmd.go b/sync_cmd.go new file mode 100644 index 0000000..883b7df --- /dev/null +++ b/sync_cmd.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/signal" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + cfg "github.com/grmrgecko/repo-sync/config" + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/mirror" +) + +// SyncArgs holds the flags and positional arguments shared by every +// repository type. +type SyncArgs struct { + Trim int `help:"Number of leading URL path components to drop when building the destination path." default:"0"` + Flat bool `help:"Place repositories directly in the destination directory without copying the URL path."` + Discover bool `help:"Treat each URL as a directory index and crawl it for repositories."` + Depth int `help:"Maximum directory depth crawled in discovery mode." default:"5"` + + DiscoverCache time.Duration `help:"Reuse a discovery crawl's results for this long before scanning the tree again (e.g. 72h); 0 scans every run, and unset defaults to the configured crawler value." default:"-1s"` + + Workers int `help:"Number of concurrent download workers; defaults to the configured crawler workers."` + Verify bool `help:"Re-verify checksums of files that already exist locally."` + Prune bool `help:"Delete local files that are no longer part of the repository."` + + Exclude []string `help:"Glob of directory or file names discovery never crawls; repeatable, and matched against the path below the crawled URL when it contains a slash, which a leading slash anchors to that URL (e.g. sles, 'yum/docker*', /docker)."` + IncludeFile []string `help:"Glob of loose files found outside repositories that discovery mirrors as well; repeatable, and matched by path on the same rules as --exclude (e.g. '*.rpm', 'RPM-GPG-KEY-*', '/*.rpm')."` + + PruneGrace time.Duration `help:"Keep files that left the repository for this long before pruning them (e.g. 72h); defaults to the configured crawler prune grace."` + DryRun bool `help:"Report what would be downloaded or pruned without changing the repository."` + + Missing string `help:"How to handle a package the repository metadata lists but the upstream does not serve: fail the repository, retry it across runs before accepting it as gone, or ignore it; defaults to the configured crawler mode." enum:",fail,retry,ignore" default:""` + MissingRetries int `help:"Number of consecutive runs a file must be missing upstream before --missing=retry accepts it as gone; defaults to the configured crawler value." default:"-1"` + + Args []string `arg:"" name:"url" help:"Repository or mirrorlist URLs followed by the destination directory."` +} + +// options validates the shared arguments and builds the mirror options. +// The types limit which repository formats the run accepts; an empty list +// accepts every supported format. +func (s *SyncArgs) options(types []mirror.RepoType) (*mirror.Options, error) { + if len(s.Args) < 2 { + return nil, errors.New("expected at least one repository URL and a destination directory") + } + urls := s.Args[:len(s.Args)-1] + for _, raw := range urls { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("invalid repository URL %q", raw) + } + } + dest, err := filepath.Abs(s.Args[len(s.Args)-1]) + if err != nil { + return nil, err + } + if s.Trim < 0 { + return nil, errors.New("trim must not be negative") + } + if s.Depth < 0 { + return nil, errors.New("depth must not be negative") + } + if s.PruneGrace < 0 { + return nil, errors.New("prune grace must not be negative") + } + + // Reject unusable globs up front: a malformed pattern would otherwise + // silently match nothing, which reads as a repository that vanished. + for _, pattern := range slices.Concat(s.Exclude, s.IncludeFile) { + if err := mirror.ValidPattern(pattern); err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + } + // The patterns describe a crawl, so without one they would do nothing. + if !s.Discover && (len(s.Exclude) > 0 || len(s.IncludeFile) > 0) { + return nil, errors.New("--exclude and --include-file apply to discovery; add --discover") + } + + // Unset worker and grace flags fall back to the crawler configuration, + // so a config file tunes the sync commands the same way it tunes the + // server's background crawls. + if s.Workers == 0 { + s.Workers = cfg.C.Crawler.Workers + } + if s.PruneGrace == 0 { + s.PruneGrace = cfg.C.Crawler.PruneGrace + } + if s.Missing == "" { + s.Missing = cfg.C.Crawler.MissingMode + } + if s.MissingRetries < 0 { + s.MissingRetries = cfg.C.Crawler.MissingRetries + } + if s.DiscoverCache < 0 { + s.DiscoverCache = cfg.C.Crawler.DiscoverCache + } + missing, err := fetch.ParseMissingMode(s.Missing) + if err != nil { + return nil, err + } + opts := &mirror.Options{ + Types: types, + URLs: urls, + Destination: dest, + Trim: s.Trim, + Flat: s.Flat, + Discover: s.Discover, + DiscoverDepth: s.Depth, + DiscoverCache: s.DiscoverCache, + Exclude: s.Exclude, + IncludeFiles: s.IncludeFile, + Workers: s.Workers, + Verify: s.Verify, + Prune: s.Prune, + PruneGrace: s.PruneGrace, + DryRun: s.DryRun, + Missing: missing, + MissingRetries: s.MissingRetries, + Trace: cfg.C.Trace.Enabled, + TraceInfo: mirror.TraceInfo{ + Host: cfg.C.Trace.HostName(), + Maintainer: cfg.C.Trace.Maintainer, + Sponsor: cfg.C.Trace.Sponsor, + Country: cfg.C.Trace.Country, + Location: cfg.C.Trace.Location, + Throughput: cfg.C.Trace.Throughput, + }, + } + + // A run limited to one format synchronizes every URL as that format, + // rather than identifying each from its metadata. + if len(types) == 1 { + opts.Type = types[0] + } + return opts, nil +} + +// run executes a synchronization with signal-aware cancellation, reporting +// what the run transferred once it ends. +func (s *SyncArgs) run(opts *mirror.Options) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + summary, err := mirror.Sync(ctx, opts) + printSummary(os.Stdout, summary) + return err +} + +// printSummary reports the run totals as "Field: value" lines, in the style +// of rsync's own statistics, so a calling script can act on what changed by +// reading the output. It is written to standard output rather than through +// the logger, which a configuration is free to send elsewhere or format as +// JSON, and it is reported even for a failed run, as a run that failed part +// way through still changed the destination. +func printSummary(w io.Writer, summary *mirror.Summary) { + if summary == nil { + return + } + fmt.Fprintln(w) + fmt.Fprintf(w, "Number of repositories: %d\n", summary.Repositories) + fmt.Fprintf(w, "Number of failed repositories: %d\n", summary.Failed) + + // A dry run transfers metadata to plan against, so reporting that as + // files transferred would read as a change. It reports what a real run + // would have moved instead. + if summary.DryRun { + fmt.Fprintf(w, "Dry run: true\n") + fmt.Fprintf(w, "Number of files to transfer: %d\n", summary.Planned) + fmt.Fprintf(w, "Total file size to transfer: %d bytes\n", summary.PlannedBytes) + } else { + fmt.Fprintf(w, "Number of files transferred: %d\n", summary.Fetched) + fmt.Fprintf(w, "Number of files pruned: %d\n", summary.Pruned) + fmt.Fprintf(w, "Total transferred file size: %d bytes\n", summary.FetchedBytes) + } + fmt.Fprintf(w, "Number of files unchanged: %d\n", summary.Unchanged) + fmt.Fprintf(w, "Number of files missing upstream: %d\n", summary.Missing) + fmt.Fprintf(w, "Repository changed: %t\n", summary.Changed()) +} + +// DebArgs holds the apt-specific filters, shared by the deb command and the +// type-agnostic sync command, which reaches apt repositories too. +type DebArgs struct { + Component []string `help:"Limit apt synchronization to these components."` + Arch []string `help:"Limit apt synchronization to these architectures; include \"source\" to keep source indexes."` +} + +// apply copies the apt filters onto the mirror options. +func (d *DebArgs) apply(opts *mirror.Options) { + opts.Components = d.Component + opts.Architectures = d.Arch +} + +// parseTypes converts requested type names into repository types, rejecting +// unsupported ones. An empty request accepts every supported format. +func parseTypes(names []string) ([]mirror.RepoType, error) { + var types []mirror.RepoType + for _, name := range names { + typ := mirror.RepoType(strings.ToLower(strings.TrimSpace(name))) + if !slices.Contains(mirror.AllTypes, typ) { + return nil, fmt.Errorf("unsupported repository type %q; expected one of %s", + name, strings.Join(mirror.TypeNames(), ", ")) + } + if !slices.Contains(types, typ) { + types = append(types, typ) + } + } + return types, nil +} + +// SyncCmd synchronizes repositories of any supported type. Each repository +// is identified by its own metadata, so one run covers an upstream that +// publishes several formats; --type limits which formats are accepted. +type SyncCmd struct { + Type []string `help:"Repository types to synchronize; repeatable, and defaults to every supported type."` + + DebArgs `embed:""` + SyncArgs `embed:""` +} + +// Run performs the synchronization. +func (c *SyncCmd) Run() error { + types, err := parseTypes(c.Type) + if err != nil { + return err + } + opts, err := c.options(types) + if err != nil { + return err + } + c.apply(opts) + return c.run(opts) +} diff --git a/sync_cmd_test.go b/sync_cmd_test.go new file mode 100644 index 0000000..f82c79c --- /dev/null +++ b/sync_cmd_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "strings" + "testing" + + "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/mirror" +) + +// summaryFields parses a printed summary back into its fields. +func summaryFields(t *testing.T, out string) map[string]string { + t.Helper() + fields := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" { + continue + } + name, value, ok := strings.Cut(line, ": ") + if !ok { + t.Fatalf("summary line %q is not a field", line) + } + fields[name] = value + } + return fields +} + +// TestPrintSummary verifies the printed summary reports what a calling script +// keys off, and that a dry run never reports a change. +func TestPrintSummary(t *testing.T) { + tests := []struct { + name string + summary *mirror.Summary + want map[string]string + absent []string + }{ + { + name: "a run that transferred files reports a change", + summary: &mirror.Summary{ + Repositories: 2, + Fetched: 7, + FetchedBytes: 4096, + Unchanged: 31, + Pruned: 2, + }, + want: map[string]string{ + "Number of repositories": "2", + "Number of failed repositories": "0", + "Number of files transferred": "7", + "Number of files pruned": "2", + "Total transferred file size": "4096 bytes", + "Number of files unchanged": "31", + "Repository changed": "true", + }, + absent: []string{"Dry run", "Number of files to transfer"}, + }, + { + name: "an idle run reports no change", + summary: &mirror.Summary{Repositories: 1, Unchanged: 12}, + want: map[string]string{ + "Number of files transferred": "0", + "Number of files pruned": "0", + "Repository changed": "false", + }, + }, + { + name: "pruning alone is a change", + summary: &mirror.Summary{Repositories: 1, Pruned: 1}, + want: map[string]string{ + "Number of files pruned": "1", + "Repository changed": "true", + }, + }, + { + name: "a failed run still reports its totals", + summary: &mirror.Summary{Repositories: 3, Failed: 1, Fetched: 4}, + want: map[string]string{ + "Number of repositories": "3", + "Number of failed repositories": "1", + "Repository changed": "true", + }, + }, + { + name: "a dry run reports the work it planned but no change", + summary: &mirror.Summary{ + Repositories: 1, + DryRun: true, + Fetched: 4, + Planned: 9, + PlannedBytes: 8192, + }, + want: map[string]string{ + "Dry run": "true", + "Number of files to transfer": "9", + "Total file size to transfer": "8192 bytes", + "Repository changed": "false", + }, + // The metadata a dry run fetches must never be reported as + // files it transferred. + absent: []string{"Number of files transferred", "Number of files pruned"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + printSummary(&out, tc.summary) + got := summaryFields(t, out.String()) + for name, want := range tc.want { + if got[name] != want { + t.Errorf("%s = %q, want %q", name, got[name], want) + } + } + for _, name := range tc.absent { + if _, ok := got[name]; ok { + t.Errorf("%s was reported when it does not apply", name) + } + } + }) + } +} + +// TestPrintSummaryNone verifies a run with no summary to report prints +// nothing, rather than an empty set of fields a script would misread. +func TestPrintSummaryNone(t *testing.T) { + var out bytes.Buffer + printSummary(&out, nil) + if out.Len() != 0 { + t.Errorf("printed %q without a summary", out.String()) + } +} + +// TestSyncArgsMissing verifies missing file handling falls back to the +// crawler configuration and that the flags override it. +func TestSyncArgsMissing(t *testing.T) { + dest := t.TempDir() + + // An unset flag pair takes the configured defaults, which is what a run + // without a config file gets from the built-in ones. + args := SyncArgs{Args: []string{"https://example.com/repo", dest}, MissingRetries: -1} + opts, err := args.options(nil) + if err != nil { + t.Fatal(err) + } + if opts.Missing != fetch.MissingRetry || opts.MissingRetries != 3 { + t.Errorf("missing = %q after %d runs, want retry after 3", opts.Missing, opts.MissingRetries) + } + + // Explicit flags win, including a zero retry budget, which accepts an + // absence the first time it is seen. + args = SyncArgs{Args: []string{"https://example.com/repo", dest}, Missing: "ignore", MissingRetries: 0} + opts, err = args.options(nil) + if err != nil { + t.Fatal(err) + } + if opts.Missing != fetch.MissingIgnore || opts.MissingRetries != 0 { + t.Errorf("missing = %q after %d runs, want ignore after 0", opts.Missing, opts.MissingRetries) + } + + // An unsupported mode is rejected rather than silently ignored. + args = SyncArgs{Args: []string{"https://example.com/repo", dest}, Missing: "skip", MissingRetries: -1} + if _, err := args.options(nil); err == nil { + t.Error("an unsupported missing mode was accepted") + } +}