GRMrGecko/repo-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

first commit
Some checks are pending
Go package / build (push) Waiting to run
Details

Browse source
This commit is contained in:
James Coleman 2026-07-27 14:16:32 -05:00
commit b6b3bfc43f
63 changed files with 11391 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
.github/workflows/release.yaml vendored Normal file
View file

@ -0,0 +1,31 @@
on:
release:
types: [created]
permissions:
contents: write
packages: write
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v7
with:
fetch-depth: 0
-
name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
-
name: Run GoReleaser
uses: goreleaser/goreleaser-action@v7
with:
distribution: goreleaser
version: '~> v2'
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

21
.github/workflows/test_golang.yaml vendored Normal file
View file

@ -0,0 +1,21 @@
name: Go package
on: [push]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build
run: go build -v ./...
- name: Test
run: go test -v ./...

7
.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
/repo-sync
/dist/
*.exe
*.test
*.out
*.prof

61
.goreleaser.yaml Normal file
View file

@ -0,0 +1,61 @@
# GoReleaser config for repo-sync.
# https://goreleaser.com
#
# CGO is disabled so the binary is fully static (no glibc dependency) and runs
# unmodified across modern Linux distributions.
version: 2
project_name: repo-sync
before:
hooks:
- go mod tidy
- go test ./...
builds:
- id: repo-sync
main: .
binary: repo-sync
env:
- CGO_ENABLED=0
flags:
- -trimpath
ldflags:
- -s -w
- -X github.com/grmrgecko/repo-sync/config.Version={{ .Version }}
- -X github.com/grmrgecko/repo-sync/config.Commit={{ .ShortCommit }}
- -X github.com/grmrgecko/repo-sync/config.Date={{ .Date }}
- -X github.com/grmrgecko/repo-sync/config.Mode=release
goos:
- linux
goarch:
- "386"
- amd64
- arm
- arm64
- ppc64le
goarm:
- "6"
archives:
- id: default
formats: [tar.gz]
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
files:
- README.md
- LICENSE
checksum:
name_template: "checksums.txt"
snapshot:
version_template: "{{ incpatch .Version }}-snapshot"
changelog:
use: git
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
- "^chore:"

19
LICENSE Normal file
View file

@ -0,0 +1,19 @@
Copyright (c) 2026 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

37
Makefile Normal file
View file

@ -0,0 +1,37 @@
BINARY := repo-sync
MODULE := github.com/grmrgecko/repo-sync/config
# VERSION is the single source of truth for the version string. COMMIT and DATE
# are derived from git and the build clock.
VERSION ?= $(shell cat VERSION 2>/dev/null || echo dev)
COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null)
DATE := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
LDFLAGS := -s -w -X $(MODULE).Version=$(VERSION) -X $(MODULE).Commit=$(COMMIT) -X $(MODULE).Date=$(DATE)
.PHONY: all build test vet fmt snapshot release clean
all: build
## build: native static binary into dist/
build:
CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(BINARY) .
## test: run the unit tests
test:
go test ./...
vet:
go vet ./...
fmt:
gofmt -w .
## snapshot: local GoReleaser build without publishing (artifacts in dist/)
snapshot:
goreleaser release --snapshot --clean
## release: full GoReleaser release (CI runs this on a tag)
release:
goreleaser release --clean
clean:
rm -rf dist

225
README.md Normal file
View file

@ -0,0 +1,225 @@
# repo-sync
A universal Linux package repository synchronization tool. It mirrors remote
repositories into a local directory tree, preserving the upstream layout so
the result can be served directly to package managers.
Supported repository types, named by `--type` when a run has to be limited
to some of them:
- **rpm** — yum/dnf repositories (`repodata/repomd.xml`), including
plain-text mirrorlist and metalink URLs with failover between mirrors.
- **deb** — apt repositories, both standard (`dists/<suite>` with a shared
`pool/`) and flat layouts, including `Acquire-By-Hash` population.
- **arch** — pacman repositories (`<name>.db`), including detached package
signatures and companion metadata (`.files`, `.db.tar.gz`, `.links.tar.gz`).
- **apk** — Alpine Linux repositories (`APKINDEX.tar.gz` with packages
beside it).
## Usage
```
repo-sync sync [flags] <url> [<url> ...] <destination-directory>
```
The last argument is always the destination directory; every argument before
it is a repository or mirrorlist URL, synchronized one after the other.
Each repository is identified by its own metadata, so no type has to be
given: an upstream publishing both yum and apt repositories is mirrored in
one run. `--type` limits which formats are accepted, and is repeatable; a
run limited to a single type synchronizes every URL as that type rather than
identifying it, which is what a mirrorlist or metalink URL needs.
A configuration file is optional: without one the built-in defaults apply.
When one is present its `crawler` section supplies the defaults for every
command, so `workers`, `prune_grace`, `request_timeout`, `user_agent`,
`missing_mode`, and `missing_retries` are shared by the sync commands and
the server. The global flags `--config-path`, `--log-level`, `--user-agent`,
and `--request-timeout` override the configuration for any command.
```sh
# Mirror one repository. The URL path is copied below the destination, so
# this produces ./mirror/repos/CentOS/7/EA4/.
repo-sync sync https://example.com/repos/CentOS/7/EA4/ ./mirror
# Trim the first two path components: ./mirror/7/EA4/.
repo-sync sync --trim 2 https://example.com/repos/CentOS/7/EA4/ ./mirror
# No path copying at all: the repository lands directly in ./mirror.
repo-sync sync --flat https://example.com/repos/CentOS/7/EA4/ ./mirror
# Multiple repositories in one run.
repo-sync sync https://example.com/repos/a/ https://example.com/repos/b/ ./mirror
# Crawl directory listings for repositories, at most 4 levels deep.
repo-sync sync --discover --depth 4 https://example.com/repos/ ./mirror
# A pacman repository. The database name is discovered from the directory
# listing, or by probing URL path segments when listings are disabled.
repo-sync sync https://example.com/archlinux/core/os/x86_64/ ./mirror
# An Alpine repository is one arch directory; discovery syncs every arch
# of a release/repo tree in one run.
repo-sync sync https://dl-cdn.alpinelinux.org/alpine/v3.24/community/x86_64/ ./mirror
repo-sync sync --discover --depth 1 https://dl-cdn.alpinelinux.org/alpine/v3.24/community/ ./mirror
# A Fedora-style metalink URL; mirrors are used in preference order. Nothing
# can be crawled or probed through a metalink, so the type is given. The
# mirrors' paths differ, so --flat keeps the destination stable.
repo-sync sync --type rpm --flat 'https://mirrors.fedoraproject.org/metalink?repo=epel-9&arch=x86_64' ./mirror/epel9
# An apt suite. Pool files resolve against the archive root, producing
# ./mirror/debian/dists/bookworm/ and ./mirror/debian/pool/.
repo-sync sync https://deb.example.com/debian/dists/bookworm ./mirror
# Limit an apt mirror to specific components and architectures. Include
# "source" as an architecture to keep source indexes.
repo-sync sync --component main --arch amd64,source https://deb.example.com/debian/dists/bookworm ./mirror
# Limit a crawl to one format when an upstream publishes several.
repo-sync sync --type rpm --discover https://repo.example.com/ ./mirror
```
## Discovery
`--discover` treats each URL as a directory index and crawls it for
repositories, at most `--depth` levels down. A directory holding repository
metadata is mirrored as a whole and is not crawled further, so a vendor
archive is mirrored by pointing one run at its root.
`--exclude` keeps parts of that tree out of the crawl. Each pattern is a
glob, repeatable, and matched against the entry name wherever it appears in
the tree; a pattern containing a slash is matched against the path below the
crawled URL instead, pinning it to one place, and a leading slash anchors it
to that URL. An excluded directory is never listed, so nothing below it is
mirrored either.
`--include-file` mirrors loose files the crawl passes on its way, which no
repository's metadata lists — signing keys, release notes, and packages
published outside a repository. Each pattern is a glob, repeatable, and
matched by name or by path on the same rules as `--exclude`, so `'*.rpm'`
takes loose packages from anywhere in the tree while `'/*.rpm'` takes only
those beside the crawled URL itself. Only files in directories that are
*not* repositories are considered: a repository's own files come from its
metadata, and pruning there would remove anything else. These files carry no
published checksum, so they are revalidated against their modification time
rather than re-downloaded, and `--prune` never removes them.
```sh
# Mirror MySQL's yum and apt repositories in one run, skipping the
# distributions this mirror does not serve, and keeping the signing keys
# and loose packages published outside the repositories.
repo-sync sync --discover --depth 6 \
--exclude sles --exclude 'fc*' --exclude docker \
--include-file '*.rpm' --include-file '*.deb' --include-file 'RPM-GPG-KEY-mysql*' \
http://repo.mysql.com/ ./mirror
```
## Trace files
A public mirror is expected to say something about itself: who runs it,
where it is, and when it last synchronized. Debian archives established a
convention for this — a file at `project/trace/<host>` inside the archive —
and downstream mirrors and mirror checkers read it. `--trace` publishes one
into every repository synchronized, and the mirror server publishes one
into every repository it crawls when the `trace` section is enabled.
```sh
repo-sync sync --trace --trace-maintainer 'Jane <jane@example.com>' \
https://example.com/repos/CentOS/7/EA4/ ./mirror
```
```
Mon Jul 27 20:31:58 UTC 2026
Date: Mon, 27 Jul 2026 20:31:58 +0000
Date-Started: Mon, 27 Jul 2026 20:29:14 +0000
Creator: repo-sync 0.1.0
Running on host: mirror.example.com
Maintainer: Jane <jane@example.com>
Repository type: rpm
Upstream-mirror: https://example.com/repos/CentOS/7/EA4
Total bytes received: 5726208
Total time spent syncing: 164
Average rate: 34915 B/s
```
`Upstream-mirror` records the mirror actually fetched from, which for a
mirrorlist or metalink is the one that answered rather than the list's own
URL — the reason a trace is written here rather than by whatever schedules
the sync, which cannot know that.
The file is placed in the repository directory, or for apt at the archive
root, where the convention puts it. It is registered as part of the
repository, so `--prune` never removes it even when the repository and the
trace share a directory. `--dry-run` reports what would change without
writing one.
Everything but the dates and transfer figures comes from configuration,
and each field has a matching flag: `--trace-host` (defaults to the system
hostname), `--trace-maintainer`, `--trace-sponsor`, `--trace-country`,
`--trace-location`, and `--trace-throughput`. Setting them in the `trace`
section of a config file avoids repeating them on every run; `--trace` and
`--no-trace` then switch tracing on or off per run. Fields left unset are
omitted from the file rather than written empty.
Upstream's own traces are mirrored alongside, so `project/trace/` names
every mirror the content passed through and a reader can follow the chain
back to the archive it originated from. A trace upstream publishes under
this mirror's own host name is skipped: that file describes a different
mirror, and the one written here has to win. Upstreams that publish no
traces, or that do not serve a directory index for them, are not an error;
nothing is copied and the run continues.
Traces work the same on the mirror server, whose crawls are concurrent:
each crawl accounts for its own transfer rather than reading the shared
counters, so the totals in each trace describe that crawl alone. Every
completed crawl republishes the repository's trace, so the timestamps
track how current the cached copy is.
## Mirror server
`repo-sync server` runs a caching mirror in front of upstream repositories.
Repositories are discovered from client requests and then kept synchronized
in the background, so package managers can be pointed straight at it.
```sh
repo-sync server --config-path ./config.yaml
```
Without `--config-path` the config is read from `./config.yaml`,
`~/.config/repo-sync/config.yaml`, or `/etc/repo-sync/config.yaml`. See
[config.example.yaml](config.example.yaml) for a documented configuration
covering the listener, domains, mounts, and crawler tuning. The server
needs at least one domain and one mount; the sync commands do not.
To run it as a systemd service, install `/etc/repo-sync/config.yaml` and:
```sh
repo-sync service install
repo-sync service start
```
`service` also accepts `stop`, `restart`, `status`, and `uninstall`. The
installed unit runs `repo-sync server` as a notify service, restarts on
failure, and reloads its configuration on `systemctl reload repo-sync`.
## Building
```sh
make
```
The build stamps the binary with the contents of the `VERSION` file plus the
git commit and build date, shown by `repo-sync --version`. A plain
`go build` also works but reports the version as `dev`.
## Testing
```sh
make test
```
The suite is hermetic: fixture repositories for every supported format
(rpm, deb, arch, apk) are generated in temp directories and served over
local HTTP, so no network access is required.

1
VERSION Normal file
View file

@ -0,0 +1 @@
0.1.0

148
config.example.yaml Normal file
View file

@ -0,0 +1,148 @@
# Example repo-sync mirror server configuration.
#
# The server proxies and caches upstream package mirrors. Repositories are
# discovered from client requests: fetching a repository's entry-point
# metadata (repodata/repomd.xml, dists/<suite>/InRelease, <name>.db,
# APKINDEX.tar.gz) registers it for full synchronization and periodic
# refreshing. Repositories listed under a mount's `repos` are kept
# synchronized regardless of traffic.
# state_path persists discovered repositories across restarts. When unset
# it defaults to state.yaml next to the config file (./config.yaml keeps
# state in ./state.yaml), or /etc/repo-sync/state.yaml without one.
state_path: /etc/repo-sync/state.yaml
state_flush_interval: 30s
# repo_crawl_interval is how often known repositories are refreshed while
# they keep being requested. failure_retry_interval schedules the next
# attempt after a failed crawl. crawl_check_interval is the scheduler tick.
repo_crawl_interval: 4h
failure_retry_interval: 1h
crawl_check_interval: 30m
# generic_max_age bounds how stale a plain cached file (anything outside a
# repository) may be before it is revalidated upstream. Upstream 404s are
# remembered for negative_cache_ttl and transient errors for
# transient_error_ttl so failures are not amplified per client.
generic_max_age: 6h
negative_cache_ttl: 1h
transient_error_ttl: 1m
http:
bind_addr: ""
port: 8080
read_header_timeout: 10s
read_timeout: 5m
write_timeout: 30m
idle_timeout: 2m
# directory_indexes enables directory listings: upstream index pages are
# passed through, and when no catch-all "/" mount exists the root serves
# a generated index of the mounts and their configured repositories.
# When false, directory requests answer with a static notice page.
directory_indexes: true
log:
# Severity threshold: debug, info, warn, error.
level: info
# Formatter: console (text) or json.
type: console
# Outputs may include "console" (stderr) or explicit file paths.
outputs:
- console
# domains map the request Host header to a filesystem tree. Exactly one
# domain has the online role: its tree is the writable cache that misses
# are fetched into and crawls write to. Offline domains are read-only
# published trees that fall back to the online tree for anything missing,
# and are promoted out-of-band (e.g. by snapshot tooling).
domains:
- domain: mirror.example.com
role: online
root: /var/lib/repo-sync/online
- domain: stage.mirror.example.com
role: offline
root: /var/lib/repo-sync/stage
# mounts map base request paths onto upstream mirrors; the longest matching
# path wins, and "/" is allowed as a catch-all. A request for
# /almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml resolves through the
# /almalinux mount and registers that path as an RPM repository. Each mount
# may also pin repositories that stay synchronized without client traffic.
mounts:
- path: /almalinux
upstream: https://mirror.example.com/almalinux
repos:
- path: /almalinux/9/BaseOS/x86_64/os
type: rpm
- path: /centos
upstream: https://vault.centos.org
- path: /ubuntu
upstream: https://mirror.example.com/ubuntu
- path: /archlinux
upstream: https://mirror.example.com/archlinux
- path: /alpine
upstream: https://dl-cdn.alpinelinux.org/alpine
# The crawler section also applies to the sync commands: they use these
# workers, prune_grace, request_timeout, and user_agent unless the
# equivalent flag overrides them.
crawler:
# workers is the number of download workers inside a single crawl;
# concurrent_crawls bounds how many repository crawls run at once.
workers: 4
concurrent_crawls: 4
# request_timeout bounds how long an upstream may take to start
# responding to a request (response header timeout).
request_timeout: 2m
# user_agent overrides the default repo-sync/<version>.
user_agent: ""
# prune_grace keeps files that left a repository for this long before
# crawl-time pruning removes them.
prune_grace: 24h
# missing_mode decides what happens when an upstream does not serve a
# package its repository metadata lists, which published repositories are
# not rare in doing:
# retry - mirror the rest of the repository and keep reporting the
# absence as a failure until the same file has been missing for
# missing_retries consecutive runs, after which it is recorded
# in the repository as a known absence and stops failing runs
# fail - stop the repository at the first missing file
# ignore - skip missing files without ever failing
# A repository's own metadata is always required regardless of the mode.
missing_mode: retry
missing_retries: 3
# Backoff steps applied once a repository stops being requested. The
# first tier is repo_crawl_interval; each entry extends the refresh
# interval as the resource ages, and once the total budget elapses the
# resource is evicted from the online cache.
refresh_schedule:
- 12h
- 24h
- 48h
- 72h
- 96h
- 120h
# The trace section publishes a file describing this mirror into every
# repository the sync commands synchronize and the server crawls, following
# the convention Debian archives established at project/trace/<host>.
# Downstream mirrors and mirror checkers read it to learn who runs a mirror,
# where it is, and when it last synchronized. The traces upstream publishes
# are mirrored beside it, so the directory names every mirror the content
# passed through. It is disabled by default, and every field can be
# overridden with the matching --trace-* flag.
trace:
# Traces are written only when this is enabled, or --trace is passed.
enabled: false
# The name this mirror is traced under, which is also the trace file's
# name. Defaults to the system hostname.
host: ""
# Contact for whoever runs this mirror, in "name <email>" form.
maintainer: ""
# Who provides the hardware or bandwidth, if anyone.
sponsor: ""
# Where the mirror is, for clients choosing a nearby one.
country: ""
location: ""
# How much bandwidth the mirror has available.
throughput: ""

509
config/config.go Normal file
View file

@ -0,0 +1,509 @@
package config
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/go-playground/validator/v10"
"github.com/spf13/viper"
)
// Domain roles: the online domain is the writable upstream cache, offline
// domains are read-only published trees that fall back to the online tree.
const (
RoleOnline = "online"
RoleOffline = "offline"
)
// HTTPConfig configures the mirror server listener.
type HTTPConfig struct {
BindAddr string `mapstructure:"bind_addr" yaml:"bind_addr"`
Port uint `mapstructure:"port" yaml:"port" validate:"gt=0,lte=65535"`
ReadHeaderTimeout time.Duration `mapstructure:"read_header_timeout" yaml:"read_header_timeout" validate:"gt=0"`
ReadTimeout time.Duration `mapstructure:"read_timeout" yaml:"read_timeout" validate:"gt=0"`
WriteTimeout time.Duration `mapstructure:"write_timeout" yaml:"write_timeout" validate:"gt=0"`
IdleTimeout time.Duration `mapstructure:"idle_timeout" yaml:"idle_timeout" validate:"gt=0"`
// DirectoryIndexes enables directory listings: upstream index pages are
// passed through, and the root serves a generated index of the mounts
// when no catch-all mount covers it. When disabled, every directory
// request answers with a static notice page instead.
DirectoryIndexes bool `mapstructure:"directory_indexes" yaml:"directory_indexes"`
}
// ListenAddr renders the bind address and port for net.Listen, bracketing
// IPv6 addresses as required.
func (h HTTPConfig) ListenAddr() string {
return net.JoinHostPort(h.BindAddr, strconv.Itoa(int(h.Port)))
}
// LogConfig configures the server's logging output.
type LogConfig struct {
Level string `mapstructure:"level" yaml:"level" validate:"oneof=debug info warn error"`
Type string `mapstructure:"type" yaml:"type" validate:"oneof=console json"`
Outputs []string `mapstructure:"outputs" yaml:"outputs" validate:"required,min=1,dive,required"`
}
// DomainConfig maps a request Host header to a filesystem root and role.
type DomainConfig struct {
Domain string `mapstructure:"domain" yaml:"domain" validate:"required"`
Root string `mapstructure:"root" yaml:"root" validate:"required"`
Role string `mapstructure:"role" yaml:"role" validate:"required,oneof=online offline"`
}
// MountRepoConfig is a repository kept synchronized regardless of client
// traffic, identified by its request path and format.
type MountRepoConfig struct {
Path string `mapstructure:"path" yaml:"path" validate:"required"`
Type string `mapstructure:"type" yaml:"type" validate:"required,oneof=rpm deb arch apk"`
}
// MountConfig maps a base request path onto an upstream mirror URL.
type MountConfig struct {
Path string `mapstructure:"path" yaml:"path" validate:"required"`
Upstream string `mapstructure:"upstream" yaml:"upstream" validate:"required,url"`
Repos []MountRepoConfig `mapstructure:"repos" yaml:"repos" validate:"omitempty,dive"`
}
// CrawlerConfig tunes the background synchronization behavior.
type CrawlerConfig struct {
Workers int `mapstructure:"workers" yaml:"workers" validate:"gt=0"`
ConcurrentCrawls int `mapstructure:"concurrent_crawls" yaml:"concurrent_crawls" validate:"gt=0"`
RequestTimeout time.Duration `mapstructure:"request_timeout" yaml:"request_timeout" validate:"gt=0"`
UserAgent string `mapstructure:"user_agent" yaml:"user_agent"`
PruneGrace time.Duration `mapstructure:"prune_grace" yaml:"prune_grace" validate:"gte=0"`
RefreshSchedule []time.Duration `mapstructure:"refresh_schedule" yaml:"refresh_schedule" validate:"required,min=1,dive,gt=0"`
// MissingMode decides what a synchronization does with a package the
// repository metadata lists but the upstream does not serve. The mode
// names are the mirror's own; the fetch package parses them.
MissingMode string `mapstructure:"missing_mode" yaml:"missing_mode" validate:"omitempty,oneof=fail retry ignore"`
// MissingRetries is how many consecutive runs must miss a file before
// the retry mode accepts it as gone.
MissingRetries int `mapstructure:"missing_retries" yaml:"missing_retries" validate:"gte=0"`
}
// TraceConfig describes the mirror this instance publishes, and is written
// into each synchronized or crawled repository as a trace file when
// enabled. The fields follow the trace convention Debian archives
// established, which downstream mirrors and mirror checkers read to learn
// who runs a mirror, where it is, and when it last synchronized.
type TraceConfig struct {
Enabled bool `mapstructure:"enabled" yaml:"enabled"`
// Host names this mirror in the trace, and is also the trace file's
// name. It defaults to the system hostname.
Host string `mapstructure:"host" yaml:"host"`
Maintainer string `mapstructure:"maintainer" yaml:"maintainer"`
Sponsor string `mapstructure:"sponsor" yaml:"sponsor"`
Country string `mapstructure:"country" yaml:"country"`
Location string `mapstructure:"location" yaml:"location"`
Throughput string `mapstructure:"throughput" yaml:"throughput"`
}
// HostName returns the name this mirror is traced under, falling back to
// the system hostname when none is configured. The fallback lives here
// rather than in the defaults so it also applies to the built-in
// configuration, which is what a run without a config file reads.
func (t TraceConfig) HostName() string {
if t.Host != "" {
return t.Host
}
if name, err := os.Hostname(); err == nil && name != "" {
return name
}
return "unknown"
}
// Config is the mirror server configuration.
type Config struct {
HTTP HTTPConfig `mapstructure:"http" yaml:"http"`
Log LogConfig `mapstructure:"log" yaml:"log"`
StatePath string `mapstructure:"state_path" yaml:"state_path" validate:"required"`
StateFlushInterval time.Duration `mapstructure:"state_flush_interval" yaml:"state_flush_interval" validate:"gt=0"`
RepoCrawlInterval time.Duration `mapstructure:"repo_crawl_interval" yaml:"repo_crawl_interval" validate:"gt=0"`
FailureRetryInterval time.Duration `mapstructure:"failure_retry_interval" yaml:"failure_retry_interval" validate:"gt=0"`
CrawlCheckInterval time.Duration `mapstructure:"crawl_check_interval" yaml:"crawl_check_interval" validate:"gt=0"`
GenericMaxAge time.Duration `mapstructure:"generic_max_age" yaml:"generic_max_age" validate:"gt=0"`
NegativeCacheTTL time.Duration `mapstructure:"negative_cache_ttl" yaml:"negative_cache_ttl" validate:"gt=0"`
TransientErrorTTL time.Duration `mapstructure:"transient_error_ttl" yaml:"transient_error_ttl" validate:"gt=0"`
Domains []DomainConfig `mapstructure:"domains" yaml:"domains" validate:"omitempty,dive"`
Mounts []MountConfig `mapstructure:"mounts" yaml:"mounts" validate:"omitempty,dive"`
Crawler CrawlerConfig `mapstructure:"crawler" yaml:"crawler"`
Trace TraceConfig `mapstructure:"trace" yaml:"trace"`
domainsByName map[string]int
onlineIndex int
mountOrder []int
}
// C is the active server configuration. A reload builds and validates a
// replacement in full before repointing C at it, so readers always see a
// complete configuration; requests already in flight finish against the
// one they picked up.
var C *Config
// C starts out holding the built-in defaults so it is never nil. Commands
// that run without a configuration file, and packages exercised by tests
// that never call Init, read those defaults instead of guarding for nil.
func init() {
C = defaults()
}
// defaults builds a configuration from the registered defaults alone,
// consulting no file. Nothing here can fail: the values are static and the
// indexes start empty, so validation is left to Init.
func defaults() *Config {
v := viper.New()
setDefaults(v)
c := &Config{}
// Unmarshaling static defaults into their own struct cannot fail.
_ = v.Unmarshal(c)
c.StatePath = "/etc/repo-sync/state.yaml"
c.domainsByName = map[string]int{}
c.onlineIndex = -1
return c
}
// loadConfig reads and validates a configuration file. Init installs the
// result; callers work with the C singleton rather than loading their own.
func loadConfig(configPath string) (*Config, error) {
v := viper.New()
setDefaults(v)
file := findConfig(configPath)
if file != "" {
v.SetConfigFile(file)
if err := v.ReadInConfig(); err != nil {
return nil, fmt.Errorf("read config %s: %w", file, err)
}
}
c := &Config{}
if err := v.Unmarshal(c); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
// Default the state file next to the config file, so ./config.yaml
// keeps state in ./state.yaml, or to the system path without one.
if c.StatePath == "" {
if file != "" {
c.StatePath = filepath.Join(filepath.Dir(file), "state.yaml")
} else {
c.StatePath = "/etc/repo-sync/state.yaml"
}
}
if err := c.finalize(); err != nil {
return nil, err
}
return c, nil
}
// Init loads and validates the configuration, installs it as the active
// configuration, and applies its logging settings. A failed reload leaves
// the previous configuration in place.
func Init(configPath string) error {
c, err := loadConfig(configPath)
if err != nil {
return err
}
install(c)
return nil
}
// InitServer is Init with the mirror server's additional requirements
// checked before the configuration is installed, so a bad reload leaves
// the running server on the configuration it is already serving.
func InitServer(configPath string) error {
c, err := loadConfig(configPath)
if err != nil {
return err
}
if err := c.RequireServer(); err != nil {
return err
}
install(c)
return nil
}
// install publishes a configuration and applies the settings derived from
// it.
func install(c *Config) {
C = c
SetupLogging(c.Log)
}
// findConfig resolves the configuration file path: the explicit flag, the
// working directory, the user config directory, then the system directory.
func findConfig(explicit string) string {
if explicit != "" {
return explicit
}
candidates := []string{"config.yaml"}
if home, err := os.UserHomeDir(); err == nil {
candidates = append(candidates, filepath.Join(home, ".config", "repo-sync", "config.yaml"))
}
candidates = append(candidates, "/etc/repo-sync/config.yaml")
for _, name := range candidates {
if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() {
return name
}
}
return ""
}
// setDefaults registers every configuration default.
func setDefaults(v *viper.Viper) {
v.SetDefault("state_flush_interval", 30*time.Second)
v.SetDefault("repo_crawl_interval", 4*time.Hour)
v.SetDefault("failure_retry_interval", time.Hour)
v.SetDefault("crawl_check_interval", 30*time.Minute)
v.SetDefault("generic_max_age", 6*time.Hour)
v.SetDefault("negative_cache_ttl", time.Hour)
v.SetDefault("transient_error_ttl", time.Minute)
v.SetDefault("http.port", 8080)
v.SetDefault("http.directory_indexes", true)
v.SetDefault("http.read_header_timeout", 10*time.Second)
v.SetDefault("http.read_timeout", 5*time.Minute)
v.SetDefault("http.write_timeout", 30*time.Minute)
v.SetDefault("http.idle_timeout", 2*time.Minute)
v.SetDefault("log.level", "info")
v.SetDefault("log.type", "console")
v.SetDefault("log.outputs", []string{"console"})
v.SetDefault("crawler.workers", 4)
v.SetDefault("crawler.concurrent_crawls", 4)
v.SetDefault("crawler.request_timeout", 2*time.Minute)
v.SetDefault("crawler.user_agent", Name+"/"+Version)
v.SetDefault("crawler.prune_grace", 0)
v.SetDefault("crawler.missing_mode", "retry")
v.SetDefault("crawler.missing_retries", 3)
v.SetDefault("crawler.refresh_schedule", []time.Duration{
12 * time.Hour, 24 * time.Hour, 48 * time.Hour,
72 * time.Hour, 96 * time.Hour, 120 * time.Hour,
})
v.SetDefault("trace.enabled", false)
}
// finalize canonicalizes, validates, and indexes a parsed configuration.
func (c *Config) finalize() error {
c.canonicalize()
if err := validateConfig(c); err != nil {
return err
}
if err := c.indexDomains(); err != nil {
return err
}
return c.indexMounts()
}
// canonicalize normalizes user-supplied values before validation.
func (c *Config) canonicalize() {
c.StatePath = expandHome(strings.TrimSpace(c.StatePath))
for i := range c.Domains {
c.Domains[i].Domain = strings.ToLower(strings.TrimSpace(c.Domains[i].Domain))
c.Domains[i].Role = strings.ToLower(strings.TrimSpace(c.Domains[i].Role))
c.Domains[i].Root = expandHome(strings.TrimSpace(c.Domains[i].Root))
}
// Trace values are written verbatim into a published file, so trim them
// and drop anything that would break the one-field-per-line format.
c.Trace.Host = TraceValue(c.Trace.Host)
c.Trace.Maintainer = TraceValue(c.Trace.Maintainer)
c.Trace.Sponsor = TraceValue(c.Trace.Sponsor)
c.Trace.Country = TraceValue(c.Trace.Country)
c.Trace.Location = TraceValue(c.Trace.Location)
c.Trace.Throughput = TraceValue(c.Trace.Throughput)
for i := range c.Mounts {
c.Mounts[i].Path = cleanRequestPath(c.Mounts[i].Path)
c.Mounts[i].Upstream = strings.TrimRight(strings.TrimSpace(c.Mounts[i].Upstream), "/")
for j := range c.Mounts[i].Repos {
c.Mounts[i].Repos[j].Path = cleanRequestPath(c.Mounts[i].Repos[j].Path)
c.Mounts[i].Repos[j].Type = strings.ToLower(strings.TrimSpace(c.Mounts[i].Repos[j].Type))
}
}
}
// cleanRequestPath normalizes a request path to a single leading slash with
// no trailing slash.
func cleanRequestPath(p string) string {
return path.Clean("/" + strings.Trim(strings.TrimSpace(p), "/"))
}
// TraceValue normalizes a trace field for publication. Each field occupies
// one line of the trace file, so embedded newlines are folded to spaces
// rather than trimmed away, which would let a configured value forge
// additional trace fields.
func TraceValue(s string) string {
s = strings.Map(func(r rune) rune {
if r == '\n' || r == '\r' || r == '\t' {
return ' '
}
// Drop the remaining control characters outright.
if r < 0x20 || r == 0x7f {
return -1
}
return r
}, s)
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
}
// expandHome resolves a leading ~ against the user's home directory.
func expandHome(p string) string {
if p == "~" || strings.HasPrefix(p, "~/") {
if home, err := os.UserHomeDir(); err == nil {
return filepath.Join(home, strings.TrimPrefix(p, "~"))
}
}
return p
}
// validateConfig applies the declarative validation rules, reporting
// failures by their YAML key names.
func validateConfig(c *Config) error {
validate := validator.New()
validate.RegisterTagNameFunc(func(field reflect.StructField) string {
name := strings.SplitN(field.Tag.Get("mapstructure"), ",", 2)[0]
if name == "-" {
return ""
}
return name
})
err := validate.Struct(c)
if err == nil {
return nil
}
var verrs validator.ValidationErrors
if !errors.As(err, &verrs) {
return err
}
var fields []string
for _, verr := range verrs {
fields = append(fields, fmt.Sprintf("%s (%s)", strings.ToLower(verr.Namespace()), verr.Tag()))
}
return fmt.Errorf("invalid configuration: %s", strings.Join(fields, ", "))
}
// indexDomains builds the host lookup and enforces exactly one online
// domain.
func (c *Config) indexDomains() error {
c.domainsByName = map[string]int{}
c.onlineIndex = -1
for i := range c.Domains {
d := c.Domains[i]
if _, ok := c.domainsByName[d.Domain]; ok {
return fmt.Errorf("domain %s is configured more than once", d.Domain)
}
c.domainsByName[d.Domain] = i
if d.Role == RoleOnline {
if c.onlineIndex != -1 {
return fmt.Errorf("only one online domain may be configured")
}
c.onlineIndex = i
}
}
return nil
}
// RequireServer reports whether the configuration carries what the mirror
// server needs. Domains and mounts are only meaningful to the server, so
// they are checked here rather than at load; the sync commands run against
// a configuration that has neither.
func (c *Config) RequireServer() error {
if len(c.Domains) == 0 {
return errors.New("at least one domain must be configured")
}
if c.onlineIndex == -1 {
return errors.New("one online domain must be configured")
}
if len(c.Mounts) == 0 {
return errors.New("at least one mount must be configured")
}
return nil
}
// indexMounts validates mount uniqueness and upstream URLs, orders mounts
// longest path first for prefix lookup, and checks configured repositories
// sit under their mount.
func (c *Config) indexMounts() error {
seen := map[string]bool{}
for i := range c.Mounts {
m := &c.Mounts[i]
if seen[m.Path] {
return fmt.Errorf("mount path %s is configured more than once", m.Path)
}
seen[m.Path] = true
u, err := url.Parse(m.Upstream)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("mount %s upstream must include scheme and host", m.Path)
}
// Request URLs are built by joining a relative path onto the
// upstream; a query or fragment would swallow that path, so reject
// them rather than compose a malformed URL later.
if u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("mount %s upstream must not contain a query or fragment", m.Path)
}
for _, repo := range m.Repos {
if !pathUnder(repo.Path, m.Path) {
return fmt.Errorf("repository %s is not under mount %s", repo.Path, m.Path)
}
}
}
c.mountOrder = make([]int, len(c.Mounts))
for i := range c.mountOrder {
c.mountOrder[i] = i
}
sort.Slice(c.mountOrder, func(a, b int) bool {
return len(c.Mounts[c.mountOrder[a]].Path) > len(c.Mounts[c.mountOrder[b]].Path)
})
return nil
}
// pathUnder reports whether a request path equals or sits below a base
// path.
func pathUnder(p, base string) bool {
if base == "/" {
return true
}
return p == base || strings.HasPrefix(p, base+"/")
}
// Domain looks up the configuration for a request Host header. The port is
// stripped and IPv6 literals are unbracketed before the lookup.
func (c *Config) Domain(host string) (DomainConfig, bool) {
host = strings.ToLower(strings.TrimSpace(host))
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
host = strings.Trim(host, "[]")
i, ok := c.domainsByName[host]
if !ok {
return DomainConfig{}, false
}
return c.Domains[i], true
}
// OnlineDomain returns the single writable cache domain.
func (c *Config) OnlineDomain() DomainConfig {
return c.Domains[c.onlineIndex]
}
// MountFor finds the longest mount matching a request path.
func (c *Config) MountFor(reqPath string) (MountConfig, bool) {
for _, i := range c.mountOrder {
if pathUnder(reqPath, c.Mounts[i].Path) {
return c.Mounts[i], true
}
}
return MountConfig{}, false
}

142
config/config_test.go Normal file
View file

@ -0,0 +1,142 @@
package config
import (
"os"
"path/filepath"
"testing"
)
// writeConfig writes a temp config file and returns its path.
func writeConfig(t *testing.T, content string) string {
t.Helper()
name := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(name, []byte(content), 0644); err != nil {
t.Fatal(err)
}
return name
}
// TestInitExample verifies the shipped example configuration is valid.
func TestInitExample(t *testing.T) {
if err := Init("../config.example.yaml"); err != nil {
t.Fatal(err)
}
if _, ok := C.Domain("mirror.example.com:443"); !ok {
t.Error("online domain not resolvable with a port")
}
if C.OnlineDomain().Domain != "mirror.example.com" {
t.Errorf("online domain = %q", C.OnlineDomain().Domain)
}
mount, ok := C.MountFor("/almalinux/9/BaseOS/x86_64/os")
if !ok || mount.Path != "/almalinux" {
t.Errorf("mount = %+v, %v", mount, ok)
}
}
// TestStatePathDefault verifies an unset state path lands next to the
// config file.
func TestStatePathDefault(t *testing.T) {
path := writeConfig(t, `
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://example.com"}
`)
if err := Init(path); err != nil {
t.Fatal(err)
}
want := filepath.Join(filepath.Dir(path), "state.yaml")
if C.StatePath != want {
t.Errorf("StatePath = %q, want %q", C.StatePath, want)
}
}
// TestMountLongestPrefix verifies the most specific mount wins.
func TestMountLongestPrefix(t *testing.T) {
path := writeConfig(t, `
state_path: /tmp/state.yaml
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://fallback.example.com"}
- {path: /centos, upstream: "https://centos.example.com"}
- {path: /centos/vault, upstream: "https://vault.example.com"}
`)
if err := Init(path); err != nil {
t.Fatal(err)
}
cases := map[string]string{
"/centos/vault/7/os": "https://vault.example.com",
"/centos/8/BaseOS": "https://centos.example.com",
"/debian/dists/testy": "https://fallback.example.com",
}
for reqPath, want := range cases {
mount, ok := C.MountFor(reqPath)
if !ok || mount.Upstream != want {
t.Errorf("MountFor(%q) = %q, want %q", reqPath, mount.Upstream, want)
}
}
}
// TestInitServerRejections verifies invalid server configurations fail
// with clear errors.
func TestInitServerRejections(t *testing.T) {
cases := map[string]string{
"two online domains": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
- {domain: b.test, role: online, root: /tmp/b}
mounts:
- {path: /, upstream: "https://example.com"}
`,
"no online domain": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: offline, root: /tmp/a}
mounts:
- {path: /, upstream: "https://example.com"}
`,
"repo outside mount": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
mounts:
- path: /centos
upstream: "https://example.com"
repos:
- {path: /debian/dists/x, type: deb}
`,
"bad repo type": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
mounts:
- path: /centos
upstream: "https://example.com"
repos:
- {path: /centos/7/os, type: gentoo}
`,
}
for name, content := range cases {
if err := InitServer(writeConfig(t, content)); err == nil {
t.Errorf("%s: configuration unexpectedly accepted", name)
}
}
}
// TestInitWithoutServerSections verifies a configuration carrying no
// domains or mounts loads for the sync commands but is refused for the
// server.
func TestInitWithoutServerSections(t *testing.T) {
path := writeConfig(t, "state_path: /tmp/state.yaml\ncrawler:\n workers: 9\n")
if err := Init(path); err != nil {
t.Fatalf("Init rejected a sync-only configuration: %v", err)
}
if C.Crawler.Workers != 9 {
t.Errorf("Crawler.Workers = %d, want 9", C.Crawler.Workers)
}
if err := InitServer(path); err == nil {
t.Error("InitServer accepted a configuration with no domains")
}
}

19
config/info.go Normal file
View file

@ -0,0 +1,19 @@
// Package config loads and validates the mirror server configuration,
// publishes it for reload-aware readers, configures logging, and carries
// the application identity and build identifiers.
package config
// Build identifiers populated at build time via -ldflags.
var (
Version = "dev"
Commit = ""
Date = ""
Mode = "dev"
)
// Application identifiers used by the CLI and the upstream user agent.
const (
Name = "repo-sync"
DisplayName = "Repository Sync"
Description = "Synchronize Linux package repositories into a local directory tree."
)

56
config/log.go Normal file
View file

@ -0,0 +1,56 @@
package config
import (
"io"
"os"
log "github.com/sirupsen/logrus"
)
// logFiles holds the files opened by the previous SetupLogging call so a
// reload can close them once the new outputs are installed.
var logFiles []*os.File
// SetupLogging applies the configured log level, formatter, and outputs to
// the global logger. Unknown output paths are treated as files opened for
// appending; failures fall back to stderr so the server always logs.
func SetupLogging(lc LogConfig) {
level, err := log.ParseLevel(lc.Level)
if err != nil {
level = log.InfoLevel
}
log.SetLevel(level)
if lc.Type == "json" {
log.SetFormatter(&log.JSONFormatter{})
} else {
log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
}
var writers []io.Writer
var files []*os.File
for _, output := range lc.Outputs {
if output == "console" {
writers = append(writers, os.Stderr)
continue
}
f, err := os.OpenFile(expandHome(output), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.WithError(err).WithField("output", output).Warn("Unable to open log output; skipping.")
continue
}
writers = append(writers, f)
files = append(files, f)
}
if len(writers) == 0 {
writers = append(writers, os.Stderr)
}
log.SetOutput(io.MultiWriter(writers...))
// Close the previous reload's files only after the new writer is
// installed; the logger serializes writes, so none remain in flight.
for _, f := range logFiles {
f.Close()
}
logFiles = files
}

83
config/trace_test.go Normal file
View file

@ -0,0 +1,83 @@
package config
import (
"os"
"strings"
"testing"
)
// TestTraceValue verifies trace fields are reduced to a single line, so a
// configured value cannot forge additional trace fields.
func TestTraceValue(t *testing.T) {
cases := []struct{ in, want string }{
{" Example Inc ", "Example Inc"},
{"a\nb", "a b"},
{"a\r\nb", "a b"},
{"a\tb", "a b"},
{"name <mail@example.com>", "name <mail@example.com>"},
{"Maintainer\nCountry: XX", "Maintainer Country: XX"},
{"has\x00null", "hasnull"},
{" ", ""},
}
for _, c := range cases {
if got := TraceValue(c.in); got != c.want {
t.Errorf("TraceValue(%q) = %q, want %q", c.in, got, c.want)
}
}
// Whatever the input, the result never spans lines.
for _, in := range []string{"a\nb\nc", "x\r\ny", "p\tq"} {
if strings.ContainsAny(TraceValue(in), "\r\n") {
t.Errorf("TraceValue(%q) still contains a line break", in)
}
}
}
// TestTraceHostName verifies the host falls back to the system hostname
// when none is configured.
func TestTraceHostName(t *testing.T) {
if got := (TraceConfig{Host: "mirror.example.com"}).HostName(); got != "mirror.example.com" {
t.Errorf("configured host = %q", got)
}
want, err := os.Hostname()
if err != nil || want == "" {
t.Skip("system hostname unavailable")
}
if got := (TraceConfig{}).HostName(); got != want {
t.Errorf("fallback host = %q, want %q", got, want)
}
}
// TestTraceConfigLoads verifies the trace section is read and normalized,
// and that tracing stays off unless it is asked for.
func TestTraceConfigLoads(t *testing.T) {
path := writeConfig(t, `
trace:
enabled: true
host: mirror.example.com
maintainer: "Test Maintainer <test@example.com>"
sponsor: "Example Inc"
country: US
`)
if err := Init(path); err != nil {
t.Fatal(err)
}
if !C.Trace.Enabled {
t.Error("trace not enabled")
}
if C.Trace.Host != "mirror.example.com" {
t.Errorf("host = %q", C.Trace.Host)
}
if C.Trace.Maintainer != "Test Maintainer <test@example.com>" {
t.Errorf("maintainer = %q, want internal whitespace collapsed", C.Trace.Maintainer)
}
if C.Trace.Location != "" || C.Trace.Throughput != "" {
t.Error("unset trace fields should stay empty")
}
if err := Init(writeConfig(t, "log:\n level: info\n")); err != nil {
t.Fatal(err)
}
if C.Trace.Enabled {
t.Error("tracing must default to off")
}
}

793
fetch/fetch.go Normal file
View file

@ -0,0 +1,793 @@
// Package fetch downloads repository files over HTTP with mirror failover,
// verifying published sizes and checksums, staging updates so a live tree
// is never served a partially updated repository, and pruning files the
// upstream no longer lists.
package fetch
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"hash/fnv"
"io"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/klauspost/compress/zstd"
log "github.com/sirupsen/logrus"
"github.com/ulikunitz/xz"
)
// client is the shared HTTP client used for all repository fetches. It is
// atomic because a SIGHUP reload swaps it while requests are in flight.
var client atomic.Pointer[http.Client]
func init() {
// Build a client immediately so the package is usable without an
// explicit Reload. The config package is initialized first, so its
// defaults are already installed.
Reload()
}
// Reload rebuilds the shared HTTP client from the active crawler
// configuration. A header timeout is used instead of a whole-request
// timeout so large package downloads are not cut short.
func Reload() {
old := client.Load()
client.Store(&http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
ResponseHeaderTimeout: cfg.C.Crawler.RequestTimeout,
},
})
// Release the replaced transport's idle pool; in-flight requests on it
// finish undisturbed.
if old != nil {
old.CloseIdleConnections()
}
}
// ErrNotFound marks an upstream file that does not exist, letting callers
// treat missing optional files differently from transport failures.
var ErrNotFound = errors.New("upstream file not found")
// ErrNotModified marks a conditional request the upstream answered with
// 304, meaning the local copy is already current.
var ErrNotModified = errors.New("upstream file not modified")
// StagedSuffix is appended to staged downloads until they are promoted.
const StagedSuffix = ".staged"
// PartialSuffix is appended to in-progress downloads. Partials of files
// with published checksums survive failures so the next run can resume.
const PartialSuffix = ".partial"
// LockFileName is the flock target guarding a destination directory.
const LockFileName = ".repo-sync.lock"
// PruneStateName is the prune grace bookkeeping file kept at the root of a
// pruned tree.
const PruneStateName = ".repo-sync-prune.json"
// Expect describes the size and checksums a fetched file must match. A size
// of -1 means the size is unknown, and Sums maps lowercase algorithm names
// to hex digests.
type Expect struct {
Size int64
Sums map[string]string
}
// MakeExpect builds an Expect from a size and a single checksum, treating
// non-positive sizes and empty checksums as unknown.
func MakeExpect(size int64, algo, digest string) *Expect {
e := &Expect{Size: size, Sums: map[string]string{}}
if size <= 0 {
e.Size = -1
}
if algo != "" && digest != "" {
e.Sums[strings.ToLower(algo)] = strings.ToLower(digest)
}
return e
}
// hashOrder lists checksum algorithms from most to least preferred.
var hashOrder = []string{"sha512", "sha384", "sha256", "sha224", "sha1", "sha", "md5", "md5sum"}
// newHasher returns a hash implementation for a checksum algorithm name as
// found in repository metadata, or nil when the algorithm is unknown.
func newHasher(algo string) hash.Hash {
switch strings.ToLower(algo) {
case "md5", "md5sum":
return md5.New()
case "sha", "sha1":
return sha1.New()
case "sha224":
return sha256.New224()
case "sha256":
return sha256.New()
case "sha384":
return sha512.New384()
case "sha512":
return sha512.New()
}
return nil
}
// bestSum picks the strongest checksum available in the expectation.
func (e *Expect) bestSum() (string, string) {
for _, algo := range hashOrder {
if digest, ok := e.Sums[algo]; ok && digest != "" {
return algo, digest
}
}
return "", ""
}
// FileState reports what File did with a file. It is only meaningful when
// the accompanying error is nil.
type FileState int
const (
// FileMissing marks an optional file the upstream does not serve.
FileMissing FileState = iota
// FileUnchanged marks an existing local file that matched expectations.
FileUnchanged
// FileFetched marks a file downloaded into its final location.
FileFetched
// FileStaged marks a file downloaded next to its final location for a
// later promote.
FileStaged
)
// verifyLocal reports whether an existing regular file already matches the
// expected size and, when checkHash is set, the expected checksum.
func verifyLocal(name string, want *Expect, checkHash bool) bool {
info, err := os.Stat(name)
if err != nil || !info.Mode().IsRegular() {
return false
}
if want.Size >= 0 && info.Size() != want.Size {
return false
}
if !checkHash {
return true
}
algo, digest := want.bestSum()
if digest == "" {
return true
}
sum, err := hashFile(name, algo)
if err != nil {
return false
}
return sum == digest
}
// hashFile computes the named checksum of a file, returned as lowercase hex.
func hashFile(name, algo string) (string, error) {
h := newHasher(algo)
if h == nil {
return "", fmt.Errorf("unsupported checksum algorithm %q", algo)
}
f, err := os.Open(name)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// fileStripes bounds the destination lock table. Downloads to the same
// destination are serialized by stripe; a collision between unrelated
// destinations only delays a download briefly.
const fileStripes = 512
// fileLocks serializes downloads by destination path. The mirror server's
// on-demand fetches and background crawls can target the same file
// concurrently, and their partial downloads must not interleave.
var fileLocks [fileStripes]sync.Mutex
// lockFile holds the lock for a destination path and returns its unlock
// function.
func lockFile(dst string) func() {
h := fnv.New32a()
_, _ = io.WriteString(h, dst)
m := &fileLocks[h.Sum32()%fileStripes]
m.Lock()
return m.Unlock
}
// File downloads reqPath from the source into dst, verifying size and
// checksum when known. Existing files that already match are left alone.
// With stage set the download is written next to dst for a later promote so
// readers of a live tree never see a partially updated repository.
func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting bool) (FileState, error) {
// Serialize writers to this destination so concurrent callers, such
// as the mirror server's on-demand fetches and background crawls,
// cannot interleave partial downloads of the same file.
unlock := lockFile(dst)
defer unlock()
// Clear any staged leftover from an interrupted run so a later promote
// can never publish stale content.
if stage {
_ = os.Remove(dst + StagedSuffix)
}
// Skip files that already match the expected content.
if want != nil && verifyLocal(dst, want, verifyExisting) {
recordUnchanged(ctx)
return FileUnchanged, nil
}
// Checksummed files with a known size can resume a failed download's
// partial file with a ranged request.
var resumeFrom int64
if want != nil && want.Size > 0 {
if _, digest := want.bestSum(); digest != "" {
if info, err := os.Stat(dst + PartialSuffix); err == nil && info.Size() > 0 && info.Size() < want.Size {
resumeFrom = info.Size()
}
}
}
for {
state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom)
if err != nil && resumeFrom > 0 && !errors.Is(err, ErrNotFound) && ctx.Err() == nil {
// The partial may not be a prefix of the current upstream file;
// retry once from scratch.
log.WithError(err).WithField("path", reqPath).Debug("Resumed download failed; retrying in full.")
_ = os.Remove(dst + PartialSuffix)
resumeFrom = 0
continue
}
return state, err
}
}
// download performs one transfer attempt for File, appending to the
// partial file when resuming from a prior failure.
func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64) (FileState, error) {
// Files without published checksums cannot be verified locally, so an
// existing copy is revalidated with a conditional request instead of
// being re-downloaded every run.
var modifiedSince time.Time
if want == nil {
if info, err := os.Stat(dst); err == nil && info.Mode().IsRegular() {
modifiedSince = info.ModTime()
}
}
// Request the file, failing over between mirrors as needed.
resp, err := src.Get(ctx, reqPath, GetOptions{ModifiedSince: modifiedSince, RangeFrom: resumeFrom})
if errors.Is(err, ErrNotModified) {
recordUnchanged(ctx)
log.WithField("path", reqPath).Debug("Kept unchanged repository file.")
return FileUnchanged, nil
}
if err != nil {
return FileMissing, err
}
defer resp.Body.Close()
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return FileMissing, err
}
var hasher hash.Hash
var algo, digest string
if want != nil {
algo, digest = want.bestSum()
if digest != "" {
hasher = newHasher(algo)
}
}
// Partials are only kept for files whose content the next attempt can
// verify by checksum.
keepPartial := digest != "" && want.Size > 0
// A 206 continues the partial file, whose existing bytes must feed the
// hash first; any other success replaces it from the start.
partial := dst + PartialSuffix
resumed := resumeFrom > 0 && resp.StatusCode == http.StatusPartialContent
openFlags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
if resumed {
openFlags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
if hasher != nil {
pf, err := os.Open(partial)
if err != nil {
return FileMissing, err
}
_, err = io.Copy(hasher, pf)
pf.Close()
if err != nil {
return FileMissing, err
}
}
log.WithFields(log.Fields{"path": reqPath, "offset": resumeFrom}).Debug("Resuming partial download.")
}
// Stream the body into the partial file so a crash never leaves an
// incomplete file at the final location.
f, err := os.OpenFile(partial, openFlags, 0644)
if err != nil {
return FileMissing, err
}
w := io.Writer(f)
if hasher != nil {
w = io.MultiWriter(f, hasher)
}
written, copyErr := io.Copy(w, resp.Body)
closeErr := f.Close()
if copyErr != nil {
if !keepPartial {
_ = os.Remove(partial)
}
return FileMissing, fmt.Errorf("download %s: %w", reqPath, copyErr)
}
if closeErr != nil {
if !keepPartial {
_ = os.Remove(partial)
}
return FileMissing, closeErr
}
// Verify the transfer against the published size and checksum. A
// mismatch invalidates the partial file entirely.
total := written
if resumed {
total += resumeFrom
}
if want != nil && want.Size >= 0 && total != want.Size {
_ = os.Remove(partial)
return FileMissing, fmt.Errorf("download %s: size %d does not match expected %d", reqPath, total, want.Size)
}
if hasher != nil {
if sum := hex.EncodeToString(hasher.Sum(nil)); sum != digest {
_ = os.Remove(partial)
return FileMissing, fmt.Errorf("download %s: %s checksum %s does not match expected %s", reqPath, algo, sum, digest)
}
}
// Move the download into place, or next to it when staging.
final := dst
state := FileFetched
if stage {
final = dst + StagedSuffix
state = FileStaged
}
if err := os.Rename(partial, final); err != nil {
_ = os.Remove(partial)
return FileMissing, err
}
// Carry the upstream modification time so later runs can revalidate
// with conditional requests. Promotion preserves it.
if lm, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
_ = os.Chtimes(final, time.Now(), lm)
}
recordFetched(ctx, written)
log.WithField("path", reqPath).Debug("Fetched repository file.")
return state, nil
}
// PlanJobs records what a real run would download without transferring
// anything, still marking every destination as part of the repository so a
// dry-run prune report stays accurate. Optional files the upstream may not
// serve are counted, so the result is an upper bound.
func PlanJobs(jobs []Job, verifyExisting bool, keep *KeepSet) {
for _, job := range jobs {
keep.Add(job.Dst)
if job.Want != nil && verifyLocal(job.Dst, job.Want, verifyExisting) {
Stats.Unchanged.Add(1)
continue
}
Stats.Planned.Add(1)
if job.Want != nil && job.Want.Size > 0 {
Stats.PlannedBytes.Add(job.Want.Size)
}
log.WithField("path", job.ReqPath).Debug("Would fetch repository file.")
}
}
// RemoveStale deletes a local file the upstream no longer serves, so an
// outdated copy is never served beside refreshed metadata. Missing files
// are not an error.
func RemoveStale(name string) {
if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
log.WithError(err).WithField("path", name).Warn("Failed to remove stale file.")
}
}
// StagedOrFinal returns the staged copy of dst when one exists, falling
// back to the promoted file for runs where the upstream was unchanged.
func StagedOrFinal(dst string) string {
staged := dst + StagedSuffix
if _, err := os.Stat(staged); err == nil {
return staged
}
return dst
}
// PromoteStaged moves a staged download into its final location. Missing
// staged files are not an error, as unchanged files are never staged.
func PromoteStaged(dst string) error {
staged := dst + StagedSuffix
if _, err := os.Stat(staged); errors.Is(err, fs.ErrNotExist) {
return nil
}
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
return os.Rename(staged, dst)
}
// PromoteOrDiscard finishes a staged file: promoted normally, or removed in
// a dry run so no published repository state changes.
func PromoteOrDiscard(dst string, dryRun bool) error {
if dryRun {
_ = os.Remove(dst + StagedSuffix)
return nil
}
return PromoteStaged(dst)
}
// Job describes one file for Many to download.
type Job struct {
ReqPath string
Dst string
Want *Expect
Optional bool
Stage bool
}
// Many downloads jobs with a fixed worker pool, recording each job's
// resulting state. The first hard failure aborts the remaining work. Files
// the upstream does not serve abort it too, unless the job is optional or
// miss tolerates the absence.
func Many(ctx context.Context, src *Source, jobs []Job, workers int, verifyExisting bool, keep *KeepSet, miss *Missing) ([]FileState, error) {
states := make([]FileState, len(jobs))
indexes := make(chan int)
errs := make(chan error, 1)
var wg sync.WaitGroup
if workers < 1 {
workers = 1
}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for idx := range indexes {
job := jobs[idx]
state, err := File(ctx, src, job.ReqPath, job.Dst, job.Want, job.Stage, verifyExisting)
if errors.Is(err, ErrNotFound) {
if job.Optional {
log.WithField("path", job.ReqPath).Debug("Skipped file the upstream does not serve.")
continue
}
// A file the metadata lists is expected to exist, so its
// absence is counted and reported however it is handled.
recordMissing(ctx)
if miss.Tolerate(job.ReqPath) {
continue
}
}
if err != nil {
select {
case errs <- err:
default:
}
return
}
states[idx] = state
keep.Add(job.Dst)
}
}()
}
for idx := range jobs {
select {
case err := <-errs:
close(indexes)
wg.Wait()
return states, err
case <-ctx.Done():
close(indexes)
wg.Wait()
return states, ctx.Err()
case indexes <- idx:
}
}
close(indexes)
wg.Wait()
select {
case err := <-errs:
return states, err
default:
return states, nil
}
}
// KeepSet tracks the files that belong to the current sync so pruning can
// remove everything else. A nil KeepSet ignores all operations.
type KeepSet struct {
mu sync.Mutex
paths map[string]bool
}
// NewKeepSet returns a tracking set when pruning is enabled, or nil so all
// tracking becomes a no-op.
func NewKeepSet(enabled bool) *KeepSet {
if !enabled {
return nil
}
return &KeepSet{paths: map[string]bool{}}
}
// Add records a local file as part of the repository.
func (k *KeepSet) Add(name string) {
if k == nil {
return
}
abs, err := filepath.Abs(name)
if err != nil {
return
}
k.mu.Lock()
k.paths[abs] = true
k.mu.Unlock()
}
// has reports whether a local file is part of the repository.
func (k *KeepSet) has(name string) bool {
if k == nil {
return false
}
abs, err := filepath.Abs(name)
if err != nil {
return false
}
k.mu.Lock()
defer k.mu.Unlock()
return k.paths[abs]
}
// loadPruneState reads the grace bookkeeping for a pruned tree, mapping
// root-relative paths to when they were first seen unreferenced.
func loadPruneState(root string) map[string]time.Time {
state := map[string]time.Time{}
data, err := os.ReadFile(filepath.Join(root, PruneStateName))
if err != nil {
return state
}
if err := json.Unmarshal(data, &state); err != nil {
log.WithError(err).WithField("path", root).Warn("Failed to parse prune state; starting fresh.")
return map[string]time.Time{}
}
return state
}
// savePruneState persists the grace bookkeeping, or removes it when no
// files are waiting out their grace period.
func savePruneState(root string, state map[string]time.Time) {
name := filepath.Join(root, PruneStateName)
if len(state) == 0 {
_ = os.Remove(name)
return
}
data, err := json.Marshal(state)
if err == nil {
err = os.WriteFile(name, data, 0644)
}
if err != nil {
log.WithError(err).WithField("path", name).Warn("Failed to save prune state.")
}
}
// PruneTree removes files under root that are not part of the current sync,
// then clears out any directories left empty. A grace period defers
// deletion until a file has been unreferenced across runs for that long,
// so clients holding earlier metadata keep working. Failures are logged
// and do not fail the sync.
func PruneTree(root string, keep *KeepSet, grace time.Duration, dryRun bool) {
if keep == nil {
return
}
var state map[string]time.Time
if grace > 0 {
state = loadPruneState(root)
}
now := time.Now()
var dirs []string
err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if name != root {
dirs = append(dirs, name)
}
return nil
}
base := filepath.Base(name)
if base == PruneStateName || base == MissingStateName || base == LockFileName {
return nil
}
rel, relErr := filepath.Rel(root, name)
if relErr != nil {
return nil
}
if keep.has(name) {
delete(state, rel)
return nil
}
// Internal work files are removed regardless of grace; anything
// else waits out the grace period across runs first.
internal := strings.HasSuffix(base, StagedSuffix) || strings.HasSuffix(base, PartialSuffix)
if grace > 0 && !internal {
first, ok := state[rel]
if !ok {
// A dry run records nothing, so report the pending grace
// entry instead of staying silent about the file.
if dryRun {
log.WithField("path", name).Info("Stale file would enter the prune grace period.")
return nil
}
state[rel] = now
log.WithField("path", name).Debug("Stale file entered the prune grace period.")
return nil
}
if now.Sub(first) < grace {
return nil
}
}
if dryRun {
Stats.Pruned.Add(1)
log.WithField("path", name).Info("Would prune stale file.")
return nil
}
if err := os.Remove(name); err != nil {
log.WithError(err).WithField("path", name).Warn("Failed to prune stale file.")
return nil
}
delete(state, rel)
Stats.Pruned.Add(1)
log.WithField("path", name).Debug("Pruned stale file.")
return nil
})
if err != nil {
log.WithError(err).WithField("path", root).Warn("Failed to walk repository for pruning.")
}
// Drop bookkeeping for files that no longer exist, then persist it.
if grace > 0 && !dryRun {
for rel := range state {
if _, err := os.Stat(filepath.Join(root, rel)); errors.Is(err, fs.ErrNotExist) {
delete(state, rel)
}
}
savePruneState(root, state)
}
if dryRun {
return
}
// Remove now-empty directories deepest first. Separator count is the
// true depth, so a parent is always removed after its children even
// when a sibling's child has a longer name.
sep := string(os.PathSeparator)
sort.Slice(dirs, func(i, j int) bool {
return strings.Count(dirs[i], sep) > strings.Count(dirs[j], sep)
})
for _, dir := range dirs {
_ = os.Remove(dir)
}
}
// Decompressor wraps a reader with the decompressor matching the file name
// extension, returning an optional close function for the wrapper. Staged
// suffixes are ignored so staged indexes can be parsed before promotion.
func Decompressor(r io.Reader, filename string) (io.Reader, func(), error) {
filename = strings.TrimSuffix(filename, StagedSuffix)
switch {
case strings.HasSuffix(filename, ".gz"):
gz, err := gzip.NewReader(r)
if err != nil {
return nil, nil, err
}
return gz, func() { _ = gz.Close() }, nil
case strings.HasSuffix(filename, ".bz2"):
return bzip2.NewReader(r), nil, nil
case strings.HasSuffix(filename, ".xz"):
xr, err := xz.NewReader(r)
if err != nil {
return nil, nil, err
}
return xr, nil, nil
case strings.HasSuffix(filename, ".zst"):
zr, err := zstd.NewReader(r)
if err != nil {
return nil, nil, err
}
return zr, zr.Close, nil
default:
return r, nil, nil
}
}
// SniffDecompressor wraps a reader with the decompressor matching its
// leading magic bytes, for formats whose file names carry no extension.
func SniffDecompressor(r io.Reader) (io.Reader, func(), error) {
br := bufio.NewReader(r)
magic, err := br.Peek(6)
if err != nil && !errors.Is(err, io.EOF) {
return nil, nil, err
}
switch {
case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b:
gz, err := gzip.NewReader(br)
if err != nil {
return nil, nil, err
}
return gz, func() { _ = gz.Close() }, nil
case len(magic) >= 3 && magic[0] == 'B' && magic[1] == 'Z' && magic[2] == 'h':
return bzip2.NewReader(br), nil, nil
case len(magic) >= 6 && bytes.Equal(magic[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
xr, err := xz.NewReader(br)
if err != nil {
return nil, nil, err
}
return xr, nil, nil
case len(magic) >= 4 && bytes.Equal(magic[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}):
zr, err := zstd.NewReader(br)
if err != nil {
return nil, nil, err
}
return zr, zr.Close, nil
default:
return br, nil, nil
}
}
// LocalJoin resolves a repository-relative path under root, rejecting paths
// that would escape the destination tree.
func LocalJoin(root, rel string) (string, error) {
clean := path.Clean("/" + strings.ReplaceAll(rel, "\\", "/"))
dst := filepath.Join(root, filepath.FromSlash(clean))
rootClean := filepath.Clean(root)
prefix := rootClean + string(os.PathSeparator)
if rootClean == string(os.PathSeparator) {
prefix = rootClean
}
if dst != rootClean && !strings.HasPrefix(dst, prefix) {
return "", fmt.Errorf("path %q escapes destination %q", rel, root)
}
return dst, nil
}

212
fetch/fetch_test.go Normal file
View file

@ -0,0 +1,212 @@
package fetch
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// resumeFixture serves a distinct-content file and records the Range
// headers of incoming requests.
func resumeFixture(t *testing.T, stripRange bool) (*Source, []byte, *[]string) {
t.Helper()
content := make([]byte, 3000)
for i := range content {
content[i] = byte(i % 251)
}
www := t.TempDir()
testrepos.WriteFile(t, filepath.Join(www, "big.bin"), content)
var mu sync.Mutex
ranges := &[]string{}
fs := http.FileServer(http.Dir(www))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
*ranges = append(*ranges, r.Header.Get("Range"))
mu.Unlock()
if stripRange {
r.Header.Del("Range")
}
fs.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
return NewSource([]string{srv.URL}), content, ranges
}
// TestFetchFileResume verifies a valid partial file is completed with a
// ranged request.
func TestFetchFileResume(t *testing.T) {
src, content, ranges := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, content[:1000])
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, content) {
t.Error("resumed content mismatch")
}
if len(*ranges) != 1 || (*ranges)[0] != "bytes=1000-" {
t.Errorf("ranges = %v, want one bytes=1000- request", *ranges)
}
}
// TestFetchFileResumeIgnored verifies a server that ignores ranges still
// produces a correct full download.
func TestFetchFileResumeIgnored(t *testing.T) {
src, content, _ := resumeFixture(t, true)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, content[:1000])
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, _ := os.ReadFile(dst)
if !bytes.Equal(got, content) {
t.Error("content mismatch after ignored range")
}
}
// TestFetchFileResumeBadPartial verifies a partial that is not a prefix of
// the upstream file falls back to a full download.
func TestFetchFileResumeBadPartial(t *testing.T) {
src, content, ranges := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, bytes.Repeat([]byte("X"), 1000))
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, _ := os.ReadFile(dst)
if !bytes.Equal(got, content) {
t.Error("content mismatch after bad partial")
}
if len(*ranges) != 2 || (*ranges)[1] != "" {
t.Errorf("ranges = %v, want a ranged then a full request", *ranges)
}
}
// TestFetchFileConditional verifies files without published checksums are
// revalidated with conditional requests: unchanged upstreams are kept, and
// changed upstreams are fetched again.
func TestFetchFileConditional(t *testing.T) {
www := t.TempDir()
upstream := filepath.Join(www, "meta.tar.gz")
testrepos.WriteFile(t, upstream, []byte("metadata"))
srv := testrepos.ServeDir(t, www)
src := NewSource([]string{srv.URL})
dst := filepath.Join(t.TempDir(), "meta.tar.gz")
state, err := File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileStaged {
t.Fatalf("first fetch = %v, %v", state, err)
}
if err := PromoteStaged(dst); err != nil {
t.Fatal(err)
}
// An unchanged upstream answers the conditional request with 304.
state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileUnchanged {
t.Errorf("unchanged fetch = %v, %v", state, err)
}
// A newer upstream file must be fetched again. The modification time
// is pushed forward because Last-Modified has one-second resolution.
testrepos.WriteFile(t, upstream, []byte("metadata-v2"))
future := time.Now().Add(2 * time.Second)
if err := os.Chtimes(upstream, future, future); err != nil {
t.Fatal(err)
}
state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileStaged {
t.Fatalf("changed fetch = %v, %v", state, err)
}
got, err := os.ReadFile(dst + StagedSuffix)
if err != nil {
t.Fatal(err)
}
if string(got) != "metadata-v2" {
t.Errorf("staged content = %q", got)
}
}
// TestFetchFileConcurrent verifies concurrent downloads to the same
// destination are serialized: no caller's partial download can interleave
// with another's.
func TestFetchFileConcurrent(t *testing.T) {
src, content, _ := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
var wg sync.WaitGroup
errs := make(chan error, 8)
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := File(context.Background(), src, "big.bin", dst, want, false, true); err != nil {
errs <- err
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, content) {
t.Error("content mismatch after concurrent downloads")
}
}
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
Reload()
os.Exit(m.Run())
}
// TestLocalJoin verifies traversal attempts cannot escape the destination.
func TestLocalJoin(t *testing.T) {
root := t.TempDir()
got, err := LocalJoin(root, "../../etc/passwd")
if err != nil {
t.Fatal(err)
}
want := filepath.Join(root, "etc", "passwd")
if got != want {
t.Errorf("got %q, want %q", got, want)
}
if _, err := LocalJoin(root, "a/../b/c.rpm"); err != nil {
t.Errorf("clean relative path rejected: %v", err)
}
got, err = LocalJoin("/", "etc/passwd")
if err != nil {
t.Fatalf("filesystem root destination rejected: %v", err)
}
if got != "/etc/passwd" {
t.Errorf("got %q, want %q", got, "/etc/passwd")
}
}

238
fetch/missing.go Normal file
View file

@ -0,0 +1,238 @@
package fetch
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
log "github.com/sirupsen/logrus"
)
// MissingMode selects what a synchronization does with a file its
// repository metadata lists but the upstream does not serve. Published
// repositories are not always complete: an index can outlive the packages it
// references, and a whole repository is not worth abandoning over one of
// them.
type MissingMode string
const (
// MissingFail stops a repository at the first missing file.
MissingFail MissingMode = "fail"
// MissingRetry keeps synchronizing the rest of the repository and fails
// the run until the same file has been missing across enough consecutive
// runs to be accepted as gone.
MissingRetry MissingMode = "retry"
// MissingIgnore skips missing files without ever failing the run.
MissingIgnore MissingMode = "ignore"
)
// MissingModes lists every supported mode, in the order they escalate from
// tolerant to strict.
var MissingModes = []MissingMode{MissingIgnore, MissingRetry, MissingFail}
// MissingModeNames lists the supported mode names, for user facing messages.
func MissingModeNames() []string {
names := make([]string, 0, len(MissingModes))
for _, mode := range MissingModes {
names = append(names, string(mode))
}
return names
}
// ParseMissingMode converts a configured mode name into a MissingMode. An
// empty name selects the strict mode, which is what a caller that configured
// none expects.
func ParseMissingMode(name string) (MissingMode, error) {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return MissingFail, nil
}
mode := MissingMode(name)
if !slices.Contains(MissingModes, mode) {
return "", fmt.Errorf("unsupported missing file mode %q; expected one of %s",
name, strings.Join(MissingModeNames(), ", "))
}
return mode, nil
}
// MissingStateName is the known-missing bookkeeping file kept in a
// synchronized repository.
const MissingStateName = ".repo-sync-missing.json"
// missingRecord is one file's absence: when it was first and last missed,
// how many consecutive runs have missed it, and whether it has been accepted
// as permanently gone.
type missingRecord struct {
First time.Time `json:"first"`
Last time.Time `json:"last"`
Runs int `json:"runs"`
Known bool `json:"known"`
}
// Missing tracks the files an upstream did not serve during one repository
// synchronization and decides whether those absences fail it. Records live
// in the repository itself so an absence that repeats across runs can be
// told apart from one that has only just appeared, which is the difference
// between a package that was withdrawn and an upstream having a bad day.
//
// A nil Missing is the strict mode: nothing is tracked and every absence
// fails the run.
type Missing struct {
mode MissingMode
retries int
root string
dryRun bool
// mu guards everything below; download workers report concurrently.
mu sync.Mutex
records map[string]*missingRecord
seen map[string]bool
// unresolved counts the files missed this run that are still inside
// their retry budget, and first names one of them for the error.
unresolved int
first string
}
// NewMissing starts tracking the files an upstream does not serve for a
// repository rooted at dir, loading what earlier runs recorded there. The
// strict mode tracks nothing, so it returns nil and every absence keeps
// failing the run.
func NewMissing(dir string, mode MissingMode, retries int, dryRun bool) *Missing {
if mode != MissingRetry && mode != MissingIgnore {
return nil
}
m := &Missing{
mode: mode,
retries: retries,
root: dir,
dryRun: dryRun,
records: map[string]*missingRecord{},
seen: map[string]bool{},
}
m.load()
return m
}
// load reads the records an earlier run left in the repository. A missing or
// unreadable file starts fresh: the records only decide how much longer an
// absence is retried, so losing them costs retries rather than correctness.
func (m *Missing) load() {
data, err := os.ReadFile(filepath.Join(m.root, MissingStateName))
if err != nil {
return
}
records := map[string]*missingRecord{}
if err := json.Unmarshal(data, &records); err != nil {
log.WithError(err).WithField("path", m.root).Warn("Failed to parse the missing file state; starting fresh.")
return
}
// Drop null records a hand-edited file may carry, as every lookup
// dereferences the stored pointer.
for name, rec := range records {
if rec != nil {
m.records[name] = rec
}
}
}
// Tolerate records that the upstream did not serve reqPath and reports
// whether the synchronization may continue without it. A file inside its
// retry budget is still tolerated for this run, so the rest of the
// repository is mirrored either way; Finish is what reports it as a failure.
func (m *Missing) Tolerate(reqPath string) bool {
if m == nil {
return false
}
m.mu.Lock()
defer m.mu.Unlock()
// Count each file once per run: a repository may list the same file from
// several indexes, and the budget counts runs rather than references.
if m.seen[reqPath] {
return true
}
m.seen[reqPath] = true
now := time.Now()
rec := m.records[reqPath]
if rec == nil {
rec = &missingRecord{First: now}
m.records[reqPath] = rec
}
rec.Runs++
rec.Last = now
// A file missing for its whole retry budget is accepted as gone and
// stops failing runs from here on. It is still requested every run, so
// an upstream that publishes it again is picked up without intervention.
if rec.Runs >= m.retries {
rec.Known = true
}
if m.mode == MissingIgnore || rec.Known {
log.WithField("path", reqPath).Debug("Skipped a file the upstream does not serve.")
return true
}
m.unresolved++
if m.first == "" {
m.first = reqPath
}
log.WithFields(log.Fields{"path": reqPath, "runs": rec.Runs, "retries": m.retries}).
Warn("Upstream does not serve a file its metadata lists.")
return true
}
// Finish persists what this run observed and reports the absences that have
// not yet been accepted as permanent, which is what fails the repository
// while a missing file is still being retried.
func (m *Missing) Finish() error {
if m == nil {
return nil
}
m.mu.Lock()
defer m.mu.Unlock()
m.save()
if m.unresolved == 0 {
return nil
}
return fmt.Errorf("%d file(s) listed by the metadata are missing upstream and will be retried, starting with %s",
m.unresolved, m.first)
}
// save rewrites the records for the files missed this run, and removes the
// file once nothing is missing. Only files missed this run are kept:
// anything else is served again, or is no longer listed at all, and either
// way its retry budget starts over.
func (m *Missing) save() {
if m.dryRun {
return
}
name := filepath.Join(m.root, MissingStateName)
if len(m.seen) == 0 {
if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
log.WithError(err).WithField("path", name).Warn("Failed to remove the missing file state.")
}
return
}
kept := make(map[string]*missingRecord, len(m.seen))
for reqPath := range m.seen {
kept[reqPath] = m.records[reqPath]
}
data, err := json.Marshal(kept)
if err == nil {
err = os.MkdirAll(m.root, 0755)
}
if err == nil {
err = os.WriteFile(name, data, 0644)
}
if err != nil {
log.WithError(err).WithField("path", name).Warn("Failed to save the missing file state.")
}
}

149
fetch/missing_test.go Normal file
View file

@ -0,0 +1,149 @@
package fetch
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
// readMissingState reads the records a run left in a repository.
func readMissingState(t *testing.T, root string) map[string]*missingRecord {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, MissingStateName))
if err != nil {
t.Fatal(err)
}
records := map[string]*missingRecord{}
if err := json.Unmarshal(data, &records); err != nil {
t.Fatal(err)
}
return records
}
// TestParseMissingMode verifies mode names are accepted case insensitively
// and that an unconfigured mode is the strict one.
func TestParseMissingMode(t *testing.T) {
cases := map[string]MissingMode{
"": MissingFail,
"fail": MissingFail,
"retry": MissingRetry,
" Retry": MissingRetry,
"ignore": MissingIgnore,
}
for name, want := range cases {
got, err := ParseMissingMode(name)
if err != nil || got != want {
t.Errorf("ParseMissingMode(%q) = %q, %v, want %q", name, got, err, want)
}
}
if _, err := ParseMissingMode("skip"); err == nil {
t.Error("expected an unsupported mode name to be rejected")
}
}
// TestMissingStrictMode verifies the strict mode tracks nothing and
// tolerates nothing, so every absence keeps failing the run.
func TestMissingStrictMode(t *testing.T) {
root := t.TempDir()
m := NewMissing(root, MissingFail, 3, false)
if m != nil {
t.Fatal("the strict mode installed a tracker")
}
if m.Tolerate("pool/gone.deb") {
t.Error("the strict mode tolerated a missing file")
}
if err := m.Finish(); err != nil {
t.Errorf("Finish = %v, want nil", err)
}
if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) {
t.Error("the strict mode wrote a state file")
}
}
// TestMissingRetryBudget verifies an absence has to repeat across
// consecutive runs before it stops failing them, and that a file the
// upstream serves again starts its budget over.
func TestMissingRetryBudget(t *testing.T) {
root := t.TempDir()
const gone = "pool/main/h/hello/hello_1.0_amd64.deb"
// The first two runs mirror the rest of the repository but still report
// the absence as a failure.
for run := 1; run <= 2; run++ {
m := NewMissing(root, MissingRetry, 3, false)
if !m.Tolerate(gone) {
t.Fatalf("run %d aborted on a missing file", run)
}
if err := m.Finish(); err == nil {
t.Errorf("run %d reported success while the file was still being retried", run)
}
if rec := readMissingState(t, root)[gone]; rec == nil || rec.Runs != run || rec.Known {
t.Errorf("run %d recorded %+v", run, rec)
}
}
// The third run exhausts the budget, so the file is accepted as gone.
m := NewMissing(root, MissingRetry, 3, false)
m.Tolerate(gone)
if err := m.Finish(); err != nil {
t.Errorf("run 3 still failed after the retry budget: %v", err)
}
if rec := readMissingState(t, root)[gone]; rec == nil || !rec.Known {
t.Errorf("run 3 recorded %+v, want a known absence", rec)
}
// A later run that misses nothing clears the bookkeeping entirely.
m = NewMissing(root, MissingRetry, 3, false)
if err := m.Finish(); err != nil {
t.Errorf("a run that missed nothing failed: %v", err)
}
if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) {
t.Error("records lingered after the upstream served everything again")
}
}
// TestMissingRepeatedPath verifies a file listed by several indexes counts
// once per run, as the budget counts runs rather than references.
func TestMissingRepeatedPath(t *testing.T) {
root := t.TempDir()
m := NewMissing(root, MissingRetry, 3, false)
m.Tolerate("pool/gone.deb")
m.Tolerate("pool/gone.deb")
if err := m.Finish(); err == nil {
t.Fatal("a single run exhausted the retry budget")
}
if rec := readMissingState(t, root)["pool/gone.deb"]; rec == nil || rec.Runs != 1 {
t.Errorf("recorded %+v, want one run", rec)
}
}
// TestMissingIgnoreMode verifies absences never fail a run in the ignore
// mode, while still being recorded for whoever reads the repository.
func TestMissingIgnoreMode(t *testing.T) {
root := t.TempDir()
m := NewMissing(root, MissingIgnore, 3, false)
if !m.Tolerate("pool/gone.deb") {
t.Fatal("the ignore mode aborted on a missing file")
}
if err := m.Finish(); err != nil {
t.Errorf("Finish = %v, want nil", err)
}
if rec := readMissingState(t, root)["pool/gone.deb"]; rec == nil {
t.Error("the ignore mode recorded nothing")
}
}
// TestMissingDryRun verifies a dry run reports absences without leaving
// bookkeeping behind, so it never spends a run's retry budget.
func TestMissingDryRun(t *testing.T) {
root := t.TempDir()
m := NewMissing(root, MissingRetry, 3, true)
m.Tolerate("pool/gone.deb")
if err := m.Finish(); err == nil {
t.Error("a dry run hid a missing file")
}
if _, err := os.Stat(filepath.Join(root, MissingStateName)); !os.IsNotExist(err) {
t.Error("a dry run wrote a state file")
}
}

260
fetch/source.go Normal file
View file

@ -0,0 +1,260 @@
package fetch
import (
"bufio"
"bytes"
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
log "github.com/sirupsen/logrus"
)
// Source is the set of base URLs a repository can be fetched from. Direct
// repositories have a single base; mirrorlists provide several with
// failover between them.
type Source struct {
// mu guards bases and active.
mu sync.Mutex
bases []string
active int
}
// NewSource builds a Source from base URLs, normalizing trailing slashes.
func NewSource(bases []string) *Source {
s := &Source{}
for _, base := range bases {
s.bases = append(s.bases, strings.TrimRight(base, "/"))
}
return s
}
// Rebase trims a common relative suffix off every base URL so later fetches
// resolve against the parent directory. Bases without the suffix are kept
// unchanged.
func (s *Source) Rebase(suffix string) {
suffix = "/" + strings.Trim(suffix, "/")
s.mu.Lock()
defer s.mu.Unlock()
for i, base := range s.bases {
if strings.HasSuffix(base, suffix) {
s.bases[i] = strings.TrimSuffix(base, suffix)
} else {
log.WithField("mirror", base).Warn("Mirror URL does not share the expected path suffix; using it unchanged.")
}
}
}
// GetOptions adjusts a single Source request. A non-zero ModifiedSince
// adds a conditional header, surfacing a 304 as ErrNotModified. A positive
// RangeFrom requests the remainder of a partially downloaded file.
type GetOptions struct {
ModifiedSince time.Time
RangeFrom int64
}
// Get requests a repository-relative path, failing over to the next mirror
// on transport or server errors. A 404 from the responding mirror is
// treated as authoritative so optional files do not trigger failover.
func (s *Source) Get(ctx context.Context, reqPath string, o GetOptions) (*http.Response, error) {
// Snapshot the bases so a concurrent Rebase cannot race the loop.
s.mu.Lock()
start := s.active
bases := append([]string(nil), s.bases...)
s.mu.Unlock()
if len(bases) == 0 {
return nil, errors.New("source has no mirror URLs")
}
var lastErr error
for i := 0; i < len(bases); i++ {
idx := (start + i) % len(bases)
target := bases[idx] + "/" + strings.TrimPrefix(reqPath, "/")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
if !o.ModifiedSince.IsZero() {
req.Header.Set("If-Modified-Since", o.ModifiedSince.UTC().Format(http.TimeFormat))
}
if o.RangeFrom > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", o.RangeFrom))
}
resp, err := client.Load().Do(req)
if err != nil {
lastErr = fmt.Errorf("fetch %s: %w", target, err)
log.WithError(err).WithField("url", target).Debug("Mirror request failed.")
continue
}
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
resp.Body.Close()
return nil, fmt.Errorf("fetch %s: %w", target, ErrNotFound)
}
if resp.StatusCode == http.StatusNotModified && !o.ModifiedSince.IsZero() {
resp.Body.Close()
return nil, fmt.Errorf("fetch %s: %w", target, ErrNotModified)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
resp.Body.Close()
lastErr = fmt.Errorf("fetch %s: status %d", target, resp.StatusCode)
log.WithField("url", target).WithField("status", resp.StatusCode).Debug("Mirror request failed.")
continue
}
if idx != start {
s.mu.Lock()
s.active = idx
s.mu.Unlock()
}
return resp, nil
}
return nil, lastErr
}
// ReadURL retrieves a raw URL fully into memory, limited to 4 MiB. It is
// used for probing mirrorlists, whose URLs may carry query strings that a
// Source cannot join paths onto.
func ReadURL(ctx context.Context, rawURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
resp, err := client.Load().Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
return nil, fmt.Errorf("fetch %s: %w", rawURL, ErrNotFound)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("fetch %s: status %d", rawURL, resp.StatusCode)
}
// Read one byte past the cap so an oversized document can be detected
// and reported rather than silently truncated into a bad parse.
const maxSize = 4 << 20
data, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1))
if err != nil {
return nil, err
}
if len(data) > maxSize {
log.WithField("url", rawURL).Warn("Response exceeds the read limit and was truncated.")
data = data[:maxSize]
}
return data, nil
}
// ParseMirrorlist extracts mirror base URLs from a mirrorlist body,
// skipping comments and lines that are not absolute HTTP URLs.
func ParseMirrorlist(body []byte) []string {
seen := map[string]bool{}
var bases []string
scanner := bufio.NewScanner(bytes.NewReader(body))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
u, err := url.Parse(line)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
continue
}
base := strings.TrimRight(line, "/")
if seen[base] {
continue
}
seen[base] = true
bases = append(bases, base)
}
return bases
}
// ParseMetalink extracts mirror base URLs from a metalink document's
// repomd.xml entries, keeping the document's preference order.
func ParseMetalink(body []byte) []string {
const repomdSuffix = "/repodata/repomd.xml"
var ml struct {
Files []struct {
Name string `xml:"name,attr"`
URLs []struct {
Value string `xml:",chardata"`
} `xml:"resources>url"`
} `xml:"files>file"`
}
if xml.Unmarshal(body, &ml) != nil {
return nil
}
seen := map[string]bool{}
var bases []string
for _, file := range ml.Files {
if file.Name != "repomd.xml" {
continue
}
for _, u := range file.URLs {
v := strings.TrimSpace(u.Value)
if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") {
continue
}
base := strings.TrimSuffix(v, repomdSuffix)
if base == v {
continue
}
base = strings.TrimRight(base, "/")
if seen[base] {
continue
}
seen[base] = true
bases = append(bases, base)
}
}
return bases
}
// agent returns the user agent sent with every upstream request, falling
// back to the build identity when the configuration clears it.
func agent() string {
if ua := cfg.C.Crawler.UserAgent; ua != "" {
return ua
}
return cfg.Name + "/" + cfg.Version
}
// Get performs a plain GET of a raw URL with the mirror user agent, used
// for directory listings and other non-repository requests.
func Get(ctx context.Context, rawURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
return client.Load().Do(req)
}
// Bases returns a copy of the mirror base URLs, primarily for tests and
// diagnostics.
func (s *Source) Bases() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.bases...)
}
// Active returns the base URL of the mirror currently being fetched from,
// which for a mirrorlist or metalink is the one that last answered rather
// than the list's own URL. It is empty when the source has no mirrors.
func (s *Source) Active() string {
s.mu.Lock()
defer s.mu.Unlock()
if s.active < 0 || s.active >= len(s.bases) {
return ""
}
return s.bases[s.active]
}

97
fetch/source_test.go Normal file
View file

@ -0,0 +1,97 @@
package fetch
import (
"context"
"errors"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseMirrorlist verifies URL extraction, comment skipping, and
// deduplication.
func TestParseMirrorlist(t *testing.T) {
body := "# comment\n\nhttp://mirror1.example.com/repo/\nhttps://mirror2.example.com/repo\nhttp://mirror1.example.com/repo\n/bare/path\nnot a url\n"
bases := ParseMirrorlist([]byte(body))
want := []string{"http://mirror1.example.com/repo", "https://mirror2.example.com/repo"}
if len(bases) != len(want) {
t.Fatalf("got %v, want %v", bases, want)
}
for i := range want {
if bases[i] != want[i] {
t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i])
}
}
}
// TestSourceFailover verifies a dead first mirror fails over to a working
// one and that missing files surface the not-found sentinel.
func TestSourceFailover(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
src := NewSource([]string{"http://127.0.0.1:1/el9", srv.URL + "/el9"})
resp, err := src.Get(context.Background(), "repodata/repomd.xml", GetOptions{})
if err != nil {
t.Fatal("failover did not reach the working mirror:", err)
}
resp.Body.Close()
if src.active != 1 {
t.Errorf("active mirror = %d, want 1", src.active)
}
if _, err := src.Get(context.Background(), "no/such/file", GetOptions{}); !errors.Is(err, ErrNotFound) {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
// TestParseMetalink verifies base URL extraction from metalink documents.
func TestParseMetalink(t *testing.T) {
body := `<?xml version="1.0"?><metalink xmlns="urn:ietf:params:xml:ns:metalink">` +
`<files><file name="repomd.xml"><resources>` +
`<url protocol="https" type="https">https://mirror1.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="rsync">rsync://mirror2.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="http">http://mirror3.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="https">https://mirror1.example.com/fedora/repodata/repomd.xml</url>` +
`</resources></file></files></metalink>`
bases := ParseMetalink([]byte(body))
want := []string{"https://mirror1.example.com/fedora", "http://mirror3.example.com/fedora"}
if len(bases) != len(want) {
t.Fatalf("got %v, want %v", bases, want)
}
for i := range want {
if bases[i] != want[i] {
t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i])
}
}
if ParseMetalink([]byte("not xml at all")) != nil {
t.Error("non-XML input produced bases")
}
}
// TestSourceNoBases verifies a Source with no mirrors reports an error and
// no response.
func TestSourceNoBases(t *testing.T) {
src := NewSource(nil)
resp, err := src.Get(context.Background(), "anything", GetOptions{})
if err == nil {
t.Fatal("expected an error for a source without bases")
}
if resp != nil {
t.Error("response must be nil on error")
}
}
// TestSourceRebase verifies suffix trimming for suite-relative sources.
func TestSourceRebase(t *testing.T) {
src := NewSource([]string{"http://example.com/debian/dists/test", "http://other.example.com/mirror"})
src.Rebase("dists/test")
if src.bases[0] != "http://example.com/debian" {
t.Errorf("bases[0] = %q", src.bases[0])
}
if src.bases[1] != "http://other.example.com/mirror" {
t.Errorf("bases[1] changed unexpectedly: %q", src.bases[1])
}
}

103
fetch/stats.go Normal file
View file

@ -0,0 +1,103 @@
package fetch
import (
"context"
"fmt"
"sync/atomic"
)
// Counters accumulates transfer activity for one repository's summary log
// line. Counters are atomic because download workers run concurrently.
type Counters struct {
Fetched atomic.Int64
FetchedBytes atomic.Int64
Unchanged atomic.Int64
Pruned atomic.Int64
Planned atomic.Int64
PlannedBytes atomic.Int64
Missing atomic.Int64
}
// Stats collects transfer activity for the repository currently
// synchronizing. The CLI synchronizes repositories sequentially, so one
// package-level collector suffices there. The server's concurrent crawls
// interleave into these counters, so their values are only meaningful in
// the sequential path; the server must not report them.
var Stats = &Counters{}
// statsKey carries a caller's own collector on a context.
type statsKey struct{}
// WithStats returns a context whose transfers accumulate into c as well as
// into Stats. It gives one synchronization its own totals even while other
// synchronizations run concurrently, which is what the mirror server's
// crawls and their trace files need.
func WithStats(ctx context.Context, c *Counters) context.Context {
if c == nil {
return ctx
}
return context.WithValue(ctx, statsKey{}, c)
}
// runStats returns the caller's collector carried by ctx, or nil when the
// caller installed none.
func runStats(ctx context.Context) *Counters {
c, _ := ctx.Value(statsKey{}).(*Counters)
return c
}
// recordFetched counts a completed download in the shared collector and in
// the caller's own, when one is installed.
func recordFetched(ctx context.Context, bytes int64) {
Stats.Fetched.Add(1)
Stats.FetchedBytes.Add(bytes)
if c := runStats(ctx); c != nil {
c.Fetched.Add(1)
c.FetchedBytes.Add(bytes)
}
}
// recordUnchanged counts a file that needed no transfer in both collectors.
func recordUnchanged(ctx context.Context) {
Stats.Unchanged.Add(1)
if c := runStats(ctx); c != nil {
c.Unchanged.Add(1)
}
}
// recordMissing counts a file the upstream did not serve in both
// collectors, whether or not the run tolerates its absence.
func recordMissing(ctx context.Context) {
Stats.Missing.Add(1)
if c := runStats(ctx); c != nil {
c.Missing.Add(1)
}
}
// Reset clears the collector for the next repository.
func (s *Counters) Reset() {
s.Fetched.Store(0)
s.FetchedBytes.Store(0)
s.Unchanged.Store(0)
s.Pruned.Store(0)
s.Planned.Store(0)
s.PlannedBytes.Store(0)
s.Missing.Store(0)
}
// FormatBytes renders a byte count in a human readable unit.
func FormatBytes(n int64) string {
const unit = 1024
const prefixes = "KMGTPE"
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
// Cap the exponent at the last available prefix so an unexpectedly
// large count can never index past the prefix table.
for m := n / unit; m >= unit && exp < len(prefixes)-1; m /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), prefixes[exp])
}

92
flags.go Normal file
View file

@ -0,0 +1,92 @@
package main
import (
"fmt"
"runtime/debug"
"strings"
"time"
"github.com/alecthomas/kong"
cfg "github.com/grmrgecko/repo-sync/config"
)
// VersionFlag prints build information and exits.
type VersionFlag bool
// Decode satisfies kong.MapperValue. The flag is treated as a boolean toggle.
func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil }
// IsBool reports the flag as a boolean for kong's parser.
func (v VersionFlag) IsBool() bool { return true }
// BeforeApply emits version information then exits before the rest of the
// command is executed.
func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error {
fmt.Printf("%s: %s (%s)\n", cfg.Name, cfg.Version, cfg.Mode)
if cfg.Commit != "" {
fmt.Printf(" commit: %s\n", cfg.Commit)
}
if cfg.Date != "" {
fmt.Printf(" built: %s\n", cfg.Date)
}
if cfg.Commit == "" {
if bi, ok := debug.ReadBuildInfo(); ok {
for _, s := range bi.Settings {
switch s.Key {
case "vcs.revision":
fmt.Printf(" commit: %s\n", s.Value)
case "vcs.time":
fmt.Printf(" built: %s\n", s.Value)
}
}
}
}
app.Exit(0)
return nil
}
// Flags is the root command line configuration. The global flags override
// the configuration file for every command, so a sync run and the server
// honor the same settings; each is empty when unset so the configured
// value stands.
type Flags struct {
ConfigPath string `help:"The path to the config file." optional:"" type:"existingfile"`
LogLevel string `help:"Override the configured log level." enum:",debug,info,warn,error" default:""`
UserAgent string `help:"Override the configured user agent sent to upstreams." optional:""`
RequestTimeout time.Duration `help:"Override the configured upstream response timeout (e.g. 2m)." optional:""`
Version VersionFlag `name:"version" help:"Print version information and quit"`
// Trace flags override the configuration's trace section. Enabling is a
// pointer so an unset flag leaves the configured value alone while
// --trace and --no-trace each override it.
Trace *bool `help:"Write a trace file into each synchronized repository." negatable:""`
TraceHost string `help:"Override the host name this mirror is traced under; defaults to the system hostname." optional:""`
TraceMaintainer string `help:"Override the maintainer recorded in traces, in \"name <email>\" form." optional:""`
TraceSponsor string `help:"Override the sponsor recorded in traces." optional:""`
TraceCountry string `help:"Override the country recorded in traces." optional:""`
TraceLocation string `help:"Override the location recorded in traces." optional:""`
TraceThroughput string `help:"Override the throughput recorded in traces." optional:""`
Sync SyncCmd `cmd:"" name:"sync" help:"Synchronize package repositories of any supported type."`
Server ServerCmd `cmd:"" name:"server" help:"Run the caching mirror server."`
Service ServiceCmd `cmd:"" name:"service" help:"Manage the mirror server system service."`
}
// flags holds the parsed command line configuration.
var flags Flags
// ParseFlags parses the command line, exiting with usage information on
// error.
func ParseFlags() *kong.Context {
return kong.Parse(&flags,
kong.Name(cfg.Name),
kong.Description(cfg.Description),
kong.UsageOnError(),
kong.ConfigureHelp(kong.HelpOptions{
Compact: true,
}),
kong.Vars{
"serviceActions": strings.Join(ServiceAction, ","),
},
)
}

80
flags_test.go Normal file
View file

@ -0,0 +1,80 @@
package main
import (
"testing"
cfg "github.com/grmrgecko/repo-sync/config"
)
// TestApplyTraceFlags verifies the trace flags override the configuration
// only when given, so a configured value stands on its own.
func TestApplyTraceFlags(t *testing.T) {
configured := cfg.TraceConfig{
Enabled: true,
Host: "configured.example.com",
Maintainer: "Configured <configured@example.com>",
Country: "US",
}
yes, no := true, false
tests := []struct {
name string
flags Flags
want cfg.TraceConfig
}{
{
name: "unset flags leave the configuration alone",
flags: Flags{},
want: configured,
},
{
name: "--no-trace disables tracing",
flags: Flags{Trace: &no},
want: func() cfg.TraceConfig { c := configured; c.Enabled = false; return c }(),
},
{
name: "--trace enables tracing",
flags: Flags{Trace: &yes},
want: func() cfg.TraceConfig { c := configured; c.Enabled = true; return c }(),
},
{
name: "value flags override individually",
flags: Flags{TraceHost: "flag.example.com", TraceSponsor: "Flag Sponsor"},
want: func() cfg.TraceConfig {
c := configured
c.Host = "flag.example.com"
c.Sponsor = "Flag Sponsor"
return c
}(),
},
{
name: "flag values are normalized like configured ones",
flags: Flags{TraceMaintainer: "Flag\nMaintainer"},
want: func() cfg.TraceConfig {
c := configured
c.Maintainer = "Flag Maintainer"
return c
}(),
},
}
// applyTraceFlags reads the package level flags and configuration, so
// each case installs its own and restores them afterwards.
origFlags := flags
origTrace := cfg.C.Trace
t.Cleanup(func() {
flags = origFlags
cfg.C.Trace = origTrace
})
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg.C.Trace = configured
flags = tc.flags
applyTraceFlags()
if cfg.C.Trace != tc.want {
t.Errorf("trace config = %+v, want %+v", cfg.C.Trace, tc.want)
}
})
}
}

36
go.mod Normal file
View file

@ -0,0 +1,36 @@
module github.com/grmrgecko/repo-sync
go 1.26.4
require (
github.com/alecthomas/kong v1.16.0
github.com/coreos/go-systemd/v22 v22.7.0
github.com/go-playground/validator/v10 v10.30.3
github.com/kardianos/service v1.3.0
github.com/klauspost/compress v1.19.1
github.com/sirupsen/logrus v1.9.4
github.com/spf13/viper v1.21.0
github.com/ulikunitz/xz v0.5.16
golang.org/x/net v0.57.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
)

81
go.sum Normal file
View file

@ -0,0 +1,81 @@
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4=
github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8=
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI=
github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

258
internal/testrepos/testrepos.go Normal file
View file

@ -0,0 +1,258 @@
// Package testrepos builds small fixture repositories for tests across
// the fetch, mirror, and server packages.
package testrepos
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// WriteFile creates a file with parent directories under a fixture tree.
func WriteFile(t *testing.T, name string, data []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(name, data, 0644); err != nil {
t.Fatal(err)
}
}
// GzipBytes compresses data with gzip for index fixtures.
func GzipBytes(t *testing.T, data []byte) []byte {
t.Helper()
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
if _, err := gz.Write(data); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// SHA256Hex returns the hex SHA-256 of data.
func SHA256Hex(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// MD5Hex returns the hex MD5 of data.
func MD5Hex(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
// ServeDir serves a fixture tree over HTTP with directory listings, closing
// the server when the test finishes.
func ServeDir(t *testing.T, dir string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.FileServer(http.Dir(dir)))
t.Cleanup(srv.Close)
return srv
}
// BuildRPMRepo writes a minimal yum repository into dir and returns the
// package contents by file name.
func BuildRPMRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"foo-1.0-1.x86_64.rpm": bytes.Repeat([]byte("foo"), 500),
"bar-2.0-1.x86_64.rpm": bytes.Repeat([]byte("bar"), 700),
}
var pkgXML bytes.Buffer
for name, data := range pkgs {
WriteFile(t, filepath.Join(dir, "Packages", name), data)
fmt.Fprintf(&pkgXML,
`<package type="rpm"><name>%s</name><checksum type="sha256" pkgid="YES">%s</checksum><size package="%d"/><location href="Packages/%s"/></package>`,
name, SHA256Hex(data), len(data), name)
}
primary := []byte(`<?xml version="1.0"?><metadata packages="2">` + pkgXML.String() + `</metadata>`)
primaryGz := GzipBytes(t, primary)
primaryName := SHA256Hex(primaryGz) + "-primary.xml.gz"
WriteFile(t, filepath.Join(dir, "repodata", primaryName), primaryGz)
// A prestodelta index references one delta package under drpms/.
drpm := RPMDelta()
WriteFile(t, filepath.Join(dir, "drpms", "foo-0.9_1.0-1.x86_64.drpm"), drpm)
presto := []byte(fmt.Sprintf(
`<?xml version="1.0"?><prestodelta><newpackage name="foo"><delta oldversion="0.9"><filename>drpms/foo-0.9_1.0-1.x86_64.drpm</filename><size>%d</size><checksum type="sha256">%s</checksum></delta></newpackage></prestodelta>`,
len(drpm), SHA256Hex(drpm)))
prestoGz := GzipBytes(t, presto)
prestoName := SHA256Hex(prestoGz) + "-prestodelta.xml.gz"
WriteFile(t, filepath.Join(dir, "repodata", prestoName), prestoGz)
repomd := fmt.Sprintf(
`<?xml version="1.0"?><repomd><data type="primary"><checksum type="sha256">%s</checksum><location href="repodata/%s"/><size>%d</size></data><data type="prestodelta"><checksum type="sha256">%s</checksum><location href="repodata/%s"/><size>%d</size></data></repomd>`,
SHA256Hex(primaryGz), primaryName, len(primaryGz),
SHA256Hex(prestoGz), prestoName, len(prestoGz))
WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml"), []byte(repomd))
WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.asc"), []byte("fake signature"))
return pkgs
}
// RPMDelta is the delta package content used by BuildRPMRepo.
func RPMDelta() []byte {
return bytes.Repeat([]byte("drpm"), 100)
}
// debPackagesStanza renders one binary package stanza for a Packages index.
func debPackagesStanza(name, version, poolPath string, data []byte) string {
return fmt.Sprintf(
"Package: %s\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSize: %d\nMD5sum: %s\nSHA256: %s\nDescription: Test package\n",
name, version, poolPath, len(data), MD5Hex(data), SHA256Hex(data))
}
// BuildDebRepo writes a minimal structured apt repository with main and
// universe components into dir and returns the pool file contents by
// archive-relative path.
func BuildDebRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pool := map[string][]byte{
"pool/main/h/hello/hello_1.0_amd64.deb": bytes.Repeat([]byte("hello"), 300),
"pool/universe/e/extra/extra_2.0_amd64.deb": bytes.Repeat([]byte("extra"), 400),
}
for p, data := range pool {
WriteFile(t, filepath.Join(dir, filepath.FromSlash(p)), data)
}
indexes := map[string][]byte{
"main/binary-amd64/Packages": []byte(debPackagesStanza(
"hello", "1.0", "pool/main/h/hello/hello_1.0_amd64.deb", pool["pool/main/h/hello/hello_1.0_amd64.deb"])),
"universe/binary-amd64/Packages": []byte(debPackagesStanza(
"extra", "2.0", "pool/universe/e/extra/extra_2.0_amd64.deb", pool["pool/universe/e/extra/extra_2.0_amd64.deb"])),
}
// The Release file lists both the uncompressed and gzip variants, but
// only the gzip variant is served, matching real archives.
release := "Origin: Test\nSuite: test\nCodename: test\nArchitectures: amd64\nComponents: main universe\nAcquire-By-Hash: yes\n"
listed := map[string][]byte{}
for p, data := range indexes {
gz := GzipBytes(t, data)
WriteFile(t, filepath.Join(dir, "dists", "test", filepath.FromSlash(p)+".gz"), gz)
listed[p] = data
listed[p+".gz"] = gz
}
for field, hasher := range map[string]func([]byte) string{"MD5Sum": MD5Hex, "SHA256": SHA256Hex} {
release += field + ":\n"
for p, data := range listed {
release += fmt.Sprintf(" %s %d %s\n", hasher(data), len(data), p)
}
}
WriteFile(t, filepath.Join(dir, "dists", "test", "Release"), []byte(release))
return pool
}
// BuildArchRepo writes a minimal pacman repository named name into dir and
// returns the package contents by file name.
func BuildArchRepo(t *testing.T, dir, name string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"zlib-1.3-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("zlib"), 300),
"jq-1.7-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("jq"), 400),
}
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
tw := tar.NewWriter(gz)
for fname, data := range pkgs {
WriteFile(t, filepath.Join(dir, fname), data)
WriteFile(t, filepath.Join(dir, fname+".sig"), []byte("signature"))
desc := fmt.Sprintf("%%FILENAME%%\n%s\n\n%%NAME%%\n%s\n\n%%CSIZE%%\n%d\n\n%%SHA256SUM%%\n%s\n",
fname, strings.SplitN(fname, "-", 2)[0], len(data), SHA256Hex(data))
hdr := &tar.Header{
Name: strings.TrimSuffix(fname, "-x86_64.pkg.tar.zst") + "/desc",
Mode: 0644,
Size: int64(len(desc)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(desc)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
WriteFile(t, filepath.Join(dir, name+".db"), buf.Bytes())
WriteFile(t, filepath.Join(dir, name+".files"), buf.Bytes())
return pkgs
}
// BuildApkRepo writes a minimal Alpine repository into dir and returns the
// package contents by file name. The index replicates apk's real layout: a
// signature tar segment with its end-of-archive blocks cut off and the
// index tar as separate gzip streams, concatenated.
func BuildApkRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"musl-1.2.5-r1.apk": bytes.Repeat([]byte("musl"), 300),
"zlib-1.3-r2.apk": bytes.Repeat([]byte("zlib"), 200),
}
index := ""
for fname, data := range pkgs {
name := strings.SplitN(fname, "-", 2)[0]
version := strings.TrimSuffix(strings.TrimPrefix(fname, name+"-"), ".apk")
index += fmt.Sprintf("C:Q1notarealpullchecksum=\nP:%s\nV:%s\nA:x86_64\nS:%d\n\n", name, version, len(data))
WriteFile(t, filepath.Join(dir, fname), data)
}
// tarBytes renders entries into an uncompressed tar archive.
tarBytes := func(entries map[string]string) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for name, content := range entries {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(content))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(content)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// The signature segment loses its two zero end-of-archive blocks so
// the tar stream continues into the index segment.
sig := tarBytes(map[string]string{".SIGN.RSA.test.rsa.pub": "signature"})
sig = sig[:len(sig)-1024]
idx := tarBytes(map[string]string{"DESCRIPTION": "test repo", "APKINDEX": index})
combined := append(GzipBytes(t, sig), GzipBytes(t, idx)...)
WriteFile(t, filepath.Join(dir, "APKINDEX.tar.gz"), combined)
return pkgs
}
// BuildFlatDebRepo writes a minimal flat apt repository into dir and
// returns the package contents by file name.
func BuildFlatDebRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
data := bytes.Repeat([]byte("flat"), 250)
WriteFile(t, filepath.Join(dir, "hello_3.0_amd64.deb"), data)
packages := []byte(debPackagesStanza("hello", "3.0", "hello_3.0_amd64.deb", data))
packagesGz := GzipBytes(t, packages)
WriteFile(t, filepath.Join(dir, "Packages.gz"), packagesGz)
release := "Architectures: amd64\nSHA256:\n"
for p, d := range map[string][]byte{"Packages": packages, "Packages.gz": packagesGz} {
release += fmt.Sprintf(" %s %d %s\n", SHA256Hex(d), len(d), p)
}
WriteFile(t, filepath.Join(dir, "Release"), []byte(release))
return map[string][]byte{"hello_3.0_amd64.deb": data}
}

61
main.go Normal file
View file

@ -0,0 +1,61 @@
package main
import (
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// main parses the command line, installs the configuration every command
// shares, and dispatches to the selected command.
func main() {
ctx := ParseFlags()
if err := cfg.Init(flags.ConfigPath); err != nil {
log.Fatal(err)
}
applyGlobalFlags()
err := ctx.Run()
ctx.FatalIfErrorf(err)
}
// applyGlobalFlags overrides the loaded configuration with the global
// command line flags, then reapplies the settings derived from it. It also
// runs after a SIGHUP reload so the flags keep winning over the file.
func applyGlobalFlags() {
if flags.LogLevel != "" {
cfg.C.Log.Level = flags.LogLevel
}
if flags.UserAgent != "" {
cfg.C.Crawler.UserAgent = flags.UserAgent
}
if flags.RequestTimeout > 0 {
cfg.C.Crawler.RequestTimeout = flags.RequestTimeout
}
applyTraceFlags()
cfg.SetupLogging(cfg.C.Log)
fetch.Reload()
}
// applyTraceFlags overlays the trace command line flags onto the loaded
// configuration. Each flag is empty when unset so the configured value
// stands, matching how the other global overrides behave.
func applyTraceFlags() {
if flags.Trace != nil {
cfg.C.Trace.Enabled = *flags.Trace
}
for _, o := range []struct {
flag string
dst *string
}{
{flags.TraceHost, &cfg.C.Trace.Host},
{flags.TraceMaintainer, &cfg.C.Trace.Maintainer},
{flags.TraceSponsor, &cfg.C.Trace.Sponsor},
{flags.TraceCountry, &cfg.C.Trace.Country},
{flags.TraceLocation, &cfg.C.Trace.Location},
{flags.TraceThroughput, &cfg.C.Trace.Throughput},
} {
if o.flag != "" {
*o.dst = cfg.TraceValue(o.flag)
}
}
}

179
mirror/apk.go Normal file
View file

@ -0,0 +1,179 @@
package mirror
import (
"archive/tar"
"bufio"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// APKIndexName is the fixed name of an Alpine repository index.
const APKIndexName = "APKINDEX.tar.gz"
// apkPackage is one package entry parsed from an APKINDEX. Alpine's C:
// pull checksum only covers a package's control segment, so mirroring can
// verify downloads by size alone.
type apkPackage struct {
name string
version string
size int64
}
// filename returns the package file name as served beside the index.
func (p *apkPackage) filename() string {
return p.name + "-" + p.version + ".apk"
}
// expect converts the package's metadata into a download expectation.
func (p *apkPackage) expect() *fetch.Expect {
e := &fetch.Expect{Size: p.size}
if p.size <= 0 {
e.Size = -1
}
return e
}
// syncApk synchronizes one Alpine repository from src into destDir.
func syncApk(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
miss := opts.newMissing(destDir)
tr := newTrace(opts)
ctx = tr.track(ctx)
// Stage the index so a live tree keeps a consistent view until the
// packages it references are in place.
idxDst, err := fetch.LocalJoin(destDir, APKIndexName)
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, APKIndexName, idxDst, nil, true, false); err != nil {
return fmt.Errorf("fetch %s: %w", APKIndexName, err)
}
pkgs, err := readApkIndex(fetch.StagedOrFinal(idxDst))
if err != nil {
return err
}
// Download the packages, which live beside the index.
var jobs []fetch.Job
for _, pkg := range pkgs {
dst, err := fetch.LocalJoin(destDir, pkg.filename())
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename(), Dst: dst, Want: pkg.expect()})
}
log.WithField("packages", len(pkgs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// Promote the index last so the published metadata chain is complete.
if err := fetch.PromoteOrDiscard(idxDst, opts.DryRun); err != nil {
return err
}
keep.Add(idxDst)
// Publish the traces, upstream's included, before pruning so the keep
// set covers them.
tr.publish(ctx, destDir, src, keep, opts)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
// The index is published either way; missing packages only decide
// whether the run reports itself as failed.
return miss.Finish()
}
// readApkIndex parses an APKINDEX.tar.gz archive. The file is a signature
// segment and an index segment as concatenated gzip streams, which the
// multistream gzip reader and tar reader walk through transparently.
func readApkIndex(filename string) ([]apkPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
defer gz.Close()
tr := tar.NewReader(gz)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
if hdr.Name != "APKINDEX" {
continue
}
pkgs, err := parseApkIndex(tr)
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
return pkgs, nil
}
return nil, fmt.Errorf("no APKINDEX entry in %s", filename)
}
// parseApkIndex reads APKINDEX stanzas, collecting each package's name,
// version, and file size from the P:, V:, and S: fields.
func parseApkIndex(r io.Reader) ([]apkPackage, error) {
var pkgs []apkPackage
var cur apkPackage
// flush records the current stanza when it is complete.
flush := func() {
if cur.name != "" && cur.version != "" {
pkgs = append(pkgs, cur)
}
cur = apkPackage{}
}
scanner := bufio.NewScanner(r)
scanner.Buffer(make([]byte, 0, 256*1024), 256*1024)
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == "" {
flush()
continue
}
field, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
switch field {
case "P":
cur.name = value
case "V":
cur.version = value
case "S":
cur.size, _ = strconv.ParseInt(value, 10, 64)
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
flush()
return pkgs, nil
}

100
mirror/apk_test.go Normal file
View file

@ -0,0 +1,100 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseApkIndex verifies stanza splitting and field extraction.
func TestParseApkIndex(t *testing.T) {
index := "C:Q1abc=\nP:musl\nV:1.2.5-r1\nA:x86_64\nS:1200\n\nC:Q1def=\nP:zlib\nV:1.3-r2\nS:800\n"
pkgs, err := parseApkIndex(strings.NewReader(index))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Fatalf("got %d packages, want 2", len(pkgs))
}
if pkgs[0].filename() != "musl-1.2.5-r1.apk" || pkgs[0].size != 1200 {
t.Errorf("first package = %+v", pkgs[0])
}
if pkgs[1].filename() != "zlib-1.3-r2.apk" || pkgs[1].size != 800 {
t.Errorf("second package = %+v", pkgs[1])
}
}
// TestReadApkIndex verifies parsing through the signature segment of a
// multistream index archive.
func TestReadApkIndex(t *testing.T) {
dir := t.TempDir()
testrepos.BuildApkRepo(t, dir)
pkgs, err := readApkIndex(filepath.Join(dir, APKIndexName))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Errorf("got %d packages, want 2", len(pkgs))
}
}
// TestSyncApk verifies a full synchronization mirrors packages and the
// index with no staged leftovers.
func TestSyncApk(t *testing.T) {
www := t.TempDir()
pkgs := testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/alpine/v3.24/community/x86_64", opts.Type, opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "alpine", "v3.24", "community", "x86_64")
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(local, APKIndexName)); err != nil {
t.Error("index not promoted:", err)
}
staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncApkPrune verifies stale packages are removed when pruning.
func TestSyncApkPrune(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "repo", "old-1.0-r0.apk")
if err := os.WriteFile(stale, []byte("old"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale package survived pruning")
}
}

268
mirror/arch.go Normal file
View file

@ -0,0 +1,268 @@
package mirror
import (
"archive/tar"
"bufio"
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"sort"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// archPackage is one package entry parsed from a pacman database.
type archPackage struct {
filename string
size int64
sums map[string]string
}
// expect converts the package's metadata into a download expectation.
func (p *archPackage) expect() *fetch.Expect {
e := &fetch.Expect{Size: p.size, Sums: p.sums}
if p.size <= 0 {
e.Size = -1
}
return e
}
// archExtras lists the companion metadata files mirrored beside the
// database, as suffixes appended to the repository name. All are optional;
// which ones exist varies by repository.
var archExtras = []string{
".files",
".db.tar.gz",
".files.tar.gz",
".links.tar.gz",
".db.sig",
".db.tar.gz.sig",
".files.sig",
".files.tar.gz.sig",
".links.tar.gz.sig",
}
// syncArch synchronizes one pacman repository from src into destDir.
func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
miss := opts.newMissing(destDir)
tr := newTrace(opts)
ctx = tr.track(ctx)
// Determine the database name, which matches the repository rather
// than any fixed path.
name, err := archDBName(ctx, src, repoURL)
if err != nil {
return err
}
log.WithField("database", name+".db").Debug("Resolved pacman database.")
// Stage the database so a live tree keeps a consistent view until the
// packages it references are in place.
dbDst, err := fetch.LocalJoin(destDir, name+".db")
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, name+".db", dbDst, nil, true, false); err != nil {
return fmt.Errorf("fetch %s.db: %w", name, err)
}
pkgs, err := readArchDB(fetch.StagedOrFinal(dbDst))
if err != nil {
return err
}
// Download packages with their detached signatures. Signatures have no
// published checksums, so an existing file is trusted as-is.
var jobs []fetch.Job
for _, pkg := range pkgs {
dst, err := fetch.LocalJoin(destDir, pkg.filename)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect()})
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: &fetch.Expect{Size: -1}, Optional: true})
}
log.WithField("packages", len(pkgs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// Stage the companion metadata files that exist upstream. A dry run
// only counts them, as none are needed for planning.
var extraJobs []fetch.Job
for _, suffix := range archExtras {
dst, err := fetch.LocalJoin(destDir, name+suffix)
if err != nil {
return err
}
extraJobs = append(extraJobs, fetch.Job{ReqPath: name + suffix, Dst: dst, Optional: true, Stage: true})
}
if opts.DryRun {
fetch.PlanJobs(extraJobs, opts.Verify, keep)
} else {
states, err := fetch.Many(ctx, src, extraJobs, opts.Workers, opts.Verify, keep, nil)
if err != nil {
return fmt.Errorf("fetch repository metadata: %w", err)
}
for i, job := range extraJobs {
if states[i] == fetch.FileMissing {
// The upstream dropped the file; drop the local copy so a
// stale companion or signature is never served.
fetch.RemoveStale(job.Dst)
continue
}
if err := fetch.PromoteStaged(job.Dst); err != nil {
return err
}
}
}
// Promote the database last so the published metadata chain is
// complete.
if err := fetch.PromoteOrDiscard(dbDst, opts.DryRun); err != nil {
return err
}
keep.Add(dbDst)
// Publish the traces, upstream's included, before pruning so the keep
// set covers them.
tr.publish(ctx, destDir, src, keep, opts)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
// The database is published either way; missing packages only decide
// whether the run reports itself as failed.
return miss.Finish()
}
// archDBName finds the pacman database name for a repository, preferring
// the directory listing and falling back to probing names derived from the
// URL path, deepest segment first.
func archDBName(ctx context.Context, src *fetch.Source, repoURL string) (string, error) {
if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil {
var names []string
for f := range listing.files {
if strings.HasSuffix(f, ".db") {
names = append(names, strings.TrimSuffix(f, ".db"))
}
}
sort.Strings(names)
if len(names) > 1 {
log.WithField("databases", names).Warn("Multiple pacman databases found; using the first.")
}
if len(names) > 0 {
return names[0], nil
}
}
// Listings can be disabled; a repository at .../core/os/x86_64 is
// probed as x86_64.db, os.db, then core.db.
u, err := url.Parse(repoURL)
if err != nil {
return "", err
}
var segs []string
for _, seg := range strings.Split(u.Path, "/") {
if seg != "" {
segs = append(segs, seg)
}
}
for i := len(segs) - 1; i >= 0; i-- {
resp, err := src.Get(ctx, segs[i]+".db", fetch.GetOptions{})
if err == nil {
resp.Body.Close()
return segs[i], nil
}
if !errors.Is(err, fetch.ErrNotFound) {
return "", err
}
}
return "", fmt.Errorf("no pacman database found at %s", repoURL)
}
// readArchDB parses a pacman database archive and collects each package's
// file name, size, and checksums from its desc entry.
func readArchDB(filename string) ([]archPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
// The database carries no compression extension, so the format is
// sniffed from its leading bytes.
r, closeFn, err := fetch.SniffDecompressor(f)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
tr := tar.NewReader(r)
var pkgs []archPackage
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse pacman database %s: %w", filename, err)
}
if hdr.FileInfo().IsDir() || path.Base(hdr.Name) != "desc" {
continue
}
data, err := io.ReadAll(io.LimitReader(tr, 1<<20))
if err != nil {
return nil, fmt.Errorf("read desc entry %s: %w", hdr.Name, err)
}
fields := parseDesc(data)
if fields["FILENAME"] == "" {
continue
}
size, _ := strconv.ParseInt(fields["CSIZE"], 10, 64)
sums := map[string]string{}
if v := fields["SHA256SUM"]; v != "" {
sums["sha256"] = strings.ToLower(v)
}
if v := fields["MD5SUM"]; v != "" {
sums["md5"] = strings.ToLower(v)
}
pkgs = append(pkgs, archPackage{filename: fields["FILENAME"], size: size, sums: sums})
}
return pkgs, nil
}
// parseDesc extracts the first value of each %FIELD% block from a pacman
// desc entry.
func parseDesc(data []byte) map[string]string {
fields := map[string]string{}
field := ""
scanner := bufio.NewScanner(strings.NewReader(string(data)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
switch {
case line == "":
field = ""
case strings.HasPrefix(line, "%") && strings.HasSuffix(line, "%"):
field = strings.Trim(line, "%")
case field != "":
if _, ok := fields[field]; !ok {
fields[field] = line
}
}
}
return fields
}

166
mirror/arch_test.go Normal file
View file

@ -0,0 +1,166 @@
package mirror
import (
"archive/tar"
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseDesc verifies field extraction from a pacman desc entry.
func TestParseDesc(t *testing.T) {
desc := "%FILENAME%\nzlib-1.3-1-x86_64.pkg.tar.zst\n\n%CSIZE%\n1200\n\n%SHA256SUM%\nabc123\n\n%DESC%\nFirst line\nSecond line\n"
fields := parseDesc([]byte(desc))
if fields["FILENAME"] != "zlib-1.3-1-x86_64.pkg.tar.zst" {
t.Errorf("FILENAME = %q", fields["FILENAME"])
}
if fields["CSIZE"] != "1200" {
t.Errorf("CSIZE = %q", fields["CSIZE"])
}
if fields["SHA256SUM"] != "abc123" {
t.Errorf("SHA256SUM = %q", fields["SHA256SUM"])
}
if fields["DESC"] != "First line" {
t.Errorf("DESC = %q, want first value only", fields["DESC"])
}
}
// TestReadArchDB verifies database parsing for both compressed and plain
// tar archives.
func TestReadArchDB(t *testing.T) {
dir := t.TempDir()
testrepos.BuildArchRepo(t, dir, "core")
pkgs, err := readArchDB(filepath.Join(dir, "core.db"))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Fatalf("got %d packages, want 2", len(pkgs))
}
for _, pkg := range pkgs {
if pkg.filename == "" || pkg.size <= 0 || pkg.sums["sha256"] == "" {
t.Errorf("incomplete package entry: %+v", pkg)
}
}
// A plain uncompressed tar database must parse through the sniffer.
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
desc := "%FILENAME%\nplain-1.0-1-any.pkg.tar.zst\n\n%CSIZE%\n10\n\n%SHA256SUM%\nabc\n"
if err := tw.WriteHeader(&tar.Header{Name: "plain-1.0-1/desc", Mode: 0644, Size: int64(len(desc))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(desc)); err != nil {
t.Fatal(err)
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
plain := filepath.Join(dir, "plain.db")
if err := os.WriteFile(plain, buf.Bytes(), 0644); err != nil {
t.Fatal(err)
}
pkgs, err = readArchDB(plain)
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 1 || pkgs[0].filename != "plain-1.0-1-any.pkg.tar.zst" {
t.Errorf("plain tar parse = %+v", pkgs)
}
}
// TestSyncArch verifies a full synchronization mirrors packages,
// signatures, and metadata with no staged leftovers.
func TestSyncArch(t *testing.T) {
www := t.TempDir()
pkgs := testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoArch, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/core/os/x86_64", opts.Type, opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "core", "os", "x86_64")
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
if _, err := os.Stat(filepath.Join(local, name+".sig")); err != nil {
t.Errorf("signature for %s missing: %v", name, err)
}
}
for _, name := range []string{"core.db", "core.files"} {
if _, err := os.Stat(filepath.Join(local, name)); err != nil {
t.Errorf("%s not promoted: %v", name, err)
}
}
staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncArchPrune verifies stale files are removed when pruning.
func TestSyncArchPrune(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "core"), "core")
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoArch, Destination: dest, Workers: 2, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/core", opts.Type, opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "core", "old-0.9-1-x86_64.pkg.tar.zst")
if err := os.WriteFile(stale, []byte("old"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/core", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale package survived pruning")
}
}
// TestArchDBNameFallback verifies the database name is probed from URL
// path segments when directory listings are unavailable.
func TestArchDBNameFallback(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "custom", "os", "x86_64"), "custom")
// Serve files but refuse directory listings to force the fallback.
fs := http.FileServer(http.Dir(www))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
http.NotFound(w, r)
return
}
fs.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
src := fetch.NewSource([]string{srv.URL + "/custom/os/x86_64"})
name, err := archDBName(context.Background(), src, srv.URL+"/custom/os/x86_64")
if err != nil {
t.Fatal(err)
}
if name != "custom" {
t.Errorf("name = %q, want custom", name)
}
}

100
mirror/control.go Normal file
View file

@ -0,0 +1,100 @@
package mirror
import (
"bufio"
"io"
"strings"
)
// stanza is one RFC 822 style field block from an apt control file.
type stanza map[string]string
// Get returns the first matching field, trying each name in order to cope
// with casing differences between repositories.
func (s stanza) Get(names ...string) string {
for _, name := range names {
if v, ok := s[name]; ok {
return v
}
}
return ""
}
// controlReader reads stanzas from an apt control file.
type controlReader struct {
br *bufio.Reader
}
// newControlReader wraps a reader for stanza-by-stanza parsing.
func newControlReader(r io.Reader) *controlReader {
return &controlReader{br: bufio.NewReaderSize(r, 64*1024)}
}
// readStanza parses the next stanza, returning io.EOF when the input is
// exhausted. Continuation lines are joined with newlines so multi-line
// checksum fields keep one entry per line.
func (r *controlReader) readStanza() (stanza, error) {
st := stanza{}
var last string
for {
line, err := r.br.ReadString('\n')
if err != nil && err != io.EOF {
return nil, err
}
done := err == io.EOF && line == ""
line = strings.TrimRight(line, "\r\n")
switch {
case done || strings.TrimSpace(line) == "":
if len(st) > 0 {
return st, nil
}
if done {
return nil, io.EOF
}
case line[0] == ' ' || line[0] == '\t':
if last != "" {
st[last] += "\n" + strings.TrimSpace(line)
}
default:
name, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
last = name
st[name] = strings.TrimSpace(value)
}
}
}
// stripClearsign extracts the signed payload from a clearsigned InRelease
// document, returning the input unchanged when it is not clearsigned.
func stripClearsign(data []byte) []byte {
const begin = "-----BEGIN PGP SIGNED MESSAGE-----"
const sig = "-----BEGIN PGP SIGNATURE-----"
text := strings.ReplaceAll(string(data), "\r\n", "\n")
idx := strings.Index(text, begin)
if idx < 0 {
return data
}
text = text[idx+len(begin):]
// Skip the armor headers up to the first blank line. Without one the
// document is malformed, so treat it as not clearsigned rather than
// returning the armor headers as payload.
i := strings.Index(text, "\n\n")
if i < 0 {
return data
}
text = text[i+2:]
if i := strings.Index(text, sig); i >= 0 {
text = text[:i]
}
// Reverse the dash escaping applied by the signer.
var b strings.Builder
for _, line := range strings.Split(text, "\n") {
b.WriteString(strings.TrimPrefix(line, "- "))
b.WriteString("\n")
}
return []byte(b.String())
}

70
mirror/control_test.go Normal file
View file

@ -0,0 +1,70 @@
package mirror
import (
"errors"
"io"
"strings"
"testing"
)
// TestReadStanza verifies stanza splitting, continuation joining, and EOF
// handling without a trailing newline.
func TestReadStanza(t *testing.T) {
input := "Package: hello\nSHA256:\n abc 10 path/one\n def 20 path/two\n\nPackage: world\nSize: 42"
cr := newControlReader(strings.NewReader(input))
st, err := cr.readStanza()
if err != nil {
t.Fatal(err)
}
if st.Get("Package") != "hello" {
t.Errorf("Package = %q, want hello", st.Get("Package"))
}
if want := "\nabc 10 path/one\ndef 20 path/two"; st.Get("SHA256") != want {
t.Errorf("SHA256 = %q, want %q", st.Get("SHA256"), want)
}
st, err = cr.readStanza()
if err != nil {
t.Fatal(err)
}
if st.Get("Package") != "world" || st.Get("Size") != "42" {
t.Errorf("second stanza = %v", st)
}
if _, err = cr.readStanza(); !errors.Is(err, io.EOF) {
t.Errorf("expected EOF, got %v", err)
}
}
// TestStanzaGet verifies fallback field names cover casing differences.
func TestStanzaGet(t *testing.T) {
st := stanza{"MD5Sum": "abc"}
if got := st.Get("MD5sum", "MD5Sum"); got != "abc" {
t.Errorf("got %q, want abc", got)
}
if got := st.Get("Missing"); got != "" {
t.Errorf("got %q, want empty", got)
}
}
// TestStripClearsign verifies payload extraction and dash unescaping from a
// clearsigned document, and passthrough of unsigned input.
func TestStripClearsign(t *testing.T) {
signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nOrigin: Test\n- Dashed: line\n-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n"
got := string(stripClearsign([]byte(signed)))
if !strings.Contains(got, "Origin: Test\n") {
t.Errorf("payload missing Origin: %q", got)
}
if !strings.Contains(got, "Dashed: line") || strings.Contains(got, "- Dashed") {
t.Errorf("dash escaping not reversed: %q", got)
}
if strings.Contains(got, "PGP") {
t.Errorf("armor not removed: %q", got)
}
plain := "Origin: Test\n"
if string(stripClearsign([]byte(plain))) != plain {
t.Error("unsigned input was modified")
}
}

542
mirror/deb.go Normal file
View file

@ -0,0 +1,542 @@
package mirror
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// debFile is one file referenced by repository metadata with its expected
// size and checksums, keyed by lowercase algorithm name.
type debFile struct {
path string
size int64
sums map[string]string
}
// expect converts the file's metadata into a download expectation.
func (f *debFile) expect() *fetch.Expect {
e := &fetch.Expect{Size: f.size, Sums: f.sums}
if f.size <= 0 {
e.Size = -1
}
return e
}
// debRelease is the parsed subset of an apt Release file needed to mirror
// a suite.
type debRelease struct {
components []string
architectures []string
acquireByHash bool
files map[string]*debFile
}
// releaseSumFields maps Release checksum block names to algorithm names,
// including the MD5sum casing some repositories use.
var releaseSumFields = map[string]string{
"MD5Sum": "md5sum",
"MD5sum": "md5sum",
"SHA1": "sha1",
"SHA256": "sha256",
"SHA512": "sha512",
}
// byHashDirs maps checksum algorithm names to the by-hash directory names
// apt uses.
var byHashDirs = map[string]string{
"md5sum": "MD5Sum",
"sha1": "SHA1",
"sha256": "SHA256",
"sha512": "SHA512",
}
// parseRelease extracts the fields the sync needs from a Release or
// InRelease document.
func parseRelease(data []byte) (*debRelease, error) {
st, err := newControlReader(bytes.NewReader(stripClearsign(data))).readStanza()
if err != nil {
return nil, fmt.Errorf("parse release file: %w", err)
}
rel := &debRelease{files: map[string]*debFile{}}
rel.components = strings.Fields(st.Get("Components"))
rel.architectures = strings.Fields(st.Get("Architectures"))
rel.acquireByHash = strings.EqualFold(st.Get("Acquire-By-Hash"), "yes")
// Merge each checksum block into one record per file path.
for field, algo := range releaseSumFields {
for _, line := range strings.Split(st.Get(field), "\n") {
parts := strings.Fields(line)
if len(parts) != 3 {
continue
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
continue
}
f := rel.files[parts[2]]
if f == nil {
f = &debFile{path: parts[2], sums: map[string]string{}}
rel.files[parts[2]] = f
}
f.size = size
f.sums[algo] = strings.ToLower(parts[0])
}
}
return rel, nil
}
// syncDeb synchronizes one apt suite or flat repository from repoURL.
func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
tr := newTrace(opts)
ctx = tr.track(ctx)
// Split the URL into the archive root and the suite path. Repositories
// without a dists directory are flat, with everything relative to the
// repository URL itself.
u, err := url.Parse(repoURL)
if err != nil {
return err
}
suiteRel := ""
rootURL := repoURL
if idx := strings.Index(u.Path, "/dists/"); idx >= 0 {
suiteRel = strings.Trim(u.Path[idx+1:], "/")
root := *u
root.Path = u.Path[:idx]
rootURL = root.String()
src.Rebase(suiteRel)
}
destRoot, err := opts.repoDest(rootURL)
if err != nil {
return err
}
destSuite := destRoot
suitePrefix := ""
if suiteRel != "" {
if destSuite, err = fetch.LocalJoin(destRoot, suiteRel); err != nil {
return err
}
suitePrefix = suiteRel + "/"
}
// Track the pool files this suite lists but the upstream does not serve.
// The records live in the suite directory, as the pool is shared with
// the archive's other suites.
miss := opts.newMissing(destSuite)
// Stage the release files; they are promoted last so a live tree keeps
// a consistent metadata chain.
releaseNames := []string{"InRelease", "Release", "Release.gpg"}
releaseStates := make([]fetch.FileState, len(releaseNames))
for i, name := range releaseNames {
dst, err := fetch.LocalJoin(destSuite, name)
if err != nil {
return err
}
state, err := fetch.File(ctx, src, suitePrefix+name, dst, nil, true, false)
if err != nil {
if errors.Is(err, fetch.ErrNotFound) {
// The upstream dropped this release file; drop the local
// copy so outdated metadata is never served.
if !opts.DryRun {
fetch.RemoveStale(dst)
}
continue
}
return fmt.Errorf("fetch %s: %w", name, err)
}
releaseStates[i] = state
}
// Parse the strongest release document available, which may be the
// promoted copy when the upstream reported it unchanged.
var relData []byte
switch {
case releaseStates[0] != fetch.FileMissing:
relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "InRelease")))
case releaseStates[1] != fetch.FileMissing:
relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "Release")))
default:
return errors.New("repository serves neither InRelease nor Release")
}
if err != nil {
return err
}
rel, err := parseRelease(relData)
if err != nil {
return err
}
// Stage every index file the release lists, applying component and
// architecture filters. Missing variants are normal; repositories list
// checksums for files they do not serve. A dry run only fetches the
// package and source indexes needed to plan the pool.
var indexJobs, plannedJobs []fetch.Job
for _, f := range sortedFiles(rel.files) {
// Releases commonly list their own release files; those are
// already staged above and must not be fetched twice.
if slices.Contains(releaseNames, f.path) {
continue
}
if !includeIndexFile(f.path, opts.Components, opts.Architectures) {
continue
}
dst, err := fetch.LocalJoin(destSuite, f.path)
if err != nil {
return err
}
job := fetch.Job{
ReqPath: suitePrefix + f.path,
Dst: dst,
Want: f.expect(),
Optional: true,
Stage: true,
}
if opts.DryRun {
base, _ := splitCompressExt(f.path)
if name := path.Base(base); name != "Packages" && name != "Sources" {
plannedJobs = append(plannedJobs, job)
continue
}
}
indexJobs = append(indexJobs, job)
}
fetch.PlanJobs(plannedJobs, opts.Verify, keep)
states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, opts.Verify, keep, nil)
if err != nil {
return fmt.Errorf("fetch suite indexes: %w", err)
}
// Pick one local compression variant of each package and source index
// to parse for pool file references.
variants := map[string]string{}
variantExt := map[string]string{}
prefOrder := map[string]int{".gz": 0, ".xz": 1, ".zst": 2, ".bz2": 3, "": 4}
for i, job := range indexJobs {
if states[i] == fetch.FileMissing {
continue
}
relPath := strings.TrimPrefix(job.ReqPath, suitePrefix)
base, ext := splitCompressExt(relPath)
if name := path.Base(base); name != "Packages" && name != "Sources" {
continue
}
local := job.Dst
if states[i] == fetch.FileStaged {
local = job.Dst + fetch.StagedSuffix
}
if cur, ok := variantExt[base]; !ok || prefOrder[ext] < prefOrder[cur] {
variantExt[base] = ext
variants[base] = local
}
}
// Collect the pool files referenced by the chosen indexes.
pool := map[string]*debFile{}
for _, base := range sortedKeys(variants) {
files, err := parseIndexFile(variants[base], path.Base(base))
if err != nil {
return fmt.Errorf("parse %s: %w", base, err)
}
for _, f := range files {
if _, ok := pool[f.path]; !ok {
pool[f.path] = f
}
}
}
// Download pool files relative to the archive root.
poolJobs := make([]fetch.Job, 0, len(pool))
for _, f := range sortedFiles(pool) {
dst, err := fetch.LocalJoin(destRoot, f.path)
if err != nil {
return err
}
poolJobs = append(poolJobs, fetch.Job{ReqPath: f.path, Dst: dst, Want: f.expect()})
}
log.WithField("files", len(poolJobs)).Info("Synchronizing pool files.")
if opts.DryRun {
fetch.PlanJobs(poolJobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, opts.Verify, keep, miss); err != nil {
return fmt.Errorf("fetch pool files: %w", err)
}
// Promote the staged indexes now the pool files they reference exist.
for i, job := range indexJobs {
if states[i] == fetch.FileMissing {
continue
}
if err := fetch.PromoteOrDiscard(job.Dst, opts.DryRun); err != nil {
return err
}
if rel.acquireByHash && !opts.DryRun {
addByHash(job.Dst, strings.TrimPrefix(job.ReqPath, suitePrefix), rel, keep)
}
}
// Promote the release files last to complete the metadata chain.
for _, name := range []string{"Release.gpg", "Release", "InRelease"} {
dst, err := fetch.LocalJoin(destSuite, name)
if err != nil {
return err
}
if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil {
return err
}
if _, err := os.Stat(dst); err == nil {
keep.Add(dst)
}
}
// Publish the traces at the archive root, where the convention places
// them and where they describe the archive as a whole rather than one
// suite. Writing before pruning lets the keep set cover them, which
// matters for a flat repository, whose root is itself the pruned tree.
tr.publish(ctx, destRoot, src, keep, opts)
// Prune only the suite directory for structured repositories; the pool
// can be shared with other suites of the same archive.
if opts.Prune {
if suiteRel == "" {
fetch.PruneTree(destRoot, keep, opts.PruneGrace, opts.DryRun)
} else {
log.Info("Pruning is limited to the suite directory; shared pool files are kept.")
fetch.PruneTree(destSuite, keep, opts.PruneGrace, opts.DryRun)
}
}
// The suite is published either way; missing pool files only decide
// whether the run reports itself as failed.
return miss.Finish()
}
// addByHash links an index file into its directory's by-hash tree for each
// checksum the release lists, matching apt's Acquire-By-Hash layout.
// Failures are logged and do not fail the sync.
func addByHash(dst, relPath string, rel *debRelease, keep *fetch.KeepSet) {
f := rel.files[relPath]
if f == nil || !strings.Contains(relPath, "/") {
return
}
dir := filepath.Dir(dst)
for algo, digest := range f.sums {
sub, ok := byHashDirs[algo]
if !ok {
continue
}
link := filepath.Join(dir, "by-hash", sub, digest)
if err := os.MkdirAll(filepath.Dir(link), 0755); err != nil {
log.WithError(err).WithField("path", link).Warn("Failed to create by-hash directory.")
return
}
if _, err := os.Stat(link); err == nil {
keep.Add(link)
continue
}
if err := os.Link(dst, link); err != nil {
// Fall back to copying when hard links are not supported.
if err := copyFile(dst, link); err != nil {
log.WithError(err).WithField("path", link).Warn("Failed to create by-hash entry.")
continue
}
}
keep.Add(link)
}
}
// copyFile duplicates src to dst.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
_ = os.Remove(dst)
return err
}
return out.Close()
}
// parseIndexFile reads a Packages or Sources index and returns the pool
// files it references, with paths relative to the archive root.
func parseIndexFile(filename, kind string) ([]*debFile, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
cr := newControlReader(r)
var out []*debFile
for {
st, err := cr.readStanza()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, err
}
if kind == "Sources" {
out = append(out, sourceFiles(st)...)
continue
}
name := st.Get("Filename")
if name == "" {
continue
}
size, _ := strconv.ParseInt(st.Get("Size"), 10, 64)
sums := map[string]string{}
if v := st.Get("MD5sum", "MD5Sum"); v != "" {
sums["md5sum"] = strings.ToLower(v)
}
for _, algo := range []string{"SHA1", "SHA256", "SHA512"} {
if v := st.Get(algo); v != "" {
sums[strings.ToLower(algo)] = strings.ToLower(v)
}
}
out = append(out, &debFile{path: path.Clean(name), size: size, sums: sums})
}
return out, nil
}
// sourceSumFields maps source stanza checksum list fields to algorithm
// names; Files is the historical MD5 list.
var sourceSumFields = map[string]string{
"Files": "md5sum",
"Checksums-Sha1": "sha1",
"Checksums-Sha256": "sha256",
"Checksums-Sha512": "sha512",
}
// sourceFiles extracts the files of one source package stanza, merging the
// per-algorithm checksum lists by file name.
func sourceFiles(st stanza) []*debFile {
dir := st.Get("Directory")
byName := map[string]*debFile{}
var order []string
for field, algo := range sourceSumFields {
for _, line := range strings.Split(st.Get(field), "\n") {
parts := strings.Fields(line)
if len(parts) != 3 {
continue
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
continue
}
name := parts[2]
f := byName[name]
if f == nil {
f = &debFile{path: path.Join(dir, name), sums: map[string]string{}}
byName[name] = f
order = append(order, name)
}
f.size = size
f.sums[algo] = strings.ToLower(parts[0])
}
}
out := make([]*debFile, 0, len(byName))
for _, name := range order {
out = append(out, byName[name])
}
return out
}
// includeIndexFile applies the component and architecture filters to a
// release-listed file path.
func includeIndexFile(p string, comps, archs []string) bool {
if len(comps) > 0 {
if segs := strings.Split(p, "/"); len(segs) > 1 && !slices.Contains(comps, segs[0]) {
return false
}
}
if len(archs) > 0 {
if arch := indexArch(p); arch != "" && !slices.Contains(archs, arch) {
return false
}
}
return true
}
// indexArch extracts the architecture a release-listed file applies to, or
// an empty string when the file is architecture independent.
func indexArch(p string) string {
for _, seg := range strings.Split(p, "/") {
if arch, ok := strings.CutPrefix(seg, "binary-"); ok {
return arch
}
if arch, ok := strings.CutPrefix(seg, "installer-"); ok {
return arch
}
if seg == "source" {
return "source"
}
}
base, _ := splitCompressExt(path.Base(p))
if arch, ok := strings.CutPrefix(base, "Contents-udeb-"); ok {
return arch
}
if arch, ok := strings.CutPrefix(base, "Contents-"); ok {
return arch
}
return ""
}
// splitCompressExt splits a known compression extension off a path.
func splitCompressExt(p string) (string, string) {
for _, ext := range []string{".gz", ".xz", ".bz2", ".zst"} {
if strings.HasSuffix(p, ext) {
return strings.TrimSuffix(p, ext), ext
}
}
return p, ""
}
// sortedFiles returns metadata file records in a stable path order.
func sortedFiles(files map[string]*debFile) []*debFile {
out := make([]*debFile, 0, len(files))
for _, f := range files {
out = append(out, f)
}
sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path })
return out
}
// sortedKeys returns a map's keys in a stable order.
func sortedKeys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}

291
mirror/deb_test.go Normal file
View file

@ -0,0 +1,291 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseRelease verifies field extraction and checksum block merging.
func TestParseRelease(t *testing.T) {
release := "Origin: Test\nArchitectures: amd64 arm64\nComponents: main universe\nAcquire-By-Hash: yes\n" +
"MD5Sum:\n aaa 10 main/binary-amd64/Packages\n" +
"SHA256:\n bbb 10 main/binary-amd64/Packages\n ccc 20 main/binary-amd64/Packages.gz\n"
rel, err := parseRelease([]byte(release))
if err != nil {
t.Fatal(err)
}
if len(rel.components) != 2 || rel.components[0] != "main" {
t.Errorf("components = %v", rel.components)
}
if len(rel.architectures) != 2 {
t.Errorf("architectures = %v", rel.architectures)
}
if !rel.acquireByHash {
t.Error("acquireByHash not detected")
}
f := rel.files["main/binary-amd64/Packages"]
if f == nil || f.size != 10 || f.sums["md5sum"] != "aaa" || f.sums["sha256"] != "bbb" {
t.Errorf("merged file = %+v", f)
}
if rel.files["main/binary-amd64/Packages.gz"] == nil {
t.Error("gz variant missing")
}
}
// TestIndexArch verifies architecture extraction from release file paths.
func TestIndexArch(t *testing.T) {
cases := map[string]string{
"main/binary-amd64/Packages.gz": "amd64",
"main/debian-installer/binary-i386/Packages": "i386",
"main/installer-armhf/current/images/SHA256SUMS": "armhf",
"main/source/Sources.gz": "source",
"Contents-s390x.gz": "s390x",
"main/Contents-udeb-riscv64.gz": "riscv64",
"main/i18n/Translation-en.gz": "",
"main/dep11/icons-64x64.tar.gz": "",
"Release": "",
}
for p, want := range cases {
if got := indexArch(p); got != want {
t.Errorf("indexArch(%q) = %q, want %q", p, got, want)
}
}
}
// TestIncludeIndexFile verifies component and architecture filtering.
func TestIncludeIndexFile(t *testing.T) {
cases := []struct {
path string
comps []string
archs []string
want bool
}{
{"main/binary-amd64/Packages.gz", nil, nil, true},
{"universe/binary-amd64/Packages.gz", []string{"main"}, nil, false},
{"main/binary-i386/Packages.gz", []string{"main"}, []string{"amd64"}, false},
{"main/binary-amd64/Packages.gz", []string{"main"}, []string{"amd64"}, true},
{"main/i18n/Translation-en.gz", []string{"main"}, []string{"amd64"}, true},
{"Contents-arm64.gz", []string{"main"}, []string{"amd64"}, false},
{"main/source/Sources.gz", nil, []string{"amd64"}, false},
{"main/source/Sources.gz", nil, []string{"amd64", "source"}, true},
}
for _, c := range cases {
if got := includeIndexFile(c.path, c.comps, c.archs); got != c.want {
t.Errorf("includeIndexFile(%q, %v, %v) = %v, want %v", c.path, c.comps, c.archs, got, c.want)
}
}
}
// TestSourceFiles verifies checksum list merging and directory joining for
// source package stanzas.
func TestSourceFiles(t *testing.T) {
st := stanza{
"Directory": "pool/main/h/hello",
"Files": "\naaa 100 hello_1.0.dsc\nbbb 200 hello_1.0.tar.gz",
"Checksums-Sha256": "\nccc 100 hello_1.0.dsc\nddd 200 hello_1.0.tar.gz",
}
files := sourceFiles(st)
if len(files) != 2 {
t.Fatalf("got %d files, want 2", len(files))
}
byPath := map[string]*debFile{}
for _, f := range files {
byPath[f.path] = f
}
dsc := byPath["pool/main/h/hello/hello_1.0.dsc"]
if dsc == nil || dsc.size != 100 || dsc.sums["md5sum"] != "aaa" || dsc.sums["sha256"] != "ccc" {
t.Errorf("dsc = %+v", dsc)
}
}
// TestSyncDebStructured verifies a dists-layout synchronization: pool files
// at the archive root, promoted indexes, by-hash entries, and no staged
// leftovers.
func TestSyncDebStructured(t *testing.T) {
www := t.TempDir()
pool := testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "debian")
for p, data := range pool {
got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(p)))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("pool file %s content mismatch", p)
}
}
suite := filepath.Join(root, "dists", "test")
if _, err := os.Stat(filepath.Join(suite, "Release")); err != nil {
t.Error("Release not promoted:", err)
}
if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages.gz")); err != nil {
t.Error("Packages.gz missing:", err)
}
// The uncompressed variant is listed in the Release file but not
// served, so it must be skipped without failing the sync.
if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages")); err == nil {
t.Error("unserved uncompressed Packages unexpectedly present")
}
byHash, err := filepath.Glob(filepath.Join(suite, "main", "binary-amd64", "by-hash", "SHA256", "*"))
if err != nil || len(byHash) == 0 {
t.Error("by-hash entries missing")
}
staged, _ := filepath.Glob(filepath.Join(suite, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncDebComponentFilter verifies filtered components are not mirrored.
func TestSyncDebComponentFilter(t *testing.T) {
www := t.TempDir()
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2, Components: []string{"main"}}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "debian")
if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil {
t.Error("main pool file missing:", err)
}
if _, err := os.Stat(filepath.Join(root, "pool", "universe")); err == nil {
t.Error("filtered universe pool files present")
}
if _, err := os.Stat(filepath.Join(root, "dists", "test", "universe")); err == nil {
t.Error("filtered universe indexes present")
}
}
// TestSyncDebFlat verifies a repository without a dists directory mirrors
// relative to the repository URL itself.
func TestSyncDebFlat(t *testing.T) {
www := t.TempDir()
pool := testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/flat", opts.Type, opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "flat")
for name, data := range pool {
got, err := os.ReadFile(filepath.Join(root, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(root, "Release")); err != nil {
t.Error("Release not promoted:", err)
}
}
// TestSyncDebMissingPoolRetry verifies a pool file the upstream stopped
// serving does not cost the rest of the suite: the run mirrors everything
// else, reports the absence until its retry budget runs out, and then
// synchronizes cleanly with the absence recorded as known.
func TestSyncDebMissingPoolRetry(t *testing.T) {
www := t.TempDir()
archive := filepath.Join(www, "debian")
testrepos.BuildDebRepo(t, archive)
srv := testrepos.ServeDir(t, www)
// Withdraw one pool file while its index keeps listing it, which is how
// an incomplete upstream presents itself.
const gone = "pool/universe/e/extra/extra_2.0_amd64.deb"
if err := os.Remove(filepath.Join(archive, filepath.FromSlash(gone))); err != nil {
t.Fatal(err)
}
dest := t.TempDir()
opts := &Options{
Type: RepoDeb,
Destination: dest,
Workers: 2,
Missing: fetch.MissingRetry,
MissingRetries: 2,
}
// The first run still publishes the suite, but reports the absence.
err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts)
if err == nil {
t.Fatal("the first run hid a missing pool file")
}
root := filepath.Join(dest, "debian")
if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil {
t.Error("a missing pool file cost the rest of the pool:", err)
}
if _, err := os.Stat(filepath.Join(root, "dists", "test", "Release")); err != nil {
t.Error("a missing pool file cost the suite metadata:", err)
}
// The second run exhausts the retry budget, so the absence is accepted
// and recorded in the suite directory rather than failing the run.
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil {
t.Fatal("the absence still failed the run after its retry budget:", err)
}
state := filepath.Join(root, "dists", "test", fetch.MissingStateName)
data, err := os.ReadFile(state)
if err != nil {
t.Fatal("the known absence was not recorded:", err)
}
if !bytes.Contains(data, []byte(gone)) {
t.Errorf("recorded state %s does not name the missing file", data)
}
// Serving the file again clears the record on the next run.
testrepos.WriteFile(t, filepath.Join(archive, filepath.FromSlash(gone)), bytes.Repeat([]byte("extra"), 400))
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(state); !os.IsNotExist(err) {
t.Error("the record survived the upstream serving the file again")
}
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(gone))); err != nil {
t.Error("the restored pool file was not fetched:", err)
}
}
// TestSyncDebMissingPoolModes verifies the strict mode still fails on the
// first missing pool file and the ignore mode never fails at all.
func TestSyncDebMissingPoolModes(t *testing.T) {
www := t.TempDir()
archive := filepath.Join(www, "debian")
testrepos.BuildDebRepo(t, archive)
if err := os.Remove(filepath.Join(archive, filepath.FromSlash("pool/universe/e/extra/extra_2.0_amd64.deb"))); err != nil {
t.Fatal(err)
}
srv := testrepos.ServeDir(t, www)
strict := &Options{Type: RepoDeb, Destination: t.TempDir(), Workers: 2, Missing: fetch.MissingFail}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", strict.Type, strict); err == nil {
t.Error("the strict mode tolerated a missing pool file")
}
lenient := &Options{Type: RepoDeb, Destination: t.TempDir(), Workers: 2, Missing: fetch.MissingIgnore}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", lenient.Type, lenient); err != nil {
t.Error("the ignore mode failed on a missing pool file:", err)
}
}

299
mirror/discover.go Normal file
View file

@ -0,0 +1,299 @@
package mirror
import (
"context"
"fmt"
"io"
"net/url"
"path"
"slices"
"sort"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
"golang.org/x/net/html"
)
// dirListing holds the entries parsed from an HTML directory index.
type dirListing struct {
dirs []string
files map[string]bool
}
// repoTarget is one repository to synchronize paired with the format it is
// synchronized as. A single crawl can turn up several formats, so the type
// travels with the repository rather than with the run.
type repoTarget struct {
url string
typ RepoType
}
// fileTarget is one loose file an include pattern matched during discovery,
// kept as the directory it was found in and its name so its destination
// follows the same path mapping a repository's does.
type fileTarget struct {
dirURL string
name string
}
// discovery holds what a crawl found below a base URL.
type discovery struct {
repos []repoTarget
files []fileTarget
}
// discoverRepos crawls directory listings under baseURL looking for
// repositories of the formats the run accepts, descending at most
// opts.DiscoverDepth levels. Excluded entries are never crawled, and files
// matching the include patterns are collected along the way.
func discoverRepos(ctx context.Context, baseURL string, opts *Options) (*discovery, error) {
type node struct {
url string
rel string
depth int
}
types := opts.discoverTypes()
start := strings.TrimRight(baseURL, "/") + "/"
queue := []node{{url: start}}
visited := map[string]bool{start: true}
found := &discovery{}
for len(queue) > 0 {
if err := ctx.Err(); err != nil {
return nil, err
}
n := queue[0]
queue = queue[1:]
listing, err := listDir(ctx, n.url)
if err != nil {
log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.")
continue
}
// A directory containing repository metadata is synchronized as a
// whole; its children are not crawled further.
if typ, ok := repoTypeOf(listing, types); ok {
found.repos = append(found.repos, repoTarget{url: strings.TrimRight(n.url, "/"), typ: typ})
continue
}
// Collect the loose files the run asks for. Only directories that
// are not repositories are considered: a repository's own files come
// from its metadata, and pruning there would remove anything else.
found.files = append(found.files, matchedFiles(listing, n.url, n.rel, opts)...)
if n.depth >= opts.DiscoverDepth {
continue
}
for _, dir := range listing.dirs {
rel := path.Join(n.rel, dir)
if matchAny(opts.Exclude, dir, rel) {
log.WithField("url", n.url+dir+"/").Debug("Skipped excluded directory.")
continue
}
child := n.url + url.PathEscape(dir) + "/"
if visited[child] {
continue
}
visited[child] = true
queue = append(queue, node{url: child, rel: rel, depth: n.depth + 1})
}
}
sort.Slice(found.repos, func(i, j int) bool { return found.repos[i].url < found.repos[j].url })
sort.Slice(found.files, func(i, j int) bool {
if found.files[i].dirURL != found.files[j].dirURL {
return found.files[i].dirURL < found.files[j].dirURL
}
return found.files[i].name < found.files[j].name
})
return found, nil
}
// repoTypeOf reports the format a directory listing identifies, considering
// only the formats the run accepts. The first match wins, so a directory
// carrying more than one format's markers is synchronized as the earlier
// one rather than twice.
func repoTypeOf(listing *dirListing, types []RepoType) (RepoType, bool) {
for _, typ := range types {
switch typ {
case RepoRPM:
if slices.Contains(listing.dirs, "repodata") {
return typ, true
}
case RepoDeb:
if listing.files["InRelease"] || listing.files["Release"] {
return typ, true
}
case RepoArch:
if hasArchDB(listing.files) {
return typ, true
}
case RepoApk:
if listing.files[APKIndexName] {
return typ, true
}
}
}
return "", false
}
// matchedFiles returns the files in a listing the include patterns select,
// in name order so a run's work is deterministic.
func matchedFiles(listing *dirListing, dirURL, rel string, opts *Options) []fileTarget {
if len(opts.IncludeFiles) == 0 {
return nil
}
names := make([]string, 0, len(listing.files))
for name := range listing.files {
names = append(names, name)
}
sort.Strings(names)
dir := strings.TrimRight(dirURL, "/")
var files []fileTarget
for _, name := range names {
entry := path.Join(rel, name)
if matchAny(opts.Exclude, name, entry) {
continue
}
if !matchAny(opts.IncludeFiles, name, entry) {
continue
}
files = append(files, fileTarget{dirURL: dir, name: name})
}
return files
}
// matchAny reports whether any glob matches an entry. A pattern containing
// a slash is matched against the entry's path below the crawled URL, so it
// can pin a name to one place in the tree; a plain pattern is matched
// against the name alone, wherever in the tree it appears. Patterns are
// validated before a run starts, so a bad one cannot silently never match.
func matchAny(patterns []string, name, rel string) bool {
for _, pattern := range patterns {
pattern, subject := matchSubject(pattern, name, rel)
if ok, err := path.Match(pattern, subject); err == nil && ok {
return true
}
}
return false
}
// matchSubject pairs a pattern with the string it is matched against. A
// leading slash anchors the pattern to the crawled URL, so it is stripped
// and the entry's path below that URL is matched; at the root that path is
// the bare name, which is what makes anchoring to the root expressible at
// all. Any other pattern carrying a slash is matched against the same path
// unanchored, and a plain pattern against the name alone.
func matchSubject(pattern, name, rel string) (string, string) {
if strings.HasPrefix(pattern, "/") {
return strings.TrimPrefix(pattern, "/"), rel
}
if strings.Contains(pattern, "/") {
return pattern, rel
}
return pattern, name
}
// ValidPattern reports whether a glob is usable as an exclude or include
// pattern, so a run rejects a malformed one instead of never matching it.
// The anchoring slash is not part of the glob, so it is stripped before the
// syntax is checked.
func ValidPattern(pattern string) error {
_, err := path.Match(strings.TrimPrefix(pattern, "/"), "")
return err
}
// hasArchDB reports whether a directory listing contains a pacman database.
func hasArchDB(files map[string]bool) bool {
for name := range files {
if strings.HasSuffix(name, ".db") {
return true
}
}
return false
}
// listDir fetches an HTML directory index and extracts its immediate child
// directories and files.
func listDir(ctx context.Context, dirURL string) (*dirListing, error) {
resp, err := fetch.Get(ctx, dirURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode)
}
base, err := url.Parse(dirURL)
if err != nil {
return nil, err
}
listing := &dirListing{files: map[string]bool{}}
seenDirs := map[string]bool{}
tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20))
for {
tt := tok.Next()
if tt == html.ErrorToken {
break
}
if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
continue
}
name, hasAttr := tok.TagName()
if string(name) != "a" || !hasAttr {
continue
}
var href string
for {
key, val, more := tok.TagAttr()
if string(key) == "href" {
href = string(val)
}
if !more {
break
}
}
entry, isDir := childEntry(base, href)
if entry == "" {
continue
}
if isDir {
if !seenDirs[entry] {
seenDirs[entry] = true
listing.dirs = append(listing.dirs, entry)
}
} else {
listing.files[entry] = true
}
}
return listing, nil
}
// childEntry resolves an anchor href against the directory URL, returning
// the entry name when it is an immediate child of that directory.
func childEntry(base *url.URL, href string) (string, bool) {
if href == "" || strings.HasPrefix(href, "#") {
return "", false
}
ref, err := url.Parse(href)
if err != nil || ref.RawQuery != "" || ref.Fragment != "" {
return "", false
}
resolved := base.ResolveReference(ref)
if resolved.Scheme != base.Scheme || resolved.Host != base.Host {
return "", false
}
rel := strings.TrimPrefix(resolved.Path, base.Path)
if rel == "" || rel == resolved.Path {
return "", false
}
isDir := strings.HasSuffix(rel, "/")
rel = strings.TrimSuffix(rel, "/")
if rel == "" || strings.Contains(rel, "/") {
return "", false
}
return rel, isDir
}

374
mirror/discover_test.go Normal file
View file

@ -0,0 +1,374 @@
package mirror
import (
"context"
"net/url"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestChildEntry verifies anchor resolution against a directory URL.
func TestChildEntry(t *testing.T) {
base, _ := url.Parse("http://example.com/repos/")
cases := []struct {
href string
want string
isDir bool
}{
{"el9/", "el9", true},
{"file.rpm", "file.rpm", false},
{"/repos/el8/", "el8", true},
{"../", "", false},
{"?C=N;O=D", "", false},
{"http://other.example.com/repos/x/", "", false},
{"/outside/", "", false},
{"deep/nested/", "", false},
{"#anchor", "", false},
}
for _, c := range cases {
got, isDir := childEntry(base, c.href)
if got != c.want || isDir != c.isDir {
t.Errorf("childEntry(%q) = %q,%v, want %q,%v", c.href, got, isDir, c.want, c.isDir)
}
}
}
// discoverOpts builds the options a discovery test crawls with.
func discoverOpts(typ RepoType, depth int) *Options {
return &Options{Type: typ, DiscoverDepth: depth}
}
// repoURLs reduces a crawl's repositories to their URLs, for comparison
// against what a fixture tree publishes.
func repoURLs(found *discovery) []string {
urls := make([]string, 0, len(found.repos))
for _, repo := range found.repos {
urls = append(urls, repo.url)
}
return urls
}
// checkURLs verifies a crawl found exactly the expected repositories, in
// order.
func checkURLs(t *testing.T, found *discovery, want []string) {
t.Helper()
got := repoURLs(found)
if len(got) != len(want) {
t.Fatalf("found %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Errorf("found[%d] = %q, want %q", i, got[i], want[i])
}
}
}
// TestDiscoverRepos verifies RPM repositories are found through directory
// listings and that the depth limit is honored.
func TestDiscoverRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "a", "repo1"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "b", "nested", "repo2"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "deep", "x", "y", "z", "repo3"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 3))
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/a/repo1", srv.URL + "/b/nested/repo2"})
// A deeper limit reaches the third repository.
found, err = discoverRepos(context.Background(), srv.URL, discoverOpts(RepoRPM, 6))
if err != nil {
t.Fatal(err)
}
if len(found.repos) != 3 {
t.Errorf("depth 6 found %v, want 3 repositories", repoURLs(found))
}
}
// TestDiscoverArchRepos verifies pacman repositories are recognized by
// their database files.
func TestDiscoverArchRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
testrepos.BuildArchRepo(t, filepath.Join(www, "extra", "os", "x86_64"), "extra")
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoArch, 4))
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/core/os/x86_64", srv.URL + "/extra/os/x86_64"})
}
// TestDiscoverApkRepos verifies Alpine repositories are recognized by
// their index files, matching the arch directories under a release tree.
func TestDiscoverApkRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "x86_64"))
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "aarch64"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL+"/v3.24/community", discoverOpts(RepoApk, 1))
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/v3.24/community/aarch64", srv.URL + "/v3.24/community/x86_64"})
}
// TestDiscoverDebRepos verifies suite directories are recognized by their
// release files.
func TestDiscoverDebRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, discoverOpts(RepoDeb, 4))
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/debian/dists/test", srv.URL + "/flat"})
}
// TestDiscoverAllTypes verifies a crawl that names no type finds every
// format under one URL, as a vendor archive publishing both yum and apt
// repositories requires.
func TestDiscoverAllTypes(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
testrepos.BuildDebRepo(t, filepath.Join(www, "apt"))
testrepos.BuildArchRepo(t, filepath.Join(www, "arch", "os", "x86_64"), "core")
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "x86_64"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, &Options{DiscoverDepth: 4})
if err != nil {
t.Fatal(err)
}
want := map[string]RepoType{
srv.URL + "/alpine/v3.24/x86_64": RepoApk,
srv.URL + "/apt/dists/test": RepoDeb,
srv.URL + "/arch/os/x86_64": RepoArch,
srv.URL + "/yum/el9": RepoRPM,
}
if len(found.repos) != len(want) {
t.Fatalf("found %v, want %v", found.repos, want)
}
for _, repo := range found.repos {
if want[repo.url] != repo.typ {
t.Errorf("%s discovered as %q, want %q", repo.url, repo.typ, want[repo.url])
}
}
// Limiting the crawl to one format leaves the rest of the tree alone.
found, err = discoverRepos(context.Background(), srv.URL, &Options{Types: []RepoType{RepoRPM}, DiscoverDepth: 4})
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/yum/el9"})
}
// TestDiscoverExclude verifies excluded directories are neither crawled nor
// reported, by name anywhere in the tree or by path below the crawled URL.
func TestDiscoverExclude(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el", "9"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles", "15"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "fc", "40"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "docker", "el", "9"))
srv := testrepos.ServeDir(t, www)
opts := discoverOpts(RepoRPM, 4)
opts.Exclude = []string{"sles", "fc", "docker"}
found, err := discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/yum/el/9"})
// A pattern with a slash pins the exclusion to one place in the tree.
opts.Exclude = []string{"yum/*"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/docker/el/9"})
}
// TestDiscoverIncludeFiles verifies loose files outside repositories are
// matched by the include patterns, and that files inside a repository are
// left to that repository's own synchronization.
func TestDiscoverIncludeFiles(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-mysql-2023"), []byte("key"))
testrepos.WriteFile(t, filepath.Join(www, "README.txt"), []byte("notes"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "el9", "extra.rpm"), []byte("rpm"))
srv := testrepos.ServeDir(t, www)
opts := discoverOpts(RepoRPM, 4)
opts.IncludeFiles = []string{"*.rpm", "RPM-GPG-KEY-*"}
found, err := discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, []string{srv.URL + "/yum/el9"})
want := []fileTarget{
{dirURL: srv.URL, name: "RPM-GPG-KEY-mysql-2023"},
{dirURL: srv.URL + "/yum", name: "loose-1.0.rpm"},
}
if len(found.files) != len(want) {
t.Fatalf("matched %v, want %v", found.files, want)
}
for i := range want {
if found.files[i] != want[i] {
t.Errorf("files[%d] = %v, want %v", i, found.files[i], want[i])
}
}
// An exclusion wins over an include pattern.
opts.Exclude = []string{"RPM-GPG-KEY-*"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
if len(found.files) != 1 || found.files[0].name != "loose-1.0.rpm" {
t.Errorf("matched %v, want only the loose rpm", found.files)
}
}
// TestDiscoverAnchoredPatterns verifies a leading slash limits a pattern to
// the crawled directory itself, for both include and exclude patterns.
func TestDiscoverAnchoredPatterns(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
testrepos.WriteFile(t, filepath.Join(www, "root-1.0.rpm"), []byte("rpm"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("rpm"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "deep", "buried-1.0.rpm"), []byte("rpm"))
srv := testrepos.ServeDir(t, www)
// Anchored, only the root directory's file is mirrored.
opts := discoverOpts(RepoRPM, 4)
opts.IncludeFiles = []string{"/*.rpm"}
found, err := discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
want := []fileTarget{{dirURL: srv.URL, name: "root-1.0.rpm"}}
if len(found.files) != len(want) || found.files[0] != want[0] {
t.Errorf("anchored include matched %v, want %v", found.files, want)
}
// The same pattern unanchored matches at every depth, which is what
// makes the anchor worth having.
opts.IncludeFiles = []string{"*.rpm"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
if len(found.files) != 3 {
t.Errorf("unanchored include matched %v, want 3 files", found.files)
}
// An anchored exclusion applies to the crawled directory only, leaving
// the same name deeper in the tree alone.
opts.Exclude = []string{"/root-1.0.rpm"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
if len(found.files) != 2 {
t.Errorf("anchored exclude left %v, want 2 files", found.files)
}
for _, file := range found.files {
if file.name == "root-1.0.rpm" {
t.Errorf("anchored exclude kept %v", file)
}
}
// An anchored directory exclusion keeps the crawl out of a top-level
// directory without matching that name elsewhere.
opts.Exclude = []string{"/deep"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
if len(found.files) != 3 {
t.Errorf("top-level exclude of an unrelated name left %v, want 3 files", found.files)
}
opts.Exclude = []string{"/yum"}
found, err = discoverRepos(context.Background(), srv.URL, opts)
if err != nil {
t.Fatal(err)
}
checkURLs(t, found, nil)
if len(found.files) != 1 || found.files[0].name != "root-1.0.rpm" {
t.Errorf("anchored directory exclude left %v, want only the root rpm", found.files)
}
}
// TestMatchSubject verifies the pairing of a pattern with the string it is
// matched against, which decides whether a pattern is anchored, pinned by
// path, or matched by name anywhere in the tree.
func TestMatchSubject(t *testing.T) {
cases := []struct {
pattern string
wantPattern string
wantSubject string
}{
{"*.rpm", "*.rpm", "pkg.rpm"},
{"/*.rpm", "*.rpm", "yum/deep/pkg.rpm"},
{"yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"},
{"/yum/*.rpm", "yum/*.rpm", "yum/deep/pkg.rpm"},
}
for _, c := range cases {
pattern, subject := matchSubject(c.pattern, "pkg.rpm", "yum/deep/pkg.rpm")
if pattern != c.wantPattern || subject != c.wantSubject {
t.Errorf("matchSubject(%q) = %q,%q, want %q,%q",
c.pattern, pattern, subject, c.wantPattern, c.wantSubject)
}
}
}
// TestDetectType verifies a repository URL given without a type is
// identified by its metadata, and that an unidentifiable URL is reported.
func TestDetectType(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine"))
srv := testrepos.ServeDir(t, www)
cases := []struct {
path string
want RepoType
}{
{"/el9", RepoRPM},
{"/debian/dists/test", RepoDeb},
{"/core/os/x86_64", RepoArch},
{"/alpine", RepoApk},
}
for _, c := range cases {
got, err := detectType(context.Background(), srv.URL+c.path, AllTypes)
if err != nil {
t.Errorf("detectType(%s): %v", c.path, err)
continue
}
if got != c.want {
t.Errorf("detectType(%s) = %q, want %q", c.path, got, c.want)
}
}
// A URL holding no repository metadata cannot be identified.
if _, err := detectType(context.Background(), srv.URL+"/debian", AllTypes); err == nil {
t.Error("expected an error detecting the type of an archive root")
}
}

14
mirror/mirror_test.go Normal file
View file

@ -0,0 +1,14 @@
package mirror
import (
"os"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
)
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
fetch.Reload()
os.Exit(m.Run())
}

157
mirror/options.go Normal file
View file

@ -0,0 +1,157 @@
// Package mirror synchronizes remote package repositories into a local
// directory tree, preserving the upstream layout so the result can be
// served directly to package managers.
package mirror
import (
"net/url"
"path"
"strings"
"time"
"github.com/grmrgecko/repo-sync/fetch"
)
// RepoType identifies the repository format being synchronized.
type RepoType string
const (
// RepoRPM synchronizes yum/dnf style repositories.
RepoRPM RepoType = "rpm"
// RepoDeb synchronizes apt style repositories.
RepoDeb RepoType = "deb"
// RepoArch synchronizes pacman style repositories.
RepoArch RepoType = "arch"
// RepoApk synchronizes Alpine apk style repositories.
RepoApk RepoType = "apk"
)
// AllTypes lists every supported repository format, in the order a
// directory's metadata is checked against them. RPM and deb come first as
// their markers are the most specific.
var AllTypes = []RepoType{RepoRPM, RepoDeb, RepoArch, RepoApk}
// TypeNames lists the supported format names, for user facing messages.
func TypeNames() []string {
names := make([]string, 0, len(AllTypes))
for _, typ := range AllTypes {
names = append(names, string(typ))
}
return names
}
// Options configures a synchronization run.
type Options struct {
// Type selects the repository format. It is empty for a run whose
// repositories are identified by their metadata rather than named on the
// command line, and is pinned to the repository's own format before that
// repository is synchronized.
Type RepoType
// Types limits the formats a run identifies repositories as. An empty
// list accepts every supported format.
Types []RepoType
// URLs lists the repository or mirrorlist URLs to synchronize.
URLs []string
// Destination is the local directory repositories are placed under.
Destination string
// Trim drops this many leading URL path components when building the
// destination path.
Trim int
// Flat places repositories directly in Destination without copying the
// URL path.
Flat bool
// Discover crawls each URL's directory listings for repositories.
Discover bool
// DiscoverDepth limits how deep discovery crawls below each URL.
DiscoverDepth int
// Exclude lists globs matched against the names of entries found while
// crawling; a match is neither crawled nor mirrored. A pattern
// containing a slash is matched against the entry's path below the
// crawled URL instead of its name, and a leading slash anchors it to
// that URL.
Exclude []string
// IncludeFiles lists globs matched against the names of files found
// outside of any repository while crawling; a match is mirrored as a
// plain file. Patterns match by path the same way Exclude does, so a
// leading slash limits one to the crawled directory itself. Files
// inside a repository come from its metadata and are not matched here.
IncludeFiles []string
// Workers is the number of concurrent download workers.
Workers int
// Verify re-verifies checksums of existing local files.
Verify bool
// Prune deletes local files no longer part of the repository.
Prune bool
// PruneGrace defers pruning a file until it has been unreferenced for
// this long across runs.
PruneGrace time.Duration
// DryRun reports what would change without downloading packages or
// altering the published repository.
DryRun bool
// Missing selects what happens when the upstream does not serve a
// package the repository metadata lists. The zero value is the strict
// mode, failing the repository at the first such file.
Missing fetch.MissingMode
// MissingRetries is how many consecutive runs must miss a file before
// the retry mode accepts it as gone and stops failing the run over it.
MissingRetries int
// Trace writes a trace file describing this mirror into every
// repository synchronized, and mirrors the traces upstream publishes
// beside it.
Trace bool
// TraceInfo describes the mirror recorded in that trace file.
TraceInfo TraceInfo
// destBase, when set by the mirror server, pins the repository root
// directory directly instead of deriving it from the URL path.
destBase string
// Components limits deb synchronization to these components.
Components []string
// Architectures limits deb synchronization to these architectures.
Architectures []string
}
// discoverTypes returns the formats a run identifies repositories as: the
// formats it was limited to, the single format it was given, or every
// supported format when it named none.
func (o *Options) discoverTypes() []RepoType {
if len(o.Types) > 0 {
return o.Types
}
if o.Type != "" {
return []RepoType{o.Type}
}
return AllTypes
}
// newMissing starts tracking the packages an upstream does not serve for
// one repository, recording them in dir. That is the repository's own
// directory, and for apt the suite directory rather than the archive root,
// so suites sharing a pool keep their own records.
func (o *Options) newMissing(dir string) *fetch.Missing {
return fetch.NewMissing(dir, o.Missing, o.MissingRetries, o.DryRun)
}
// repoDest maps a repository URL onto the destination directory according
// to the configured trimming rules.
func (o *Options) repoDest(repoURL string) (string, error) {
if o.destBase != "" {
return o.destBase, nil
}
if o.Flat {
return o.Destination, nil
}
u, err := url.Parse(repoURL)
if err != nil {
return "", err
}
var segs []string
for _, seg := range strings.Split(u.Path, "/") {
if seg != "" {
segs = append(segs, seg)
}
}
if o.Trim >= len(segs) {
return o.Destination, nil
}
return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...))
}

31
mirror/options_test.go Normal file
View file

@ -0,0 +1,31 @@
package mirror
import (
"path/filepath"
"testing"
)
// TestRepoDest verifies URL path copying, trimming, and flat placement.
func TestRepoDest(t *testing.T) {
dest := t.TempDir()
repoURL := "https://example.com/repos/CentOS/7/EA4/"
cases := []struct {
name string
opts Options
want string
}{
{"default", Options{Destination: dest}, filepath.Join(dest, "repos", "CentOS", "7", "EA4")},
{"trim2", Options{Destination: dest, Trim: 2}, filepath.Join(dest, "7", "EA4")},
{"trim-all", Options{Destination: dest, Trim: 10}, dest},
{"flat", Options{Destination: dest, Flat: true}, dest},
}
for _, c := range cases {
got, err := c.opts.repoDest(repoURL)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if got != c.want {
t.Errorf("%s: got %q, want %q", c.name, got, c.want)
}
}
}

126
mirror/resolve.go Normal file
View file

@ -0,0 +1,126 @@
package mirror
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// probePaths lists the metadata files that identify a repository of a type.
// Types without fixed-name metadata return nothing.
func probePaths(typ RepoType) []string {
switch typ {
case RepoRPM:
return []string{"repodata/repomd.xml"}
case RepoDeb:
return []string{"InRelease", "Release"}
case RepoApk:
return []string{APKIndexName}
}
return nil
}
// detectType identifies the format of a repository URL for a run that did
// not name one. The directory listing is consulted first, as a pacman
// database is named after its repository and has no fixed path to probe;
// the formats that do have one are then probed directly, so a URL whose
// upstream serves no listing is still identified. Only the formats the run
// accepts are considered.
func detectType(ctx context.Context, repoURL string, types []RepoType) (RepoType, error) {
// A URL carrying a query or fragment is a mirrorlist or metalink
// endpoint rather than a directory, and nothing can be joined onto it.
if hasQueryOrFragment(repoURL) {
return "", fmt.Errorf("cannot determine the repository type of %s; name it with --type", repoURL)
}
if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil {
if typ, ok := repoTypeOf(listing, types); ok {
return typ, nil
}
}
src := fetch.NewSource([]string{repoURL})
for _, typ := range types {
for _, probe := range probePaths(typ) {
resp, err := src.Get(ctx, probe, fetch.GetOptions{})
if err == nil {
resp.Body.Close()
return typ, nil
}
if !errors.Is(err, fetch.ErrNotFound) {
return "", err
}
}
}
return "", fmt.Errorf("cannot determine the repository type of %s; name it with --type", repoURL)
}
// hasQueryOrFragment reports whether a URL carries a query or fragment,
// which makes it unusable as a base for joining repository paths onto.
// Unparsable URLs are reported as plain so the caller still probes them.
func hasQueryOrFragment(rawURL string) bool {
u, err := url.Parse(rawURL)
if err != nil {
return false
}
return u.RawQuery != "" || u.Fragment != ""
}
// resolveSource decides whether repoURL is a repository root or a mirror
// list and returns the source to fetch from, along with the URL whose path
// reflects the repository layout: repoURL itself for a direct repository,
// or the first mirror's URL when repoURL is a mirror list. RPM mirror lists
// may be plain text or metalink documents.
func resolveSource(ctx context.Context, repoURL string, typ RepoType) (*fetch.Source, string, error) {
// Pacman databases are named after the repository, so there is no
// fixed path to probe and pacman mirrorlists are config snippets
// rather than URL lists; the URL is used directly.
direct := fetch.NewSource([]string{repoURL})
probes := probePaths(typ)
if len(probes) == 0 {
return direct, repoURL, nil
}
// Probe for repository metadata directly under the URL. A URL carrying a
// query or fragment cannot have a repository path joined onto it, as the
// path would land inside the query, so skip the probe: such URLs are
// mirrorlist or metalink endpoints and are read as one below.
if !hasQueryOrFragment(repoURL) {
for _, probe := range probes {
resp, err := direct.Get(ctx, probe, fetch.GetOptions{})
if err == nil {
resp.Body.Close()
return direct, repoURL, nil
}
if !errors.Is(err, fetch.ErrNotFound) {
return nil, "", err
}
}
}
// Fall back to reading the URL itself as a mirror list.
body, err := fetch.ReadURL(ctx, repoURL)
if err != nil {
return nil, "", fmt.Errorf("read mirrorlist %s: %w", repoURL, err)
}
kind := "mirrorlist"
var bases []string
if typ == RepoRPM {
if bases = fetch.ParseMetalink(body); len(bases) > 0 {
kind = "metalink"
}
}
if len(bases) == 0 {
bases = fetch.ParseMirrorlist(body)
}
if len(bases) == 0 {
return nil, "", fmt.Errorf("%s is neither a repository nor a mirrorlist", repoURL)
}
log.WithFields(log.Fields{"url": repoURL, "mirrors": len(bases), "kind": kind}).Info("Using mirror list for repository.")
return fetch.NewSource(bases), bases[0], nil
}

123
mirror/resolve_test.go Normal file
View file

@ -0,0 +1,123 @@
package mirror
import (
"context"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestResolveSource verifies direct repositories are used as-is and
// mirrorlist bodies expand into multi-mirror sources.
func TestResolveSource(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
// The mirrorlist needs the real server URL, which is only known now
// that the listener is up; files are read per-request.
testrepos.WriteFile(t, filepath.Join(www, "ml.txt"),
[]byte("http://127.0.0.1:1/el9\n"+srv.URL+"/el9\n"))
direct, layoutURL, err := resolveSource(context.Background(), srv.URL+"/el9", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(direct.Bases()) != 1 {
t.Errorf("direct bases = %v", direct.Bases())
}
if layoutURL != srv.URL+"/el9" {
t.Errorf("direct layout URL = %q", layoutURL)
}
ml, layoutURL, err := resolveSource(context.Background(), srv.URL+"/ml.txt", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(ml.Bases()) != 2 {
t.Errorf("mirrorlist bases = %v", ml.Bases())
}
if layoutURL != "http://127.0.0.1:1/el9" {
t.Errorf("mirrorlist layout URL = %q", layoutURL)
}
if _, _, err := resolveSource(context.Background(), srv.URL+"/el9/Packages", RepoRPM); err == nil {
t.Error("expected error for a non-repository URL")
}
}
// TestResolveSourceQueryMirrorlist verifies a mirrorlist URL carrying a
// query string is read as a mirrorlist rather than probed as a repository.
// A repository path cannot be joined onto such a URL: it would land inside
// the query, and endpoints that answer 200 with an error page for an
// unrecognized parameter would then look like a repository.
func TestResolveSourceQueryMirrorlist(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
repo := testrepos.ServeDir(t, www)
var probed []string
list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
probed = append(probed, r.URL.String())
if r.URL.Query().Get("repo") != "os" {
// Several real mirrorlist services answer 200 with a message.
w.Write([]byte("Invalid release/repo/arch combination\n"))
return
}
w.Write([]byte(repo.URL + "/el9\n"))
}))
t.Cleanup(list.Close)
src, layoutURL, err := resolveSource(context.Background(), list.URL+"/?release=9&repo=os", RepoRPM)
if err != nil {
t.Fatal(err)
}
if bases := src.Bases(); len(bases) != 1 || bases[0] != repo.URL+"/el9" {
t.Errorf("bases = %v, want [%s]", bases, repo.URL+"/el9")
}
if layoutURL != repo.URL+"/el9" {
t.Errorf("layout URL = %q, want %q", layoutURL, repo.URL+"/el9")
}
for _, u := range probed {
if strings.Contains(u, "repodata") {
t.Errorf("probe folded a repository path into the query: %s", u)
}
}
}
// TestResolveSourceMetalink verifies metalink documents expand into
// multi-mirror sources for RPM repositories.
func TestResolveSourceMetalink(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
metalink := `<?xml version="1.0"?><metalink><files><file name="repomd.xml"><resources>` +
`<url protocol="http">http://127.0.0.1:1/dead/repodata/repomd.xml</url>` +
`<url protocol="http">` + srv.URL + `/el9/repodata/repomd.xml</url>` +
`</resources></file></files></metalink>`
testrepos.WriteFile(t, filepath.Join(www, "metalink.xml"), []byte(metalink))
src, layoutURL, err := resolveSource(context.Background(), srv.URL+"/metalink.xml", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(src.Bases()) != 2 || src.Bases()[1] != srv.URL+"/el9" {
t.Errorf("bases = %v", src.Bases())
}
if layoutURL != "http://127.0.0.1:1/dead" {
t.Errorf("layout URL = %q", layoutURL)
}
// The dead first mirror must fail over to the live one.
resp, err := src.Get(context.Background(), "repodata/repomd.xml", fetch.GetOptions{})
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
}

338
mirror/rpm.go Normal file
View file

@ -0,0 +1,338 @@
package mirror
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"os"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// repoMD is the partial XML schema for a yum repomd.xml file. Only the
// fields the sync needs are decoded; unknown elements are ignored so
// upstream additions do not break parsing.
type repoMD struct {
Data []repoData `xml:"data"`
}
// repoData is one metadata file referenced from repomd.xml.
type repoData struct {
Type string `xml:"type,attr"`
Checksum repoChecksum `xml:"checksum"`
Location repoLocation `xml:"location"`
Size int64 `xml:"size"`
}
// repoChecksum is a checksum element with its algorithm type.
type repoChecksum struct {
Type string `xml:"type,attr"`
Value string `xml:",chardata"`
}
// repoLocation captures the relative href of a metadata or package file.
type repoLocation struct {
Href string `xml:"href,attr"`
}
// rpmPackage is the minimal package element decoded from primary metadata,
// carrying the location, checksum, and size needed to mirror the file.
type rpmPackage struct {
Checksum repoChecksum `xml:"checksum"`
Size rpmSize `xml:"size"`
Location repoLocation `xml:"location"`
}
// rpmSize carries the on-disk package size from primary metadata.
type rpmSize struct {
Package int64 `xml:"package,attr"`
}
// syncRPM synchronizes one RPM repository from src into destDir.
func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
miss := opts.newMissing(destDir)
tr := newTrace(opts)
ctx = tr.track(ctx)
// Stage repomd.xml so clients of a live tree keep a consistent view of
// the old repository until everything else is in place.
repomdDst, err := fetch.LocalJoin(destDir, "repodata/repomd.xml")
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, "repodata/repomd.xml", repomdDst, nil, true, false); err != nil {
return fmt.Errorf("fetch repomd.xml: %w", err)
}
md, err := readRepomd(fetch.StagedOrFinal(repomdDst))
if err != nil {
return err
}
// Download the metadata files referenced by repomd. Their names embed
// content hashes, so they can be written directly into place. A dry run
// only fetches the indexes that must be parsed, staged for discard.
var jobs, planned []fetch.Job
var primary *repoData
var deltaMDs []*repoData
var parseDsts []string
for i, data := range md.Data {
if data.Location.Href == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, data.Location.Href)
if err != nil {
return err
}
job := fetch.Job{
ReqPath: data.Location.Href,
Dst: dst,
Want: fetch.MakeExpect(data.Size, data.Checksum.Type, data.Checksum.Value),
}
parsed := false
switch data.Type {
case "primary":
primary = &md.Data[i]
parsed = true
case "prestodelta", "deltainfo":
deltaMDs = append(deltaMDs, &md.Data[i])
parsed = true
}
if opts.DryRun {
if !parsed {
planned = append(planned, job)
continue
}
job.Stage = true
parseDsts = append(parseDsts, dst)
}
jobs = append(jobs, job)
}
fetch.PlanJobs(planned, opts.Verify, keep)
if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, nil); err != nil {
return fmt.Errorf("fetch repository metadata: %w", err)
}
if primary == nil {
return errors.New("repository metadata lists no primary index")
}
// Collect package locations from the primary index, plus any delta
// packages referenced by prestodelta metadata.
primaryPath, err := fetch.LocalJoin(destDir, primary.Location.Href)
if err != nil {
return err
}
pkgs, err := readPrimary(fetch.StagedOrFinal(primaryPath))
if err != nil {
return err
}
jobs = jobs[:0]
for _, pkg := range pkgs {
if pkg.Location.Href == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, pkg.Location.Href)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{
ReqPath: pkg.Location.Href,
Dst: dst,
Want: fetch.MakeExpect(pkg.Size.Package, pkg.Checksum.Type, pkg.Checksum.Value),
})
}
for _, deltaMD := range deltaMDs {
deltaPath, err := fetch.LocalJoin(destDir, deltaMD.Location.Href)
if err != nil {
return err
}
deltas, err := readDeltas(fetch.StagedOrFinal(deltaPath))
if err != nil {
return err
}
for _, delta := range deltas {
if delta.Filename == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, delta.Filename)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{
ReqPath: delta.Filename,
Dst: dst,
Want: fetch.MakeExpect(delta.Size, delta.Checksum.Type, delta.Checksum.Value),
})
}
}
log.WithField("packages", len(jobs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// A dry run discards the indexes it staged for parsing.
for _, dst := range parseDsts {
_ = os.Remove(dst + fetch.StagedSuffix)
}
// Stage the optional signature material so it is published together
// with the repomd.xml it signs, never beside the previous index.
var sigDsts []string
for _, extra := range []string{"repodata/repomd.xml.asc", "repodata/repomd.xml.key"} {
dst, err := fetch.LocalJoin(destDir, extra)
if err != nil {
return err
}
_, err = fetch.File(ctx, src, extra, dst, nil, true, false)
switch {
case err == nil:
sigDsts = append(sigDsts, dst)
case errors.Is(err, fetch.ErrNotFound):
// The upstream dropped the signature; drop the local copy so
// a stale signature is never served beside a new repomd.xml.
if !opts.DryRun {
fetch.RemoveStale(dst)
}
default:
return fmt.Errorf("fetch %s: %w", extra, err)
}
}
// Promote the signatures and then repomd.xml so the published metadata
// chain is complete.
for _, dst := range sigDsts {
if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil {
return err
}
if _, err := os.Stat(dst); err == nil {
keep.Add(dst)
}
}
if err := fetch.PromoteOrDiscard(repomdDst, opts.DryRun); err != nil {
return err
}
keep.Add(repomdDst)
// Publish the traces, upstream's included, before pruning so the keep
// set covers them.
tr.publish(ctx, destDir, src, keep, opts)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
// The metadata is published either way; missing packages only decide
// whether the run reports itself as failed.
return miss.Finish()
}
// rpmDelta is one delta package referenced from prestodelta metadata.
type rpmDelta struct {
Filename string `xml:"filename"`
Size int64 `xml:"size"`
Checksum repoChecksum `xml:"checksum"`
}
// readDeltas streams prestodelta metadata and collects each delta package
// reference so drpm files are mirrored alongside full packages.
func readDeltas(filename string) ([]rpmDelta, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
dec := xml.NewDecoder(r)
var deltas []rpmDelta
for {
tok, err := dec.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse delta metadata %s: %w", filename, err)
}
start, ok := tok.(xml.StartElement)
if !ok || start.Name.Local != "delta" {
continue
}
var delta rpmDelta
if err := dec.DecodeElement(&delta, &start); err != nil {
return nil, fmt.Errorf("parse delta entry %s: %w", filename, err)
}
if delta.Filename != "" {
deltas = append(deltas, delta)
}
}
return deltas, nil
}
// readRepomd decodes a repomd.xml file.
func readRepomd(filename string) (*repoMD, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
var md repoMD
if err := xml.NewDecoder(f).Decode(&md); err != nil {
return nil, fmt.Errorf("parse repomd %s: %w", filename, err)
}
return &md, nil
}
// readPrimary streams a primary metadata index and collects each package
// entry without loading the whole document into memory.
func readPrimary(filename string) ([]rpmPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
dec := xml.NewDecoder(r)
var pkgs []rpmPackage
for {
tok, err := dec.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse primary metadata %s: %w", filename, err)
}
start, ok := tok.(xml.StartElement)
if !ok || start.Name.Local != "package" {
continue
}
var pkg rpmPackage
if err := dec.DecodeElement(&pkg, &start); err != nil {
return nil, fmt.Errorf("parse package entry %s: %w", filename, err)
}
if pkg.Location.Href != "" {
pkgs = append(pkgs, pkg)
}
}
return pkgs, nil
}

171
mirror/rpm_test.go Normal file
View file

@ -0,0 +1,171 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// syncRPMFixture builds a served RPM repository, synchronizes it, and
// returns the fixture directory, repository URL, and local repository path.
func syncRPMFixture(t *testing.T, opts *Options) (string, string, string, map[string][]byte) {
t.Helper()
www := t.TempDir()
repoDir := filepath.Join(www, "repos", "el9")
pkgs := testrepos.BuildRPMRepo(t, repoDir)
srv := testrepos.ServeDir(t, www)
repoURL := srv.URL + "/repos/el9"
opts.Type = RepoRPM
if opts.Destination == "" {
opts.Destination = t.TempDir()
}
if opts.Workers == 0 {
opts.Workers = 2
}
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
return repoDir, repoURL, filepath.Join(opts.Destination, "repos", "el9"), pkgs
}
// TestSyncRPM verifies a full synchronization mirrors packages and metadata
// with no staged leftovers.
func TestSyncRPM(t *testing.T) {
_, _, local, pkgs := syncRPMFixture(t, &Options{})
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, "Packages", name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml")); err != nil {
t.Error("repomd.xml not promoted:", err)
}
if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml.asc")); err != nil {
t.Error("repomd.xml.asc not promoted:", err)
}
drpm, err := os.ReadFile(filepath.Join(local, "drpms", "foo-0.9_1.0-1.x86_64.drpm"))
if err != nil {
t.Error("delta package not mirrored:", err)
} else if !bytes.Equal(drpm, testrepos.RPMDelta()) {
t.Error("delta package content mismatch")
}
staged, _ := filepath.Glob(filepath.Join(local, "repodata", "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncRPMVerify verifies size-only skipping keeps same-size corruption
// while the verify option repairs it.
func TestSyncRPMVerify(t *testing.T) {
opts := &Options{}
repoDir, repoURL, local, _ := syncRPMFixture(t, opts)
_ = repoDir
// Corrupt a local package without changing its size.
victim := filepath.Join(local, "Packages", "foo-1.0-1.x86_64.rpm")
corrupt := bytes.Repeat([]byte("oof"), 500)
if err := os.WriteFile(victim, corrupt, 0644); err != nil {
t.Fatal(err)
}
// A plain re-run trusts the matching size and keeps the corruption.
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(victim)
if !bytes.Equal(got, corrupt) {
t.Error("size-only run unexpectedly rewrote the file")
}
// A verify run re-hashes local files and repairs the corruption.
opts.Verify = true
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
got, _ = os.ReadFile(victim)
if !bytes.Equal(got, bytes.Repeat([]byte("foo"), 500)) {
t.Error("verify run did not repair the corruption")
}
}
// TestSyncRPMPrune verifies stale files are removed only when pruning is
// enabled.
func TestSyncRPMPrune(t *testing.T) {
opts := &Options{}
_, repoURL, local, _ := syncRPMFixture(t, opts)
stale := filepath.Join(local, "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Error("file pruned without the prune option")
}
opts.Prune = true
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale file survived pruning")
}
}
// TestSyncRPMDroppedSignature verifies a signature the upstream stops
// serving is removed locally instead of lingering beside a new repomd.xml.
func TestSyncRPMDroppedSignature(t *testing.T) {
opts := &Options{}
repoDir, repoURL, local, _ := syncRPMFixture(t, opts)
ascLocal := filepath.Join(local, "repodata", "repomd.xml.asc")
if _, err := os.Stat(ascLocal); err != nil {
t.Fatal("signature not mirrored:", err)
}
if err := os.Remove(filepath.Join(repoDir, "repodata", "repomd.xml.asc")); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(ascLocal); !os.IsNotExist(err) {
t.Error("stale signature survived after the upstream dropped it")
}
}
// TestSyncRPMChecksumMismatch verifies a package whose served content does
// not match the published checksum fails the synchronization.
func TestSyncRPMChecksumMismatch(t *testing.T) {
www := t.TempDir()
repoDir := filepath.Join(www, "repos", "el9")
testrepos.BuildRPMRepo(t, repoDir)
srv := testrepos.ServeDir(t, www)
// Replace a served package with same-size different content after the
// metadata was generated.
victim := filepath.Join(repoDir, "Packages", "foo-1.0-1.x86_64.rpm")
if err := os.WriteFile(victim, bytes.Repeat([]byte("fox"), 500), 0644); err != nil {
t.Fatal(err)
}
opts := &Options{Type: RepoRPM, Destination: t.TempDir(), Workers: 2}
err := syncOne(context.Background(), srv.URL+"/repos/el9", opts.Type, opts)
if err == nil || !strings.Contains(err.Error(), "checksum") {
t.Errorf("expected checksum mismatch error, got %v", err)
}
}

381
mirror/sync.go Normal file
View file

@ -0,0 +1,381 @@
package mirror
import (
"context"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"path/filepath"
"syscall"
"time"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// destLock is the exclusive hold a run has on its destination directory.
type destLock struct {
file *os.File
path string
}
// lockAttempts bounds the retries for a lock file removed between opening
// it and locking it. That window is a few instructions wide and only opens
// when another run finishes inside it, so a handful of attempts always
// resolves it.
const lockAttempts = 5
// lockDestination takes an exclusive lock under the destination directory
// so overlapping runs, such as from cron, cannot race on the same tree.
// The caller releases it with release.
func lockDestination(dest string) (*destLock, error) {
if err := os.MkdirAll(dest, 0755); err != nil {
return nil, err
}
name := filepath.Join(dest, fetch.LockFileName)
for attempt := 0; attempt < lockAttempts; attempt++ {
f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return nil, err
}
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
f.Close()
return nil, fmt.Errorf("destination %s is locked by another repo-sync run", dest)
}
// The lock only guards the tree while the file holding it is still
// the one at the path. A run that finished between the open and the
// lock above took its file with it, so this lock is on a file the
// next run will never look at; open the replacement and lock that.
held, herr := f.Stat()
current, cerr := os.Stat(name)
if herr == nil && cerr == nil && os.SameFile(held, current) {
return &destLock{file: f, path: name}, nil
}
f.Close()
log.WithField("path", name).Debug("Destination lock file was replaced while locking; retrying.")
}
return nil, fmt.Errorf("failed to lock destination %s", dest)
}
// release removes the lock file and then drops the lock, in that order, so
// the file is never removed out from under whichever run takes it next. It
// leaves nothing behind in the published tree, including when a run ends
// early on an interrupt.
func (l *destLock) release() {
if err := os.Remove(l.path); err != nil && !errors.Is(err, fs.ErrNotExist) {
log.WithError(err).WithField("path", l.path).Warn("Failed to remove the destination lock file.")
}
l.file.Close()
}
// Summary reports what a whole run transferred, for callers that act on the
// result rather than read the log. A repository that failed part way through
// still counts what it transferred before failing, as those transfers changed
// the destination all the same.
type Summary struct {
Repositories int
Failed int
DryRun bool
Fetched int64
FetchedBytes int64
Unchanged int64
Pruned int64
Planned int64
PlannedBytes int64
// Missing counts the files repository metadata listed that the upstream
// did not serve, whether or not the run tolerated their absence.
Missing int64
}
// Changed reports whether the run altered the destination. A dry run reports
// what it would have done in the same counters, so it never counts as a
// change.
func (s *Summary) Changed() bool {
if s.DryRun {
return false
}
return s.Fetched > 0 || s.Pruned > 0
}
// add folds one repository's counters into the run totals.
func (s *Summary) add(c *fetch.Counters) {
s.Fetched += c.Fetched.Load()
s.FetchedBytes += c.FetchedBytes.Load()
s.Unchanged += c.Unchanged.Load()
s.Pruned += c.Pruned.Load()
s.Planned += c.Planned.Load()
s.PlannedBytes += c.PlannedBytes.Load()
s.Missing += c.Missing.Load()
}
// Sync synchronizes every configured repository URL into the destination,
// continuing after per-repository failures so one bad repository does not
// block the rest. The returned summary covers every repository reached,
// including on the error paths, so a caller can still tell whether a failed
// run changed anything.
func Sync(ctx context.Context, opts *Options) (*Summary, error) {
fetch.Reload()
summary := &Summary{DryRun: opts.DryRun}
// Hold the destination for the whole run, releasing it however the run
// ends so no lock file is left in the published tree.
lock, err := lockDestination(opts.Destination)
if err != nil {
return summary, err
}
defer lock.release()
// Resolve the configured URLs into the repositories, and the loose
// files, this run works on.
repos, files, err := resolveTargets(ctx, opts)
if err != nil {
return summary, err
}
if len(repos) == 0 && len(files) == 0 {
return summary, errors.New("no repositories to synchronize")
}
// Synchronize each repository in turn. Every repository's counters are
// folded into the run totals before the next one resets them, whatever
// its outcome.
for _, repo := range repos {
if err := ctx.Err(); err != nil {
return summary, err
}
log.WithFields(log.Fields{"url": repo.url, "type": repo.typ}).Info("Synchronizing repository.")
fetch.Stats.Reset()
start := time.Now()
err := syncOne(ctx, repo.url, repo.typ, opts)
summary.Repositories++
summary.add(fetch.Stats)
if err != nil {
summary.Failed++
if errors.Is(err, context.Canceled) {
return summary, err
}
log.WithError(err).WithField("url", repo.url).Error("Repository synchronization failed.")
continue
}
logSummary(repo.url, opts, time.Since(start))
}
// Mirror the loose files last: a repository sharing a directory with
// them prunes against its own metadata, so files fetched before it ran
// would be removed as stale.
var fileErr error
if len(files) > 0 && ctx.Err() == nil {
log.WithField("files", len(files)).Info("Synchronizing files.")
fetch.Stats.Reset()
fileErr = syncFiles(ctx, files, opts)
summary.add(fetch.Stats)
if errors.Is(fileErr, context.Canceled) {
return summary, fileErr
}
if fileErr != nil {
log.WithError(fileErr).Error("File synchronization failed.")
}
}
var errs []error
if summary.Failed > 0 {
errs = append(errs, fmt.Errorf("%d of %d repositories failed to synchronize", summary.Failed, len(repos)))
}
if fileErr != nil {
errs = append(errs, fileErr)
}
return summary, errors.Join(errs...)
}
// resolveTargets expands the configured URLs into the repositories to
// synchronize and the loose files to mirror beside them. Discovery crawls
// for both; without it each URL is a repository, whose format is detected
// when the run did not name one.
func resolveTargets(ctx context.Context, opts *Options) ([]repoTarget, []fileTarget, error) {
if opts.Discover {
var repos []repoTarget
var files []fileTarget
for _, base := range opts.URLs {
found, err := discoverRepos(ctx, base, opts)
if err != nil {
return nil, nil, fmt.Errorf("discover repositories under %s: %w", base, err)
}
log.WithFields(log.Fields{
"url": base,
"repositories": len(found.repos),
"files": len(found.files),
}).Info("Discovered repositories.")
repos = append(repos, found.repos...)
files = append(files, found.files...)
}
return repos, files, nil
}
repos := make([]repoTarget, 0, len(opts.URLs))
for _, repoURL := range opts.URLs {
typ := opts.Type
if typ == "" {
detected, err := detectType(ctx, repoURL, opts.discoverTypes())
if err != nil {
return nil, nil, err
}
log.WithFields(log.Fields{"url": repoURL, "type": detected}).Info("Detected repository type.")
typ = detected
}
repos = append(repos, repoTarget{url: repoURL, typ: typ})
}
return repos, nil, nil
}
// syncFiles mirrors the loose files a crawl matched. They sit outside any
// repository and carry no published checksum, so they revalidate against
// their modification time, and one withdrawn between the crawl and the
// fetch is not a failure. Nothing here prunes: the files are not described
// by any metadata that could say one no longer belongs.
func syncFiles(ctx context.Context, files []fileTarget, opts *Options) error {
// Group by directory so each group shares one source and its request
// paths stay relative to it, keeping the crawl's order.
var dirs []string
groups := map[string][]fileTarget{}
for _, file := range files {
if _, ok := groups[file.dirURL]; !ok {
dirs = append(dirs, file.dirURL)
}
groups[file.dirURL] = append(groups[file.dirURL], file)
}
for _, dirURL := range dirs {
if err := ctx.Err(); err != nil {
return err
}
destDir, err := opts.repoDest(dirURL)
if err != nil {
return err
}
src := fetch.NewSource([]string{dirURL})
var jobs []fetch.Job
for _, file := range groups[dirURL] {
dst, err := fetch.LocalJoin(destDir, file.name)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{ReqPath: url.PathEscape(file.name), Dst: dst, Optional: true})
}
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, nil)
continue
}
if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, nil, nil); err != nil {
return fmt.Errorf("fetch files under %s: %w", dirURL, err)
}
log.WithFields(log.Fields{"url": dirURL, "files": len(jobs), "destination": destDir}).Debug("Mirrored files.")
}
return nil
}
// logSummary reports one repository's transfer activity after a successful
// synchronization.
func logSummary(repoURL string, opts *Options, elapsed time.Duration) {
fields := log.Fields{
"url": repoURL,
"fetched": fetch.Stats.Fetched.Load(),
"transferred": fetch.FormatBytes(fetch.Stats.FetchedBytes.Load()),
"unchanged": fetch.Stats.Unchanged.Load(),
"duration": elapsed.Round(time.Millisecond).String(),
}
// Report tolerated absences, so a repository that synchronized despite
// them does not look completely clean in the log.
if missing := fetch.Stats.Missing.Load(); missing > 0 {
fields["missing"] = missing
}
if opts.DryRun {
fields["would_fetch"] = fetch.Stats.Planned.Load()
fields["would_transfer"] = fetch.FormatBytes(fetch.Stats.PlannedBytes.Load())
if opts.Prune {
fields["would_prune"] = fetch.Stats.Pruned.Load()
}
log.WithFields(fields).Info("Dry run complete.")
return
}
if opts.Prune {
fields["pruned"] = fetch.Stats.Pruned.Load()
}
log.WithFields(fields).Info("Repository synchronized.")
}
// syncOne resolves the source for a single repository URL and runs the
// format-specific synchronization. Destination paths derive from the layout
// URL, so a mirror list maps to the first mirror's path rather than the
// list's own URL.
func syncOne(ctx context.Context, repoURL string, typ RepoType, opts *Options) error {
// Pin this repository's format on a copy, so a run covering several
// formats reports each repository as what it is and never mutates the
// caller's options.
repoOpts := *opts
repoOpts.Type = typ
opts = &repoOpts
src, layoutURL, err := resolveSource(ctx, repoURL, typ)
if err != nil {
return err
}
switch typ {
case RepoRPM:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncRPM(ctx, src, destDir, opts)
case RepoDeb:
return syncDeb(ctx, src, layoutURL, opts)
case RepoArch:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncArch(ctx, src, layoutURL, destDir, opts)
case RepoApk:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncApk(ctx, src, destDir, opts)
default:
return fmt.Errorf("unsupported repository type %q", typ)
}
}
// SyncInto synchronizes one repository into an explicit destination
// directory, bypassing URL-derived destination mapping. It exists for the
// mirror server, whose destinations follow request paths rather than
// upstream URL paths. For deb the destination is the archive root
// directory; for other types it is the repository directory itself.
// Concurrent callers interleave into fetch.Stats, so the counters carry no
// per-repository meaning on this path and are neither reset nor reported.
// A trace enabled here still reports correct totals: it accumulates its own
// counters for the duration of the crawl rather than reading the shared
// collector.
func SyncInto(ctx context.Context, typ RepoType, src *fetch.Source, repoURL, dest string, opts *Options) error {
// Copy before the deb path pins destBase so the caller's Options is
// never mutated, keeping the struct safe to reuse or share.
serverOpts := *opts
opts = &serverOpts
switch typ {
case RepoRPM:
return syncRPM(ctx, src, dest, opts)
case RepoDeb:
opts.destBase = dest
return syncDeb(ctx, src, repoURL, opts)
case RepoArch:
return syncArch(ctx, src, repoURL, dest, opts)
case RepoApk:
return syncApk(ctx, src, dest, opts)
default:
return fmt.Errorf("unsupported repository type %q", typ)
}
}

416
mirror/sync_test.go Normal file
View file

@ -0,0 +1,416 @@
package mirror
import (
"context"
"encoding/json"
"io/fs"
"os"
"path/filepath"
"testing"
"time"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// countFiles returns the number of regular files under root.
func countFiles(t *testing.T, root string) int {
t.Helper()
count := 0
err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
count++
}
return nil
})
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
return count
}
// TestDryRun verifies a dry run leaves the destination unchanged: no
// packages downloaded, no metadata published, and no files pruned.
func TestDryRun(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
// A dry run against an empty destination writes no files at all.
dest := t.TempDir()
opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, DryRun: true, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
if n := countFiles(t, dest); n != 0 {
t.Errorf("dry run left %d files in the destination", n)
}
// A real run works, and a later dry run with pruning keeps stale files.
opts.DryRun = false
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "el9", "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
opts.DryRun = true
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Error("dry run pruned a file:", err)
}
}
// TestPruneGrace verifies stale files wait out the grace period across
// runs before being removed.
func TestPruneGrace(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, Prune: true, PruneGrace: time.Hour}
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "el9")
stale := filepath.Join(local, "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
// The first run only starts the grace period.
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Fatal("file pruned before its grace period expired:", err)
}
statePath := filepath.Join(local, fetch.PruneStateName)
if _, err := os.Stat(statePath); err != nil {
t.Fatal("prune state not recorded:", err)
}
// A second run within the grace period still keeps the file.
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Fatal("file pruned within its grace period:", err)
}
// Age the recorded timestamp past the grace period; the next run
// prunes the file and clears the bookkeeping.
data, err := os.ReadFile(statePath)
if err != nil {
t.Fatal(err)
}
state := map[string]time.Time{}
if err := json.Unmarshal(data, &state); err != nil {
t.Fatal(err)
}
for rel := range state {
state[rel] = time.Now().Add(-2 * time.Hour)
}
data, err = json.Marshal(state)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/el9", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale file survived past its grace period")
}
if _, err := os.Stat(statePath); err == nil {
t.Error("prune state lingered after all grace periods resolved")
}
}
// TestLockDestination verifies a held destination lock blocks a second run,
// is released on release, and leaves no lock file behind in the tree.
func TestLockDestination(t *testing.T) {
dest := t.TempDir()
lock, err := lockDestination(dest)
if err != nil {
t.Fatal(err)
}
name := filepath.Join(dest, fetch.LockFileName)
if _, err := os.Stat(name); err != nil {
t.Error("no lock file while the lock is held:", err)
}
if _, err := lockDestination(dest); err == nil {
t.Error("second lock unexpectedly succeeded")
}
lock.release()
if _, err := os.Stat(name); !os.IsNotExist(err) {
t.Error("lock file survived the run that took it")
}
lock, err = lockDestination(dest)
if err != nil {
t.Error("lock not released:", err)
} else {
lock.release()
}
}
// TestLockDestinationReplaced verifies a lock taken on a file that is no
// longer the one at the path is not accepted: a run that released between
// the open and the lock takes its file with it, and the lock left over
// guards nothing the next run would consult.
func TestLockDestinationReplaced(t *testing.T) {
dest := t.TempDir()
name := filepath.Join(dest, fetch.LockFileName)
// Stand in for the released run by replacing the lock file with a new
// one, which is the state a racing run would find.
stale, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
t.Fatal(err)
}
defer stale.Close()
if err := os.Remove(name); err != nil {
t.Fatal(err)
}
replacement, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
t.Fatal(err)
}
replacement.Close()
lock, err := lockDestination(dest)
if err != nil {
t.Fatal(err)
}
defer lock.release()
held, err := lock.file.Stat()
if err != nil {
t.Fatal(err)
}
current, err := os.Stat(name)
if err != nil {
t.Fatal(err)
}
if !os.SameFile(held, current) {
t.Error("locked a file that is no longer the destination's lock file")
}
}
// TestSyncSummary verifies a run reports what it changed: the first run
// fetches, a repeat run finds everything unchanged, and a removed upstream
// file is counted as pruned.
func TestSyncSummary(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{
Type: RepoRPM,
URLs: []string{srv.URL + "/el9"},
Destination: dest,
Workers: 2,
Prune: true,
}
// The first run populates the destination, so everything it fetched is a
// change.
summary, err := Sync(context.Background(), opts)
if err != nil {
t.Fatal(err)
}
if summary.Repositories != 1 || summary.Failed != 0 {
t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed)
}
if summary.Fetched == 0 {
t.Error("first run reported nothing fetched")
}
if summary.FetchedBytes == 0 {
t.Error("first run reported no bytes fetched")
}
if !summary.Changed() {
t.Error("first run did not report a change")
}
// Repeating the run transfers nothing, which is what a hook keys off to
// stay idle.
summary, err = Sync(context.Background(), opts)
if err != nil {
t.Fatal(err)
}
if summary.Fetched != 0 || summary.Pruned != 0 {
t.Errorf("repeat run fetched %d and pruned %d, want 0 and 0", summary.Fetched, summary.Pruned)
}
if summary.Unchanged == 0 {
t.Error("repeat run reported nothing unchanged")
}
if summary.Changed() {
t.Error("repeat run reported a change")
}
// A local file the upstream never served is pruned, which counts as a
// change even though nothing was fetched.
stale := filepath.Join(dest, "el9", "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
summary, err = Sync(context.Background(), opts)
if err != nil {
t.Fatal(err)
}
if summary.Pruned != 1 {
t.Errorf("pruned = %d, want 1", summary.Pruned)
}
if !summary.Changed() {
t.Error("pruning run did not report a change")
}
}
// TestSyncSummaryDryRun verifies a dry run reports its planned work without
// ever claiming the mirror changed.
func TestSyncSummaryDryRun(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
opts := &Options{
Type: RepoRPM,
URLs: []string{srv.URL + "/el9"},
Destination: t.TempDir(),
Workers: 2,
DryRun: true,
}
summary, err := Sync(context.Background(), opts)
if err != nil {
t.Fatal(err)
}
if summary.Planned == 0 {
t.Error("dry run planned nothing")
}
if summary.PlannedBytes == 0 {
t.Error("dry run planned no bytes")
}
// A dry run still fetches metadata to plan against, but none of it
// reaches the destination, so the run must never read as a change.
if summary.Changed() {
t.Error("dry run reported a change")
}
}
// TestSyncSummaryFailure verifies a run that fails still reports the totals
// it accumulated, so a caller can tell a failed run apart from an idle one.
func TestSyncSummaryFailure(t *testing.T) {
www := t.TempDir()
srv := testrepos.ServeDir(t, www)
opts := &Options{
Type: RepoRPM,
URLs: []string{srv.URL + "/missing"},
Destination: t.TempDir(),
Workers: 2,
}
summary, err := Sync(context.Background(), opts)
if err == nil {
t.Fatal("expected a failure syncing a repository that does not exist")
}
if summary == nil {
t.Fatal("no summary returned for a failed run")
}
if summary.Repositories != 1 || summary.Failed != 1 {
t.Errorf("repositories = %d, failed = %d, want 1 and 1", summary.Repositories, summary.Failed)
}
if summary.Changed() {
t.Error("a run that transferred nothing reported a change")
}
}
// TestSyncDiscoverMixed verifies one run covers an upstream publishing more
// than one repository format, skips the trees it was told to exclude, and
// mirrors the loose files it was told to include beside the repositories.
func TestSyncDiscoverMixed(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "el9"))
testrepos.BuildDebRepo(t, filepath.Join(www, "apt"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "yum", "sles15"))
testrepos.WriteFile(t, filepath.Join(www, "RPM-GPG-KEY-test"), []byte("key"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "loose-1.0.rpm"), []byte("loose"))
testrepos.WriteFile(t, filepath.Join(www, "yum", "notes.txt"), []byte("ignored"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{
URLs: []string{srv.URL},
Destination: dest,
Discover: true,
DiscoverDepth: 4,
Exclude: []string{"sles*"},
IncludeFiles: []string{"*.rpm", "RPM-GPG-KEY-*"},
Workers: 2,
}
summary, err := Sync(context.Background(), opts)
if err != nil {
t.Fatal(err)
}
if summary.Repositories != 2 || summary.Failed != 0 {
t.Errorf("repositories = %d, failed = %d, want 2 and 0", summary.Repositories, summary.Failed)
}
// Both formats landed, each under its own upstream path.
for _, rel := range []string{
filepath.Join("yum", "el9", "repodata", "repomd.xml"),
filepath.Join("apt", "dists", "test", "Release"),
filepath.Join("apt", "pool", "main", "h", "hello", "hello_1.0_amd64.deb"),
"RPM-GPG-KEY-test",
filepath.Join("yum", "loose-1.0.rpm"),
} {
if _, err := os.Stat(filepath.Join(dest, rel)); err != nil {
t.Errorf("missing %s: %v", rel, err)
}
}
// The excluded tree was never crawled, and a file no pattern matched
// was never fetched.
for _, rel := range []string{
filepath.Join("yum", "sles15"),
filepath.Join("yum", "notes.txt"),
} {
if _, err := os.Stat(filepath.Join(dest, rel)); err == nil {
t.Errorf("%s was synchronized despite not being asked for", rel)
}
}
}
// TestSyncDetectType verifies a repository URL given without a type is
// synchronized as whatever its metadata identifies it to be.
func TestSyncDetectType(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
summary, err := Sync(context.Background(), &Options{
URLs: []string{srv.URL + "/el9"},
Destination: dest,
Workers: 2,
})
if err != nil {
t.Fatal(err)
}
if summary.Repositories != 1 || summary.Failed != 0 {
t.Errorf("repositories = %d, failed = %d, want 1 and 0", summary.Repositories, summary.Failed)
}
if _, err := os.Stat(filepath.Join(dest, "el9", "repodata", "repomd.xml")); err != nil {
t.Error("repository not synchronized:", err)
}
}

275
mirror/trace.go Normal file
View file

@ -0,0 +1,275 @@
package mirror
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// TraceDir is the directory, relative to a repository, that trace files are
// published in. Debian archives established the location and downstream
// mirror checkers look for it there.
const TraceDir = "project/trace"
// TraceInfo describes the mirror a trace file is written for. It carries
// the same fields as the configuration's trace section, kept as a separate
// type so callers populate it from wherever they hold those details rather
// than this package taking a configuration struct as part of its API.
type TraceInfo struct {
Host string
Maintainer string
Sponsor string
Country string
Location string
Throughput string
}
// traceRun records when a repository's synchronization began, and the
// transfer it accounted for, so the trace can report both. A nil run writes
// nothing, which is how tracing stays off without a condition at every call
// site.
type traceRun struct {
started time.Time
stats *fetch.Counters
}
// newTrace starts timing a repository sync when tracing is enabled.
func newTrace(opts *Options) *traceRun {
if !opts.Trace {
return nil
}
return &traceRun{started: time.Now(), stats: &fetch.Counters{}}
}
// track installs the run's own transfer counters on ctx so the trace
// reports what this synchronization moved rather than what the shared
// collector holds. Without it the mirror server's concurrent crawls would
// report each other's bytes.
func (t *traceRun) track(ctx context.Context) context.Context {
if t == nil {
return ctx
}
return fetch.WithStats(ctx, t.stats)
}
// publish brings a repository's trace directory up to date: upstream's own
// traces are mirrored first, then this run's trace is written, so a name
// collision resolves in favour of the file describing this mirror.
func (t *traceRun) publish(ctx context.Context, root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) {
if t == nil {
return
}
t.mirrorUpstream(ctx, root, src, keep, opts)
t.write(root, src, keep, opts)
}
// mirrorUpstream copies the traces upstream publishes into this mirror's
// trace directory, so a reader can follow the chain back through every
// mirror to the archive the content came from. Upstream may publish no
// traces at all or may not serve directory indexes, and neither is a
// failure of the synchronization, so problems here are logged and the run
// continues.
func (t *traceRun) mirrorUpstream(ctx context.Context, root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) {
base := src.Active()
if base == "" {
return
}
dirURL := base + "/" + TraceDir + "/"
listing, err := listDir(ctx, dirURL)
if err != nil {
log.WithError(err).WithField("url", dirURL).Debug("Upstream serves no trace directory.")
return
}
// Skip the name this mirror traces under; write publishes that file
// itself, and upstream's copy of it describes a different mirror.
own, err := traceName(opts.TraceInfo.Host)
if err != nil {
own = ""
}
names := make([]string, 0, len(listing.files))
for name := range listing.files {
if name == own || name == "." || name == ".." {
continue
}
names = append(names, name)
}
sort.Strings(names)
dir := filepath.Join(root, filepath.FromSlash(TraceDir))
var jobs []fetch.Job
for _, name := range names {
dst, err := fetch.LocalJoin(dir, name)
if err != nil {
log.WithError(err).WithField("name", name).Debug("Skipping upstream trace with an unusable name.")
continue
}
// Traces carry no checksums, so they revalidate against their
// modification time rather than being re-fetched every run, and a
// trace withdrawn between the listing and the fetch is not an error.
jobs = append(jobs, fetch.Job{ReqPath: TraceDir + "/" + name, Dst: dst, Optional: true})
}
if len(jobs) == 0 {
return
}
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
return
}
if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, nil); err != nil {
log.WithError(err).WithField("url", dirURL).Warn("Failed to mirror upstream trace files.")
return
}
log.WithFields(log.Fields{"url": dirURL, "traces": len(jobs)}).Debug("Mirrored upstream trace files.")
}
// traceName returns the file name a mirror is traced under. The value
// becomes a file name, so it is reduced to a single path element and
// rejected if nothing usable is left.
func traceName(host string) (string, error) {
name := filepath.Base(strings.TrimSpace(strings.ReplaceAll(host, "\\", "/")))
if name == "." || name == ".." || name == "/" || name == "" {
return "", fmt.Errorf("invalid trace host %q", host)
}
return name, nil
}
// write publishes the trace for a finished repository synchronization into
// root, and records it in keep so pruning does not treat the file as stale
// and remove it. On a dry run the file is left untouched but still kept,
// so the run does not report the existing trace as about to be pruned.
//
// A trace describes the mirror rather than the packages, so a failure to
// write one is logged and does not fail the synchronization that produced
// it.
func (t *traceRun) write(root string, src *fetch.Source, keep *fetch.KeepSet, opts *Options) {
if t == nil {
return
}
name, err := traceName(opts.TraceInfo.Host)
if err != nil {
log.WithError(err).Warn("Skipping trace file.")
return
}
dir := filepath.Join(root, filepath.FromSlash(TraceDir))
file := filepath.Join(dir, name)
// Keep the trace regardless of whether it is written now, so a dry run
// does not report an existing trace as stale.
keep.Add(file)
if opts.DryRun {
return
}
if err := os.MkdirAll(dir, 0755); err != nil {
log.WithError(err).WithField("path", dir).Warn("Failed to create trace directory.")
return
}
content := t.render(src, opts)
// Write through a temporary file so a reader never sees a partial
// trace, and so an interrupted write cannot truncate the previous one.
tmp, err := os.CreateTemp(dir, ".trace-*")
if err != nil {
log.WithError(err).WithField("path", dir).Warn("Failed to create trace file.")
return
}
tmpName := tmp.Name()
if _, err := tmp.WriteString(content); err != nil {
tmp.Close()
os.Remove(tmpName)
log.WithError(err).WithField("path", tmpName).Warn("Failed to write trace file.")
return
}
if err := tmp.Close(); err != nil {
os.Remove(tmpName)
log.WithError(err).WithField("path", tmpName).Warn("Failed to write trace file.")
return
}
if err := os.Chmod(tmpName, 0644); err != nil {
log.WithError(err).WithField("path", tmpName).Debug("Failed to set trace file mode.")
}
if err := os.Rename(tmpName, file); err != nil {
os.Remove(tmpName)
log.WithError(err).WithField("path", file).Warn("Failed to publish trace file.")
return
}
log.WithField("path", file).Debug("Wrote trace file.")
}
// render builds the trace file's contents. The first line is the plain
// completion date, as the convention expects, followed by one field per
// line. Fields with no configured value are omitted rather than written
// empty.
func (t *traceRun) render(src *fetch.Source, opts *Options) string {
done := time.Now().UTC()
var b strings.Builder
// The opening line matches date(1)'s default UTC output, which is what
// the convention's readers expect; _2 space-pads the day as it does.
fmt.Fprintf(&b, "%s\n", done.Format("Mon Jan _2 15:04:05 UTC 2006"))
fmt.Fprintf(&b, "Date: %s\n", done.Format(time.RFC1123Z))
fmt.Fprintf(&b, "Date-Started: %s\n", t.started.UTC().Format(time.RFC1123Z))
fmt.Fprintf(&b, "Creator: %s %s\n", cfg.Name, cfg.Version)
fmt.Fprintf(&b, "Running on host: %s\n", opts.TraceInfo.Host)
for _, f := range []struct{ key, value string }{
{"Maintainer", opts.TraceInfo.Maintainer},
{"Sponsor", opts.TraceInfo.Sponsor},
{"Country", opts.TraceInfo.Country},
{"Location", opts.TraceInfo.Location},
{"Throughput", opts.TraceInfo.Throughput},
} {
if f.value != "" {
fmt.Fprintf(&b, "%s: %s\n", f.key, f.value)
}
}
fmt.Fprintf(&b, "Repository type: %s\n", opts.Type)
if arch := traceArchitectures(opts); arch != "" {
fmt.Fprintf(&b, "Architectures: %s\n", arch)
}
// Report the mirror actually fetched from. For a mirrorlist or metalink
// that is the mirror that answered, which is the useful fact and is not
// derivable from the configured URL.
if src != nil {
if active := src.Active(); active != "" {
fmt.Fprintf(&b, "Upstream-mirror: %s\n", active)
}
}
elapsed := done.Sub(t.started.UTC())
if elapsed < 0 {
elapsed = 0
}
seconds := int64(elapsed.Round(time.Second) / time.Second)
// The run's own counters, not the shared collector, so a trace written
// by one of the server's concurrent crawls reports that crawl alone.
bytes := t.stats.FetchedBytes.Load()
fmt.Fprintf(&b, "Total bytes received: %d\n", bytes)
fmt.Fprintf(&b, "Total time spent syncing: %d\n", seconds)
if seconds > 0 {
fmt.Fprintf(&b, "Average rate: %d B/s\n", bytes/seconds)
}
return b.String()
}
// traceArchitectures reports the architectures a run was restricted to,
// which only apt repositories can express. An unrestricted run mirrors
// whatever upstream publishes, so no claim is made.
func traceArchitectures(opts *Options) string {
if opts.Type != RepoDeb || len(opts.Architectures) == 0 {
return ""
}
arch := append([]string(nil), opts.Architectures...)
sort.Strings(arch)
return strings.Join(arch, " ")
}

453
mirror/trace_test.go Normal file
View file

@ -0,0 +1,453 @@
package mirror
import (
"context"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// traceOptions builds sync options with tracing enabled and a fixed set of
// mirror details, so tests assert on known values.
func traceOptions(typ RepoType, dest string) *Options {
return &Options{
Type: typ,
Destination: dest,
Workers: 2,
Trace: true,
TraceInfo: TraceInfo{
Host: "mirror.example.com",
Maintainer: "Test Maintainer <test@example.com>",
Sponsor: "Example Inc",
Country: "US",
Location: "Texas",
Throughput: "10Gb",
},
}
}
// traceField returns the value following a field prefix in a trace.
func traceField(t *testing.T, trace, prefix string) string {
t.Helper()
for _, line := range strings.Split(trace, "\n") {
if strings.HasPrefix(line, prefix) {
return strings.TrimPrefix(line, prefix)
}
}
t.Fatalf("trace has no %q field\ngot:\n%s", prefix, trace)
return ""
}
// readTrace returns the trace file published under root.
func readTrace(t *testing.T, root string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(TraceDir), "mirror.example.com"))
if err != nil {
t.Fatal("reading trace:", err)
}
return string(data)
}
// TestTraceName verifies a trace host is reduced to a single path element,
// so a configured value cannot direct the file outside the repository.
func TestTraceName(t *testing.T) {
cases := []struct {
host string
want string
ok bool
}{
{"mirror.example.com", "mirror.example.com", true},
{" mirror.example.com ", "mirror.example.com", true},
{"../../etc/cron.d/evil", "evil", true},
{"/etc/passwd", "passwd", true},
{`..\..\windows`, "windows", true},
{"sub/dir/host", "host", true},
{"", "", false},
{"..", "", false},
{"/", "", false},
}
for _, c := range cases {
got, err := traceName(c.host)
if c.ok && err != nil {
t.Errorf("traceName(%q) unexpected error: %v", c.host, err)
continue
}
if !c.ok {
if err == nil {
t.Errorf("traceName(%q) = %q, want error", c.host, got)
}
continue
}
if got != c.want {
t.Errorf("traceName(%q) = %q, want %q", c.host, got, c.want)
}
}
}
// TestSyncTraceWritten verifies a trace lands in the repository directory
// carrying the configured mirror details and the upstream actually used.
func TestSyncTraceWritten(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
repoURL := srv.URL + "/alpine/v3.24/community/x86_64"
if err := syncOne(context.Background(), repoURL, opts.Type, opts); err != nil {
t.Fatal(err)
}
trace := readTrace(t, filepath.Join(dest, "alpine", "v3.24", "community", "x86_64"))
for _, want := range []string{
"Date: ",
"Date-Started: ",
"Creator: repo-sync ",
"Running on host: mirror.example.com",
"Maintainer: Test Maintainer <test@example.com>",
"Sponsor: Example Inc",
"Country: US",
"Location: Texas",
"Throughput: 10Gb",
"Repository type: apk",
"Upstream-mirror: " + repoURL,
"Total bytes received: ",
"Total time spent syncing: ",
} {
if !strings.Contains(trace, want) {
t.Errorf("trace missing %q\ngot:\n%s", want, trace)
}
}
// The first line is the plain completion date the convention expects,
// carrying no field name of its own and matching date(1)'s UTC output.
first, _, _ := strings.Cut(trace, "\n")
if strings.HasPrefix(first, "Date:") {
t.Errorf("first line should be the bare date, got %q", first)
}
const dateLayout = "Mon Jan _2 15:04:05 UTC 2006"
parsed, err := time.Parse(dateLayout, first)
if err != nil {
t.Fatalf("first line %q is not a date(1)-style UTC date: %v", first, err)
}
// Parsing alone is too lenient about spacing to catch a malformed day
// field, so require the line to survive a round trip unchanged.
if canonical := parsed.Format(dateLayout); canonical != first {
t.Errorf("first line %q is not canonically formatted, want %q", first, canonical)
}
}
// TestSyncTraceOmitsUnsetFields verifies fields with no configured value
// are left out rather than written empty.
func TestSyncTraceOmitsUnsetFields(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{
Type: RepoApk, Destination: dest, Workers: 2, Trace: true,
TraceInfo: TraceInfo{Host: "mirror.example.com"},
}
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
trace := readTrace(t, filepath.Join(dest, "repo"))
for _, absent := range []string{"Maintainer:", "Sponsor:", "Country:", "Location:", "Throughput:"} {
if strings.Contains(trace, absent) {
t.Errorf("trace contains unset field %q\ngot:\n%s", absent, trace)
}
}
if !strings.Contains(trace, "Running on host: mirror.example.com") {
t.Errorf("trace missing host\ngot:\n%s", trace)
}
}
// TestSyncTraceDisabled verifies no trace directory appears when tracing
// is off, which is the default.
func TestSyncTraceDisabled(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dest, "repo", "project")); err == nil {
t.Error("trace written while tracing is disabled")
}
}
// TestSyncTraceSurvivesPrune verifies a trace is not treated as a stale
// file. A flat layout puts the trace inside the pruned tree itself, which
// is the case that would otherwise delete it on the following sync.
func TestSyncTraceSurvivesPrune(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
opts.Flat = true
opts.Prune = true
// Two syncs: the first writes the trace, the second prunes with that
// trace already on disk.
for i := 0; i < 2; i++ {
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatalf("sync %d: %v", i+1, err)
}
if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil {
t.Fatalf("trace missing after sync %d: %v", i+1, err)
}
}
// A stale file beside it is still pruned, confirming pruning ran.
stale := filepath.Join(dest, "old-1.0-r0.apk")
if err := os.WriteFile(stale, []byte("old"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale package survived pruning, so pruning did not run")
}
if _, err := os.Stat(filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil {
t.Error("trace pruned:", err)
}
}
// TestSyncTraceDryRun verifies a dry run neither writes a trace nor
// reports an existing one as a file it would prune.
func TestSyncTraceDryRun(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
opts.Flat = true
opts.DryRun = true
opts.Prune = true
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dest, "project")); err == nil {
t.Error("dry run wrote a trace")
}
// With a trace already on disk, a dry run must not count it as
// prunable. The file is planted directly so this stands on its own
// rather than depending on a preceding sync having written one.
traceFile := filepath.Join(dest, filepath.FromSlash(TraceDir), "mirror.example.com")
if err := os.MkdirAll(filepath.Dir(traceFile), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(traceFile, []byte("previous trace\n"), 0644); err != nil {
t.Fatal(err)
}
fetch.Stats.Reset()
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if n := fetch.Stats.Pruned.Load(); n != 0 {
t.Errorf("dry run would prune %d files, want 0 (the trace must be kept)", n)
}
if _, err := os.Stat(traceFile); err != nil {
t.Error("dry run removed the existing trace:", err)
}
}
// TestSyncTraceDebAtArchiveRoot verifies an apt trace is published at the
// archive root, where the convention places it, rather than in the suite.
func TestSyncTraceDebAtArchiveRoot(t *testing.T) {
www := t.TempDir()
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoDeb, dest)
opts.Architectures = []string{"amd64", "source"}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts.Type, opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "debian")
trace := readTrace(t, root)
if !strings.Contains(trace, "Repository type: deb") {
t.Errorf("trace missing repository type\ngot:\n%s", trace)
}
if !strings.Contains(trace, "Architectures: amd64 source") {
t.Errorf("trace missing architectures\ngot:\n%s", trace)
}
if _, err := os.Stat(filepath.Join(root, "dists", "test", "project")); err == nil {
t.Error("trace written into the suite directory instead of the archive root")
}
}
// TestSyncIntoWritesTrace verifies the mirror server's entry point
// publishes a trace like the sync commands do, and that it still leaves the
// caller's options untouched.
func TestSyncIntoWritesTrace(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
src := fetch.NewSource([]string{srv.URL + "/repo"})
if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil {
t.Fatal(err)
}
trace := readTrace(t, dest)
if !strings.Contains(trace, "Running on host: mirror.example.com") {
t.Errorf("server trace missing host\ngot:\n%s", trace)
}
if !opts.Trace {
t.Error("SyncInto mutated the caller's options")
}
}
// TestSyncIntoTraceDisabled verifies the server path honors tracing being
// off, which is the default.
func TestSyncIntoTraceDisabled(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2}
src := fetch.NewSource([]string{srv.URL + "/repo"})
if err := SyncInto(context.Background(), RepoApk, src, srv.URL+"/repo", dest, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dest, "project")); err == nil {
t.Error("trace written while tracing is disabled")
}
}
// TestSyncTraceReportsOwnTransfer verifies a trace reports the bytes its own
// synchronization moved rather than whatever the shared collector holds.
// That isolation is what makes a trace meaningful on the server's path,
// where crawls run concurrently and all report into the same collector.
func TestSyncTraceReportsOwnTransfer(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
// Stand in for a concurrent crawl's transfer, which must not be
// attributed to this run.
const foreign = int64(1) << 40
fetch.Stats.Reset()
fetch.Stats.FetchedBytes.Add(foreign)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
got := traceField(t, readTrace(t, filepath.Join(dest, "repo")), "Total bytes received: ")
bytes, err := strconv.ParseInt(got, 10, 64)
if err != nil {
t.Fatalf("byte count %q: %v", got, err)
}
if bytes <= 0 {
t.Errorf("trace reports %d bytes received, want the run's own transfer", bytes)
}
if bytes >= foreign {
t.Errorf("trace reports %d bytes received, which includes another run's transfer", bytes)
}
}
// TestSyncTraceMirrorsUpstream verifies upstream's own traces are copied
// into this mirror's trace directory, so the chain back to the origin
// archive is preserved, while the file naming this mirror stays the one
// this run wrote.
func TestSyncTraceMirrorsUpstream(t *testing.T) {
www := t.TempDir()
repo := filepath.Join(www, "repo")
testrepos.BuildApkRepo(t, repo)
upstreamTrace := filepath.Join(repo, filepath.FromSlash(TraceDir))
testrepos.WriteFile(t, filepath.Join(upstreamTrace, "origin.example.net"), []byte("origin trace\n"))
testrepos.WriteFile(t, filepath.Join(upstreamTrace, "middle.example.net"), []byte("middle trace\n"))
// Upstream also carries a trace under this mirror's own name, which
// describes a different mirror and must not replace ours.
testrepos.WriteFile(t, filepath.Join(upstreamTrace, "mirror.example.com"), []byte("someone else\n"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
opts.Prune = true
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
traceDir := filepath.Join(dest, "repo", filepath.FromSlash(TraceDir))
for name, want := range map[string]string{
"origin.example.net": "origin trace\n",
"middle.example.net": "middle trace\n",
} {
data, err := os.ReadFile(filepath.Join(traceDir, name))
if err != nil {
t.Errorf("upstream trace %s not mirrored: %v", name, err)
continue
}
if string(data) != want {
t.Errorf("upstream trace %s = %q, want %q", name, data, want)
}
}
if trace := readTrace(t, filepath.Join(dest, "repo")); !strings.Contains(trace, "Running on host: mirror.example.com") {
t.Errorf("upstream trace replaced this mirror's own\ngot:\n%s", trace)
}
// A second sync prunes with the mirrored traces already on disk; they
// belong to the repository and must survive.
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(traceDir, "origin.example.net")); err != nil {
t.Error("mirrored upstream trace pruned:", err)
}
}
// TestSyncTraceUpstreamAbsent verifies an upstream that publishes no traces
// is not a synchronization failure.
func TestSyncTraceUpstreamAbsent(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := traceOptions(RepoApk, dest)
if err := syncOne(context.Background(), srv.URL+"/repo", opts.Type, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dest, "repo", filepath.FromSlash(TraceDir), "mirror.example.com")); err != nil {
t.Error("trace missing:", err)
}
}
// TestTraceArchitectures verifies architectures are only claimed for apt
// runs that were actually restricted to them.
func TestTraceArchitectures(t *testing.T) {
if got := traceArchitectures(&Options{Type: RepoDeb, Architectures: []string{"source", "amd64"}}); got != "amd64 source" {
t.Errorf("deb architectures = %q, want %q", got, "amd64 source")
}
if got := traceArchitectures(&Options{Type: RepoDeb}); got != "" {
t.Errorf("unrestricted deb architectures = %q, want empty", got)
}
if got := traceArchitectures(&Options{Type: RepoRPM, Architectures: []string{"x86_64"}}); got != "" {
t.Errorf("rpm architectures = %q, want empty", got)
}
}

98
server/classify.go Normal file
View file

@ -0,0 +1,98 @@
package server
import (
"path"
"strings"
"github.com/grmrgecko/repo-sync/mirror"
)
// kindGeneric marks request paths that are not repository entry points.
const kindGeneric = "generic"
// resource is a crawlable unit derived from a request path.
type resource struct {
// Kind is a RepoType string, or kindGeneric.
Kind string
// Key uniquely identifies the resource in state and the lock map.
Key string
// Path is the repository request path (the suite directory for deb).
Path string
// Root is the request path prefix covering every file the resource
// owns; deb uses the archive root so pool requests match.
Root string
// ReqPath preserves the original request path.
ReqPath string
}
// classifyRequest maps a request path onto a crawlable resource. Only
// repository entry-point metadata registers a repository; every other path
// is generic and served on demand.
func classifyRequest(reqPath string) resource {
clean := path.Clean("/" + strings.TrimPrefix(reqPath, "/"))
base := path.Base(clean)
dir := path.Dir(clean)
// RPM repositories are identified by any repodata request.
if idx := strings.Index(clean, "/repodata/"); idx >= 0 {
root := clean[:idx]
if root == "" {
root = "/"
}
return resource{
Kind: string(mirror.RepoRPM),
Key: string(mirror.RepoRPM) + ":" + root,
Path: root,
Root: root,
ReqPath: clean,
}
}
// Apt suites are identified by their release files; the archive root
// above dists also covers the pool.
if base == "InRelease" || base == "Release" || base == "Release.gpg" {
root := dir
if idx := strings.Index(clean, "/dists/"); idx >= 0 {
root = clean[:idx]
if root == "" {
root = "/"
}
}
return resource{
Kind: string(mirror.RepoDeb),
Key: string(mirror.RepoDeb) + ":" + dir,
Path: dir,
Root: root,
ReqPath: clean,
}
}
// Pacman repositories are identified by their database file.
if strings.HasSuffix(base, ".db") {
return resource{
Kind: string(mirror.RepoArch),
Key: string(mirror.RepoArch) + ":" + dir,
Path: dir,
Root: dir,
ReqPath: clean,
}
}
// Alpine repositories are identified by their index archive.
if base == mirror.APKIndexName {
return resource{
Kind: string(mirror.RepoApk),
Key: string(mirror.RepoApk) + ":" + dir,
Path: dir,
Root: dir,
ReqPath: clean,
}
}
return resource{
Kind: kindGeneric,
Key: kindGeneric + ":" + clean,
Path: clean,
ReqPath: clean,
}
}

68
server/classify_test.go Normal file
View file

@ -0,0 +1,68 @@
package server
import (
"testing"
"time"
"github.com/grmrgecko/repo-sync/state"
)
// TestClassifyRequest verifies request paths map to the right resource
// kinds, crawl paths, and keep-alive roots.
func TestClassifyRequest(t *testing.T) {
cases := []struct {
reqPath string
kind string
path string
root string
}{
{"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"},
{"/almalinux/9/BaseOS/x86_64/os/repodata/abc-primary.xml.gz", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"},
{"/debian/dists/bookworm/InRelease", "deb", "/debian/dists/bookworm", "/debian"},
{"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"},
{"/flat/Release", "deb", "/flat", "/flat"},
{"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"},
{"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"},
{"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""},
{"/notes/README.txt", kindGeneric, "/notes/README.txt", ""},
}
for _, c := range cases {
res := classifyRequest(c.reqPath)
if res.Kind != c.kind || res.Path != c.path || res.Root != c.root {
t.Errorf("classifyRequest(%q) = {%s %s %s}, want {%s %s %s}",
c.reqPath, res.Kind, res.Path, res.Root, c.kind, c.path, c.root)
}
}
}
// TestRefreshTier verifies the tier walk and staleness budget.
func TestRefreshTier(t *testing.T) {
schedule := []time.Duration{4 * time.Hour, 12 * time.Hour, 24 * time.Hour}
if interval, stale := refreshTier(time.Hour, schedule); stale || interval != 4*time.Hour {
t.Errorf("fresh resource tier = %v, %v", interval, stale)
}
if interval, stale := refreshTier(10*time.Hour, schedule); stale || interval != 12*time.Hour {
t.Errorf("aging resource tier = %v, %v", interval, stale)
}
if _, stale := refreshTier(41*time.Hour, schedule); !stale {
t.Error("resource past the budget not stale")
}
}
// TestNextCrawlAt verifies crawl gating for fresh, failed, and deferred
// entries.
func TestNextCrawlAt(t *testing.T) {
now := time.Now()
if !nextCrawlAt(state.Entry{}, time.Hour).IsZero() {
t.Error("never-crawled entry not due immediately")
}
failedAt := now.Add(30 * time.Minute)
failed := state.Entry{LastError: "boom", LastCrawled: now, NextCrawl: failedAt}
if got := nextCrawlAt(failed, time.Hour); !got.Equal(failedAt) {
t.Errorf("failed entry gate = %v, want %v", got, failedAt)
}
ok := state.Entry{LastCrawled: now, NextCrawl: now.Add(10 * time.Minute)}
if got := nextCrawlAt(ok, 2*time.Hour); !got.Equal(now.Add(2 * time.Hour)) {
t.Errorf("tier gate = %v, want %v", got, now.Add(2*time.Hour))
}
}

459
server/crawl_loop.go Normal file
View file

@ -0,0 +1,459 @@
package server
import (
"context"
"fmt"
"os"
"strings"
"sync"
"sync/atomic"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/mirror"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// lockMu guards the per-resource lock table and the in-flight crawl set.
var lockMu sync.Mutex
// resourceLock is one entry in the lock table; refs counts holders and
// waiters so the entry can be dropped once nobody references it.
type resourceLock struct {
mu sync.Mutex
refs int
}
// locks serializes crawls of the same resource across request handlers and
// the scheduled loop.
var locks = map[string]*resourceLock{}
// lockResource acquires the lock for a resource key and returns its unlock
// function. Entries are reference counted and removed when the last holder
// or waiter releases, so evicted resources do not accumulate in the table.
func lockResource(key string) func() {
lockMu.Lock()
l := locks[key]
if l == nil {
l = &resourceLock{}
locks[key] = l
}
l.refs++
lockMu.Unlock()
l.mu.Lock()
return func() {
l.mu.Unlock()
lockMu.Lock()
l.refs--
if l.refs == 0 {
delete(locks, key)
}
lockMu.Unlock()
}
}
// inventoryEvictionGrace shields resources still advertised by a mount's
// configuration from eviction after requests stop.
const inventoryEvictionGrace = 72 * time.Hour
// inflight records resources with a crawl queued or running so concurrent
// dispatches from the request handlers and the scheduler collapse into one
// crawl. Guarded by lockMu.
var inflight = map[string]bool{}
// crawlWG tracks dispatched crawl goroutines so shutdown can wait for them
// before the final state flush records their outcomes.
var crawlWG sync.WaitGroup
// crawlCtx is the context dispatched crawls run under. CrawlLoop installs
// its own so shutdown cancels crawls; the background default covers tests
// and requests arriving before the loop starts.
// A pointer is stored rather than the interface itself: atomic.Value
// rejects a second store of a different concrete type, and the background
// default and the loop's cancelable context are different types.
var crawlCtx atomic.Pointer[context.Context]
func init() {
ctx := context.Background()
crawlCtx.Store(&ctx)
}
// Crawl slots bound how many crawls run at once; slotCond re-reads the
// configured limit on every wake so a reload takes effect.
var (
slotMu sync.Mutex
slotCond = sync.NewCond(&slotMu)
slotsInUse int
)
// acquireCrawlSlot blocks until a crawl slot is free under the configured
// concurrency limit.
func acquireCrawlSlot() {
slotMu.Lock()
for slotsInUse >= cfg.C.Crawler.ConcurrentCrawls {
slotCond.Wait()
}
slotsInUse++
slotMu.Unlock()
}
// releaseCrawlSlot frees a crawl slot and wakes the waiters.
func releaseCrawlSlot() {
slotMu.Lock()
slotsInUse--
slotMu.Unlock()
slotCond.Broadcast()
}
// startCrawl dispatches an asynchronous crawl of one resource, collapsing
// duplicate dispatches and bounding concurrency. Failures are logged; the
// retry gating lives in the schedulers.
func startCrawl(res resource) {
lockMu.Lock()
if inflight[res.Key] {
lockMu.Unlock()
return
}
inflight[res.Key] = true
lockMu.Unlock()
crawlWG.Add(1)
go func() {
defer crawlWG.Done()
defer func() {
lockMu.Lock()
delete(inflight, res.Key)
lockMu.Unlock()
}()
acquireCrawlSlot()
defer releaseCrawlSlot()
ctx := *crawlCtx.Load()
if ctx.Err() != nil {
return
}
if err := crawlResource(ctx, res); err != nil {
log.WithError(err).WithField("key", res.Key).Error("Repository crawl failed.")
}
}()
}
// WaitCrawls blocks until every dispatched crawl has finished. It is called
// at shutdown after the crawl context is canceled and before the final
// state flush.
func WaitCrawls() {
crawlWG.Wait()
}
// crawlResource synchronizes one classified resource under its lock. Deb
// resources also hold an archive-root lock so suites sharing a pool never
// download the same file concurrently.
func crawlResource(ctx context.Context, res resource) error {
if res.Kind == string(mirror.RepoDeb) && res.Root != res.Path {
defer lockResource("deb-root:" + res.Root)()
}
defer lockResource(res.Key)()
err := dispatchCrawl(ctx, res)
state.S.MarkCrawled(res.Key, time.Now(), err)
return err
}
// pathBelow reports whether a request path sits strictly below a base path.
func pathBelow(p, base string) bool {
if p == base {
return false
}
if base == "/" {
return true
}
return strings.HasPrefix(p, base+"/")
}
// prunableTree reports whether a crawl may prune its destination tree. Every
// sync prunes the repository directory, which for deb is the suite directory
// and otherwise the repository's own path. A crawl's keep set covers only
// that one repository, so a tree that also holds another mount's content or
// another tracked repository must not be pruned: the neighbour's files are
// unreferenced there and would be deleted. This matters most for a
// repository sitting at the mirror root, whose tree is the whole cache.
func prunableTree(conf *cfg.Config, res resource) bool {
for _, mount := range conf.Mounts {
if pathBelow(mount.Path, res.Path) {
log.WithFields(log.Fields{"key": res.Key, "mount": mount.Path}).
Debug("Pruning disabled; the repository tree contains another mount.")
return false
}
}
for key, entry := range state.S.Snapshot() {
if key == res.Key || entry.Kind == kindGeneric {
continue
}
if pathBelow(entry.Path, res.Path) {
log.WithFields(log.Fields{"key": res.Key, "nested": key}).
Debug("Pruning disabled; the repository tree contains another repository.")
return false
}
}
return true
}
// traceInfo maps the configured trace section onto the mirror package's
// description of this mirror. The mapping is repeated rather than shared
// with the sync commands so the mirror package keeps its own trace type
// instead of taking the configuration as part of its API.
func traceInfo(t cfg.TraceConfig) mirror.TraceInfo {
return mirror.TraceInfo{
Host: t.HostName(),
Maintainer: t.Maintainer,
Sponsor: t.Sponsor,
Country: t.Country,
Location: t.Location,
Throughput: t.Throughput,
}
}
// dispatchCrawl runs the format-specific synchronization for a resource
// into the online tree.
func dispatchCrawl(ctx context.Context, res resource) error {
online := cfg.C.OnlineDomain()
mount, ok := cfg.C.MountFor(res.Path)
if !ok {
return fmt.Errorf("no mount configured for %s", res.Path)
}
// upstreamFor maps a request path onto the mount's upstream URL.
upstreamFor := func(p string) string {
rel := p
if mount.Path != "/" {
rel = strings.TrimPrefix(p, mount.Path)
}
return mount.Upstream + rel
}
missing, err := fetch.ParseMissingMode(cfg.C.Crawler.MissingMode)
if err != nil {
return err
}
// Generic files never reach the crawl path: they refresh on demand at
// serve time and expire through the failure cache and state eviction.
typ := mirror.RepoType(res.Kind)
opts := &mirror.Options{
Type: typ,
Workers: cfg.C.Crawler.Workers,
Prune: prunableTree(cfg.C, res),
PruneGrace: cfg.C.Crawler.PruneGrace,
Missing: missing,
MissingRetries: cfg.C.Crawler.MissingRetries,
Trace: cfg.C.Trace.Enabled,
TraceInfo: traceInfo(cfg.C.Trace),
}
// Deb destinations pin the archive root so pool files land beside
// their suites; every other type syncs into its own directory.
destPath := res.Path
if typ == mirror.RepoDeb {
destPath = res.Root
}
dest, err := fetch.LocalJoin(online.Root, destPath)
if err != nil {
return err
}
repoURL := upstreamFor(res.Path)
return mirror.SyncInto(ctx, typ, fetch.NewSource([]string{repoURL}), repoURL, dest, opts)
}
// CrawlLoop keeps configured and discovered resources fresh until the
// context is canceled, starting with an immediate pass. Its context also
// governs request-dispatched crawls so shutdown cancels them.
func CrawlLoop(ctx context.Context) {
crawlCtx.Store(&ctx)
runScheduledCrawls(ctx)
t := time.NewTicker(cfg.C.CrawlCheckInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
fetchFailures.Sweep(time.Now())
runScheduledCrawls(ctx)
// Re-arm from the active config so a reload takes effect.
t.Reset(cfg.C.CrawlCheckInterval)
}
}
}
// runScheduledCrawls runs one pass of configured and request-discovered
// crawls.
func runScheduledCrawls(ctx context.Context) {
crawlConfigured(ctx)
crawlDue(ctx)
}
// configuredResource builds the resource for a mount's configured
// repository.
func configuredResource(repo cfg.MountRepoConfig) resource {
root := repo.Path
if repo.Type == string(mirror.RepoDeb) {
if idx := strings.Index(repo.Path, "/dists/"); idx >= 0 {
root = repo.Path[:idx]
if root == "" {
root = "/"
}
}
}
return resource{
Kind: repo.Type,
Key: repo.Type + ":" + repo.Path,
Path: repo.Path,
Root: root,
ReqPath: repo.Path,
}
}
// crawlConfigured keeps every mount-configured repository synchronized
// regardless of client traffic. Crawls are dispatched asynchronously so a
// slow repository cannot stall the scheduler loop.
func crawlConfigured(ctx context.Context) {
now := time.Now()
for _, mount := range cfg.C.Mounts {
for _, repo := range mount.Repos {
if ctx.Err() != nil {
return
}
res := configuredResource(repo)
state.S.MarkSeenInInventory(res.Kind, res.Key, res.Path, res.Root, now)
entry, _ := state.S.Entry(res.Key)
if !dueForInventory(entry, now) {
continue
}
startCrawl(res)
}
}
}
// dueForInventory gates configured crawls on the standard interval with
// faster retries after failures.
func dueForInventory(entry state.Entry, now time.Time) bool {
if entry.LastCrawled.IsZero() {
return true
}
if !entry.NextCrawl.IsZero() {
return !now.Before(entry.NextCrawl)
}
return now.Sub(entry.LastCrawled) >= cfg.C.RepoCrawlInterval
}
// refreshTier walks the cumulative schedule, returning the refresh
// interval matching how long ago the resource was requested, or stale once
// the whole budget has elapsed.
func refreshTier(sinceRequested time.Duration, schedule []time.Duration) (time.Duration, bool) {
var cumulative time.Duration
for _, step := range schedule {
if step <= 0 {
continue
}
cumulative += step
if sinceRequested < cumulative {
return step, false
}
}
return 0, true
}
// refreshSchedule is the tier list: the active crawl interval followed by
// the configured aging backoff steps.
func refreshSchedule() []time.Duration {
schedule := make([]time.Duration, 0, 1+len(cfg.C.Crawler.RefreshSchedule))
schedule = append(schedule, cfg.C.RepoCrawlInterval)
return append(schedule, cfg.C.Crawler.RefreshSchedule...)
}
// activeInInventory reports whether a mount still advertised the resource
// recently enough to shield it from eviction.
func activeInInventory(entry state.Entry, now time.Time) bool {
if entry.LastSeenInInventory.IsZero() {
return false
}
return now.Sub(entry.LastSeenInInventory) < inventoryEvictionGrace
}
// nextCrawlAt returns when a resource should next crawl: immediately when
// never crawled, at the failure retry gate after errors, else at the tier
// interval.
func nextCrawlAt(entry state.Entry, tierInterval time.Duration) time.Time {
if entry.LastCrawled.IsZero() {
return time.Time{}
}
if entry.LastError != "" && !entry.NextCrawl.IsZero() {
return entry.NextCrawl
}
tierGate := entry.LastCrawled.Add(tierInterval)
if !entry.NextCrawl.IsZero() && entry.NextCrawl.After(tierGate) {
return entry.NextCrawl
}
return tierGate
}
// crawlDue refreshes request-discovered resources on the tier schedule and
// evicts those whose traffic stopped past the whole budget.
func crawlDue(ctx context.Context) {
now := time.Now()
schedule := refreshSchedule()
for key, entry := range state.S.Snapshot() {
if ctx.Err() != nil {
return
}
// Inventory-only entries are managed by crawlConfigured.
if entry.LastRequested.IsZero() {
continue
}
interval, stale := refreshTier(now.Sub(entry.LastRequested), schedule)
if stale {
if activeInInventory(entry, now) {
continue
}
evictStale(key, entry)
continue
}
// Generic files refresh on demand at serve time.
if entry.Kind == kindGeneric {
continue
}
next := nextCrawlAt(entry, interval)
if !next.IsZero() && now.Before(next) {
continue
}
startCrawl(resource{Kind: entry.Kind, Key: key, Path: entry.Path, Root: entry.Root, ReqPath: entry.Path})
}
}
// evictStale removes a resource whose requests stopped long enough ago,
// re-checking under the lock so a racing request wins.
func evictStale(key string, entry state.Entry) {
// Deb resources share an archive root across sibling suites; hold that
// lock too, matching crawlResource, so eviction never races an
// in-flight sibling-suite crawl touching the shared pool.
if entry.Kind == string(mirror.RepoDeb) && entry.Root != entry.Path {
defer lockResource("deb-root:" + entry.Root)()
}
defer lockResource(key)()
current, ok := state.S.Entry(key)
if ok && current.LastRequested.After(entry.LastRequested) {
return
}
target, err := fetch.LocalJoin(cfg.C.OnlineDomain().Root, entry.Path)
if err == nil && entry.Path != "/" && entry.Path != "" {
if err := os.RemoveAll(target); err != nil {
log.WithError(err).WithField("path", target).Warn("Failed to evict stale resource.")
return
}
log.WithFields(log.Fields{"key": key, "path": target}).Info("Evicted stale resource.")
}
state.S.Delete(key)
}

56
server/failure_cache.go Normal file
View file

@ -0,0 +1,56 @@
package server
import (
"sync"
"time"
)
// failureEntry is one remembered upstream failure with its expiry.
type failureEntry struct {
status int
expires time.Time
}
// failureCache remembers recent upstream failures for generic paths so a
// missing or erroring file does not trigger an upstream request per client.
// It is memory-only; nothing here reaches the state file.
type failureCache struct {
mu sync.Mutex
entries map[string]failureEntry
}
// fetchFailures is the server's shared failure cache.
var fetchFailures = &failureCache{entries: map[string]failureEntry{}}
// Hit returns the remembered status for a key, pruning expired entries.
func (c *failureCache) Hit(key string, now time.Time) (int, bool) {
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.entries[key]
if !ok {
return 0, false
}
if !now.Before(e.expires) {
delete(c.entries, key)
return 0, false
}
return e.status, true
}
// Add remembers a failure until now+ttl.
func (c *failureCache) Add(key string, status int, now time.Time, ttl time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = failureEntry{status: status, expires: now.Add(ttl)}
}
// Sweep drops expired entries that nobody re-requested.
func (c *failureCache) Sweep(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for key, e := range c.entries {
if !now.Before(e.expires) {
delete(c.entries, key)
}
}
}

81
server/http.go Normal file
View file

@ -0,0 +1,81 @@
package server
import (
"context"
"net"
"net/http"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
log "github.com/sirupsen/logrus"
)
// shutdownTimeout bounds the graceful shutdown of the listener, after which
// in-flight requests are abandoned.
const shutdownTimeout = 30 * time.Second
// Server owns the mirror's HTTP listener. It binds from the active
// configuration and rebinds on reload when the listen address moves.
type Server struct {
server *http.Server
addr string
}
// NewServer builds a mirror HTTP server from the active configuration.
func NewServer() *Server {
s := new(Server)
s.configure()
return s
}
// configure rebuilds the underlying server from the active configuration.
// A running server is never reconfigured in place; Reload binds a new one.
func (s *Server) configure() {
s.addr = cfg.C.HTTP.ListenAddr()
s.server = &http.Server{
Addr: s.addr,
Handler: Handler(),
ReadHeaderTimeout: cfg.C.HTTP.ReadHeaderTimeout,
ReadTimeout: cfg.C.HTTP.ReadTimeout,
WriteTimeout: cfg.C.HTTP.WriteTimeout,
IdleTimeout: cfg.C.HTTP.IdleTimeout,
}
}
// Start binds the listen address and serves in the background. A bind
// failure is returned so the caller can abort startup rather than run
// without a listener.
func (s *Server) Start() error {
ln, err := net.Listen("tcp", s.addr)
if err != nil {
return err
}
log.WithField("addr", s.addr).Info("Mirror server listening.")
go func() {
if err := s.server.Serve(ln); err != nil && err != http.ErrServerClosed {
log.WithError(err).Error("HTTP server failed.")
}
}()
return nil
}
// Reload rebinds the listener when the active configuration moved the
// listen address, and is a no-op otherwise. The old listener is shut down
// before the new one binds so the address is free. Other HTTP settings are
// read when the server is built, so they take effect on the next rebind.
func (s *Server) Reload() error {
if cfg.C.HTTP.ListenAddr() == s.addr {
return nil
}
s.Shutdown()
s.configure()
return s.Start()
}
// Shutdown stops accepting connections and waits for in-flight requests,
// giving up after shutdownTimeout.
func (s *Server) Shutdown() {
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
_ = s.server.Shutdown(ctx)
}

14
server/main_test.go Normal file
View file

@ -0,0 +1,14 @@
package server
import (
"os"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
)
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
fetch.Reload()
os.Exit(m.Run())
}

316
server/serve.go Normal file
View file

@ -0,0 +1,316 @@
// Package server runs the caching mirror: it answers requests from the
// local tree, fetches misses through the configured upstream mounts, and
// crawls the repositories it discovers in the background so they stay
// current without holding a request open.
package server
import (
"context"
"errors"
"fmt"
"html"
"io"
"net/http"
"os"
"path"
"sort"
"strings"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// Handler returns the mirror server's HTTP handler.
func Handler() http.Handler {
return http.HandlerFunc(serveMirror)
}
// regularFile stats a path, reporting only regular files as present.
func regularFile(name string) (os.FileInfo, bool) {
info, err := os.Stat(name)
if err != nil || !info.Mode().IsRegular() {
return nil, false
}
return info, true
}
// serveLocal resolves a request path under a domain root, mapping
// directory requests to a cached index.html.
func serveLocal(root, reqPath string, isDir bool) (string, error) {
p := reqPath
if isDir || p == "/" {
p = strings.TrimSuffix(p, "/") + "/index.html"
}
return fetch.LocalJoin(root, p)
}
// internalFile reports paths that belong to the mirror's own bookkeeping
// and must never be served.
func internalFile(reqPath string) bool {
base := path.Base(reqPath)
return base == fetch.LockFileName || base == fetch.PruneStateName || base == fetch.MissingStateName ||
strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix)
}
// fetchUpstream retrieves one file through the mount covering the request
// path and stores it under root.
func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error {
mount, ok := cfg.C.MountFor(reqPath)
if !ok {
return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
}
rel := reqPath
if mount.Path != "/" {
rel = strings.TrimPrefix(reqPath, mount.Path)
}
if isDir {
rel = strings.TrimSuffix(rel, "/") + "/"
}
dst, err := serveLocal(root, reqPath, isDir)
if err != nil {
return err
}
// Concurrent on-demand fetches and background crawls can target the
// same destination; fetch.File serializes them by path.
src := fetch.NewSource([]string{mount.Upstream})
_, err = fetch.File(ctx, src, rel, dst, nil, false, false)
return err
}
// writeFetchError reports an upstream failure to the client.
func writeFetchError(w http.ResponseWriter, err error) {
if errors.Is(err, fetch.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeCachedFailure reports a remembered upstream failure.
func writeCachedFailure(w http.ResponseWriter, status int) {
if status == http.StatusNotFound {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeIndexesDisabled answers a directory request when directory indexes
// are turned off, confirming the mirror is alive without listing content.
func writeIndexesDisabled(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, "<!DOCTYPE html>\n<html>\n<head><title>repo-sync mirror</title></head>\n<body>\n"+
"<h1>repo-sync mirror</h1>\n<p>This mirror is running. Directory indexes are disabled.</p>\n"+
"</body>\n</html>\n")
}
// writeMountIndex answers the root directory request with a generated index
// of every mount and configured repository, giving clients a navigable
// entry point when no catch-all mount covers the root.
func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) {
seen := map[string]bool{}
var paths []string
add := func(p string) {
if p == "/" || seen[p] {
return
}
seen[p] = true
paths = append(paths, p)
}
for _, mount := range conf.Mounts {
add(mount.Path)
for _, repo := range mount.Repos {
add(repo.Path)
}
}
sort.Strings(paths)
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html>\n<head><title>Index of /</title></head>\n<body>\n<h1>Index of /</h1>\n<ul>\n")
for _, p := range paths {
href := html.EscapeString(p + "/")
fmt.Fprintf(&b, "<li><a href=\"%s\">%s</a></li>\n", href, href)
}
b.WriteString("</ul>\n</body>\n</html>\n")
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, b.String())
}
// serveMirror handles every mirror request: domain lookup, classification,
// and dispatch to the role-specific handler.
func serveMirror(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
domain, ok := cfg.C.Domain(r.Host)
if !ok {
http.Error(w, "unknown mirror domain", http.StatusNotFound)
return
}
reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/"))
isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/"
if internalFile(reqPath) {
http.Error(w, "not found", http.StatusNotFound)
return
}
// Directory requests are answered before role dispatch: a notice page
// when indexes are disabled, and a generated index of the mounts for a
// root that no catch-all mount covers.
if isDir {
if !cfg.C.HTTP.DirectoryIndexes {
writeIndexesDisabled(w)
return
}
if reqPath == "/" {
if _, ok := cfg.C.MountFor("/"); !ok {
writeMountIndex(w, cfg.C)
return
}
}
}
res := classifyRequest(reqPath)
if domain.Role == cfg.RoleOnline {
handleOnline(w, r, domain, reqPath, isDir, res)
return
}
handleOffline(w, r, domain, reqPath, isDir, res)
}
// handleOnline serves from the writable cache, fetching and crawling on
// demand.
func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
now := time.Now()
// Directory listings are cached pass-through pages with no tracking.
if isDir {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
if res.Kind == kindGeneric {
// Files under a registered repository keep it alive and are
// fetched on demand until its crawl completes.
if state.S.TouchRepoMembers(reqPath, now) {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
handleGeneric(w, r, domain, reqPath, res, local, now)
return
}
// A repository entry point registers the repository and dispatches a
// background crawl on first sight so the whole repository fills in
// behind this response without holding it open. The requested file
// itself is fetched on demand so the client is answered immediately.
// Repositories already crawled only re-dispatch here to retry a failed
// crawl; missing optional files must not trigger a full crawl per
// request.
entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now)
_, exists := regularFile(local)
needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "")
if !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
// A path whose first contact failed was never shown to be a
// repository, so the registration this request created is
// dropped and the scheduler never crawls it. Registrations an
// earlier request established are kept: every entry point of one
// repository shares a key, so a repository serving Release but
// not InRelease would otherwise deregister itself on the miss.
if !tracked {
state.S.Delete(res.Key)
}
writeFetchError(w, err)
return
}
}
if needCrawl {
startCrawl(res)
}
http.ServeFile(w, r, local)
}
// handleGeneric serves a plain cached file, refreshing stale copies and
// remembering upstream failures.
func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) {
if status, hit := fetchFailures.Hit(res.Key, now); hit {
if _, exists := regularFile(local); !exists {
writeCachedFailure(w, status)
return
}
} else {
// GenericMaxAge bounds how often the upstream is revalidated,
// measured from the last upstream check recorded in state. The
// local file's modification time carries the upstream's
// Last-Modified, so it does not record when the mirror last checked.
entry, tracked := state.S.Entry(res.Key)
_, exists := regularFile(local)
if !exists || !tracked || now.Sub(entry.LastCrawled) > cfg.C.GenericMaxAge {
err := fetchUpstream(r.Context(), domain.Root, reqPath, false)
switch {
case errors.Is(err, fetch.ErrNotFound):
fetchFailures.Add(res.Key, http.StatusNotFound, now, cfg.C.NegativeCacheTTL)
state.S.Delete(res.Key)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.")
case err != nil:
fetchFailures.Add(res.Key, http.StatusBadGateway, now, cfg.C.TransientErrorTTL)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.")
default:
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
// Record the upstream check so GenericMaxAge gates the next
// request. The scheduler skips generic entries, so the retry
// time this also sets is unused.
state.S.MarkCrawled(res.Key, now, nil)
}
} else {
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
}
}
http.ServeFile(w, r, local)
}
// handleOffline serves the read-only published tree, reading through the
// online cache for anything it does not hold.
func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
if _, exists := regularFile(local); exists {
http.ServeFile(w, r, local)
return
}
handleOnline(w, r, cfg.C.OnlineDomain(), reqPath, isDir, res)
}

487
server/serve_test.go Normal file
View file

@ -0,0 +1,487 @@
package server
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
"github.com/grmrgecko/repo-sync/mirror"
"github.com/grmrgecko/repo-sync/state"
)
// serverFixture builds an upstream with one repository of each type plus a
// generic file, configures the mirror server against it, and returns the
// mirror's test server and the online root.
func serverFixture(t *testing.T) (*httptest.Server, string, string) {
t.Helper()
// Upstream content behind a single catch-all mount.
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "almalinux", "9", "BaseOS", "x86_64", "os"))
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
testrepos.BuildArchRepo(t, filepath.Join(www, "archlinux", "core", "os", "x86_64"), "core")
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "main", "x86_64"))
testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file"))
upstream := testrepos.ServeDir(t, www)
onlineRoot := t.TempDir()
offlineRoot := t.TempDir()
confDir := t.TempDir()
conf := fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
- domain: offline.test
role: offline
root: %s
mounts:
- path: /
upstream: %s
`, confDir, onlineRoot, offlineRoot, upstream.URL)
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(conf))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
// Crawls dispatched by requests run in the background; drain them so
// they do not race the next test's state reload.
t.Cleanup(WaitCrawls)
return srv, onlineRoot, offlineRoot
}
// waitFor polls a condition until it holds or the deadline passes; crawls
// dispatched by requests run asynchronously.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
// get fetches a mirror URL with an optional Host override.
func get(t *testing.T, srv *httptest.Server, host, path string) (*http.Response, []byte) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil)
if err != nil {
t.Fatal(err)
}
if host != "" {
req.Host = host
}
resp, err := srv.Client().Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Fatal(err)
}
return resp, body
}
// TestServeRPMDiscovery verifies a repomd.xml request registers and crawls
// the repository, filling the local tree behind the response.
func TestServeRPMDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
repo := "/almalinux/9/BaseOS/x86_64/os"
resp, body := get(t, srv, "", repo+"/repodata/repomd.xml")
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
if len(body) == 0 {
t.Fatal("empty repomd.xml response")
}
// The repository was registered and the background crawl mirrors its
// packages behind the response.
if _, ok := state.S.Entry("rpm:" + repo); !ok {
t.Error("repository not registered in state")
}
var pkgs []string
waitFor(t, "rpm crawl", func() bool {
pkgs, _ = filepath.Glob(filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), "Packages", "*.rpm"))
return len(pkgs) > 0
})
// Package requests are served from the crawled tree.
resp, _ = get(t, srv, "", repo+"/Packages/"+filepath.Base(pkgs[0]))
if resp.StatusCode != http.StatusOK {
t.Errorf("package request status = %d", resp.StatusCode)
}
}
// TestServeDebDiscovery verifies a Release request crawls the suite with
// pool files under the archive root.
func TestServeDebDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, _ := get(t, srv, "", "/debian/dists/test/Release")
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
if _, ok := state.S.Entry("deb:/debian/dists/test"); !ok {
t.Error("suite not registered in state")
}
waitFor(t, "deb crawl", func() bool {
_, err := os.Stat(filepath.Join(onlineRoot, "debian", "pool", "main", "h", "hello", "hello_1.0_amd64.deb"))
return err == nil
})
// A pool request under the archive root is a repository member.
resp, _ = get(t, srv, "", "/debian/pool/main/h/hello/hello_1.0_amd64.deb")
if resp.StatusCode != http.StatusOK {
t.Errorf("pool request status = %d", resp.StatusCode)
}
}
// TestServeArchAndApkDiscovery verifies database and index requests crawl
// their repositories.
func TestServeArchAndApkDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, _ := get(t, srv, "", "/archlinux/core/os/x86_64/core.db")
if resp.StatusCode != http.StatusOK {
t.Fatalf("arch status = %d", resp.StatusCode)
}
waitFor(t, "arch crawl", func() bool {
pkgs, _ := filepath.Glob(filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "*.pkg.tar.zst"))
return len(pkgs) > 0
})
resp, _ = get(t, srv, "", "/alpine/v3.24/main/x86_64/APKINDEX.tar.gz")
if resp.StatusCode != http.StatusOK {
t.Fatalf("apk status = %d", resp.StatusCode)
}
waitFor(t, "apk crawl", func() bool {
apks, _ := filepath.Glob(filepath.Join(onlineRoot, "alpine", "v3.24", "main", "x86_64", "*.apk"))
return len(apks) > 0
})
}
// TestServeGeneric verifies plain files proxy through with caching and
// missing files are answered from the negative cache.
func TestServeGeneric(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, body := get(t, srv, "", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("generic = %d %q", resp.StatusCode, body)
}
if _, err := os.Stat(filepath.Join(onlineRoot, "notes", "README.txt")); err != nil {
t.Error("generic file not cached:", err)
}
resp, _ = get(t, srv, "", "/notes/missing.txt")
if resp.StatusCode != http.StatusNotFound {
t.Errorf("missing file status = %d", resp.StatusCode)
}
if status, hit := fetchFailures.Hit("generic:/notes/missing.txt", time.Now()); !hit || status != http.StatusNotFound {
t.Error("missing file not negative-cached")
}
}
// TestServeGenericRevalidation verifies GenericMaxAge gates how often a
// cached plain file is revalidated. The upstream file carries an old
// Last-Modified that the cached copy inherits, so the gate is measured from
// the recorded upstream check rather than the local modification time.
func TestServeGenericRevalidation(t *testing.T) {
www := t.TempDir()
name := filepath.Join(www, "notes", "README.txt")
testrepos.WriteFile(t, name, []byte("generic file"))
old := time.Now().Add(-30 * 24 * time.Hour)
if err := os.Chtimes(name, old, old); err != nil {
t.Fatal(err)
}
var upstreamHits atomic.Int64
files := http.FileServer(http.Dir(www))
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamHits.Add(1)
files.ServeHTTP(w, r)
}))
t.Cleanup(upstream.Close)
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(`
state_path: %s/state.yaml
generic_max_age: 6h
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /
upstream: %s
`, confDir, t.TempDir(), upstream.URL)))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
for i := 0; i < 3; i++ {
resp, body := get(t, srv, "", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("request %d = %d %q", i, resp.StatusCode, body)
}
}
if got := upstreamHits.Load(); got != 1 {
t.Errorf("upstream requests = %d, want 1 within generic_max_age", got)
}
}
// TestServeOffline verifies offline domains serve their own tree first and
// read through the online cache otherwise.
func TestServeOffline(t *testing.T) {
srv, _, offlineRoot := serverFixture(t)
// Not present offline: delegated to online, which proxies upstream.
resp, body := get(t, srv, "offline.test", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("offline read-through = %d %q", resp.StatusCode, body)
}
// Present offline: served from the offline tree.
testrepos.WriteFile(t, filepath.Join(offlineRoot, "notes", "README.txt"), []byte("published copy"))
resp, body = get(t, srv, "offline.test", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "published copy" {
t.Errorf("offline copy = %d %q", resp.StatusCode, body)
}
}
// TestServeDirectoryIndexes verifies the generated root index without a
// catch-all mount and the notice page when indexes are disabled.
func TestServeDirectoryIndexes(t *testing.T) {
www := t.TempDir()
testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file"))
upstream := testrepos.ServeDir(t, www)
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
conf := fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /notes
upstream: %s/notes
- path: /almalinux
upstream: %s/almalinux
repos:
- path: /almalinux/9/BaseOS/x86_64/os
type: rpm
`, confDir, t.TempDir(), upstream.URL, upstream.URL)
testrepos.WriteFile(t, confPath, []byte(conf))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
// Without a catch-all mount the root serves a generated index linking
// every mount and configured repository.
resp, body := get(t, srv, "", "/")
if resp.StatusCode != http.StatusOK {
t.Fatalf("index status = %d", resp.StatusCode)
}
for _, want := range []string{`"/notes/"`, `"/almalinux/"`, `"/almalinux/9/BaseOS/x86_64/os/"`} {
if !strings.Contains(string(body), want) {
t.Errorf("index missing %s in %q", want, body)
}
}
// Disabling indexes answers every directory request with the notice.
disabled := strings.Replace(conf, "mounts:", "http:\n directory_indexes: false\nmounts:", 1)
testrepos.WriteFile(t, confPath, []byte(disabled))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
resp, body = get(t, srv, "", "/")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") {
t.Errorf("disabled root = %d %q", resp.StatusCode, body)
}
resp, body = get(t, srv, "", "/notes/")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") {
t.Errorf("disabled subdirectory = %d %q", resp.StatusCode, body)
}
}
// TestServeRejections verifies unknown domains and internal bookkeeping
// files are refused.
func TestServeRejections(t *testing.T) {
srv, _, _ := serverFixture(t)
resp, _ := get(t, srv, "unknown.test", "/notes/README.txt")
if resp.StatusCode != http.StatusNotFound {
t.Errorf("unknown domain status = %d", resp.StatusCode)
}
resp, _ = get(t, srv, "", path.Join("/almalinux", fetch.LockFileName))
if resp.StatusCode != http.StatusNotFound {
t.Errorf("internal file status = %d", resp.StatusCode)
}
}
// TestPrunableTree verifies a crawl only prunes a tree it wholly owns, so a
// repository at the mirror root cannot delete another mount's cached files.
func TestPrunableTree(t *testing.T) {
serverFixture(t)
conf := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}, {Path: "/other"}}}
if prunableTree(conf, resource{Key: "rpm:/", Path: "/"}) {
t.Error("a root repository may not prune a tree holding another mount")
}
if !prunableTree(conf, resource{Key: "rpm:/other/el9", Path: "/other/el9"}) {
t.Error("a repository owning its own tree must prune")
}
// A repository nested under another one blocks the outer one's prune.
single := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}}}
state.S.MarkRequested("rpm", "rpm:/vendor/el9", "/vendor/el9", "/vendor/el9", time.Now())
if prunableTree(single, resource{Key: "rpm:/vendor", Path: "/vendor"}) {
t.Error("an outer repository may not prune a tree holding a nested one")
}
if !prunableTree(single, resource{Key: "rpm:/vendor/el9", Path: "/vendor/el9"}) {
t.Error("a repository must not be blocked by its own registration")
}
}
// TestServeRootRepositoryKeepsOtherMounts verifies a repository discovered at
// the mirror root does not prune content another mount cached beside it.
func TestServeRootRepositoryKeepsOtherMounts(t *testing.T) {
// Two upstreams: a repository at its own root, and unrelated content.
repoWWW := t.TempDir()
testrepos.BuildRPMRepo(t, repoWWW)
repoUpstream := testrepos.ServeDir(t, repoWWW)
otherWWW := t.TempDir()
testrepos.WriteFile(t, filepath.Join(otherWWW, "notes.txt"), []byte("other mount content"))
otherUpstream := testrepos.ServeDir(t, otherWWW)
onlineRoot := t.TempDir()
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /
upstream: %s
- path: /other
upstream: %s
`, confDir, onlineRoot, repoUpstream.URL, otherUpstream.URL)))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
t.Cleanup(WaitCrawls)
// Cache a file from the unrelated mount, then crawl the root repository.
if resp, _ := get(t, srv, "", "/other/notes.txt"); resp.StatusCode != http.StatusOK {
t.Fatalf("notes status = %d", resp.StatusCode)
}
cached := filepath.Join(onlineRoot, "other", "notes.txt")
if _, err := os.Stat(cached); err != nil {
t.Fatal("unrelated file not cached:", err)
}
if resp, _ := get(t, srv, "", "/repodata/repomd.xml"); resp.StatusCode != http.StatusOK {
t.Fatalf("repomd status = %d", resp.StatusCode)
}
WaitCrawls()
if _, err := os.Stat(cached); err != nil {
t.Error("root repository crawl pruned another mount's cached file:", err)
}
}
// TestServeSiblingEntryPointMiss verifies a repository keeps its
// registration when a sibling entry-point path the upstream does not serve
// is requested. Every entry point of one repository shares a state key, and
// clients probe all of them: apt asks for InRelease, Release, and
// Release.gpg, and dnf asks for signature material beside repomd.xml.
func TestServeSiblingEntryPointMiss(t *testing.T) {
srv, _, _ := serverFixture(t)
for _, tc := range []struct{ found, missing, key string }{
{"/debian/dists/test/Release", "/debian/dists/test/InRelease", "deb:/debian/dists/test"},
{
"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml",
"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml.key",
"rpm:/almalinux/9/BaseOS/x86_64/os",
},
} {
if resp, _ := get(t, srv, "", tc.found); resp.StatusCode != http.StatusOK {
t.Fatalf("%s status = %d", tc.found, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); !ok {
t.Fatalf("%s did not register %s", tc.found, tc.key)
}
if resp, _ := get(t, srv, "", tc.missing); resp.StatusCode != http.StatusNotFound {
t.Fatalf("%s status = %d, want 404", tc.missing, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); !ok {
t.Errorf("%s deregistered %s", tc.missing, tc.key)
}
}
}
// TestServeUnresolvedEntryPoint verifies a path that only looks like a
// repository entry point is not left registered for the scheduler to crawl.
func TestServeUnresolvedEntryPoint(t *testing.T) {
srv, _, _ := serverFixture(t)
for _, tc := range []struct{ path, key string }{
{"/nope/repodata/repomd.xml", "rpm:/nope"},
{"/nope/dists/sid/InRelease", "deb:/nope/dists/sid"},
{"/nope/database.db", "arch:/nope"},
{"/nope/" + mirror.APKIndexName, "apk:/nope"},
} {
resp, _ := get(t, srv, "", tc.path)
if resp.StatusCode != http.StatusNotFound {
t.Errorf("%s status = %d", tc.path, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); ok {
t.Errorf("%s left %s registered", tc.path, tc.key)
}
}
}

137
server/trace_test.go Normal file
View file

@ -0,0 +1,137 @@
package server
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/internal/testrepos"
"github.com/grmrgecko/repo-sync/mirror"
"github.com/grmrgecko/repo-sync/state"
)
// traceFixture builds an upstream apk repository that publishes a trace of
// its own, configures the mirror server against it with tracing enabled,
// and returns the mirror's test server and the online root.
func traceFixture(t *testing.T) (*httptest.Server, string) {
t.Helper()
www := t.TempDir()
repo := filepath.Join(www, "alpine", "v3.24", "main", "x86_64")
testrepos.BuildApkRepo(t, repo)
testrepos.WriteFile(t, filepath.Join(repo, filepath.FromSlash(mirror.TraceDir), "origin.example.net"),
[]byte("origin trace\n"))
upstream := testrepos.ServeDir(t, www)
onlineRoot := t.TempDir()
confDir := t.TempDir()
conf := fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /
upstream: %s
trace:
enabled: true
host: mirror.example.com
maintainer: Test Maintainer <test@example.com>
`, confDir, onlineRoot, upstream.URL)
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(conf))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
t.Cleanup(WaitCrawls)
return srv, onlineRoot
}
// TestServeCrawlWritesTrace verifies a server crawl publishes this mirror's
// trace and mirrors upstream's, and that both are then served like any
// other repository file.
func TestServeCrawlWritesTrace(t *testing.T) {
srv, onlineRoot := traceFixture(t)
repo := "/alpine/v3.24/main/x86_64"
resp, _ := get(t, srv, "", repo+"/APKINDEX.tar.gz")
if resp.StatusCode != http.StatusOK {
t.Fatalf("index status = %d", resp.StatusCode)
}
traceDir := filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), filepath.FromSlash(mirror.TraceDir))
own := filepath.Join(traceDir, "mirror.example.com")
waitFor(t, "crawl trace", func() bool {
_, err := os.Stat(own)
return err == nil
})
data, err := os.ReadFile(own)
if err != nil {
t.Fatal("reading trace:", err)
}
trace := string(data)
for _, want := range []string{
"Running on host: mirror.example.com",
"Maintainer: Test Maintainer <test@example.com>",
"Repository type: apk",
"Total bytes received: ",
} {
if !strings.Contains(trace, want) {
t.Errorf("trace missing %q\ngot:\n%s", want, trace)
}
}
if upstreamTrace, err := os.ReadFile(filepath.Join(traceDir, "origin.example.net")); err != nil {
t.Error("upstream trace not mirrored:", err)
} else if string(upstreamTrace) != "origin trace\n" {
t.Errorf("upstream trace = %q, want %q", upstreamTrace, "origin trace\n")
}
// The published traces are served from the crawled tree.
resp, body := get(t, srv, "", repo+"/"+mirror.TraceDir+"/mirror.example.com")
if resp.StatusCode != http.StatusOK {
t.Fatalf("trace request status = %d", resp.StatusCode)
}
if string(body) != trace {
t.Errorf("served trace does not match the published file\ngot:\n%s", body)
}
}
// TestTraceInfo verifies the configured trace section maps onto the mirror
// package's description of this mirror, including the host fallback.
func TestTraceInfo(t *testing.T) {
got := traceInfo(cfg.TraceConfig{
Host: "mirror.example.com",
Maintainer: "Jane <jane@example.com>",
Sponsor: "Example Inc",
Country: "US",
Location: "Texas",
Throughput: "10Gb",
})
want := mirror.TraceInfo{
Host: "mirror.example.com",
Maintainer: "Jane <jane@example.com>",
Sponsor: "Example Inc",
Country: "US",
Location: "Texas",
Throughput: "10Gb",
}
if got != want {
t.Errorf("traceInfo = %+v, want %+v", got, want)
}
if host := traceInfo(cfg.TraceConfig{}).Host; host == "" {
t.Error("traceInfo left the host empty; the system hostname should stand in")
}
}

129
server_cmd.go Normal file
View file

@ -0,0 +1,129 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/coreos/go-systemd/v22/daemon"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/server"
"github.com/grmrgecko/repo-sync/state"
"github.com/kardianos/service"
log "github.com/sirupsen/logrus"
)
// stopChan carries a shutdown request from the service manager. It is
// buffered so a stop delivered after the signal loop exits cannot block the
// service supervisor.
var stopChan = make(chan struct{}, 1)
// ServerCmd runs the caching mirror server.
type ServerCmd struct {
Bind string `name:"http.bind" help:"Override the configured bind address." optional:""`
Port uint `name:"http.port" help:"Override the configured port." optional:""`
}
// updateCFG applies the command line overrides onto the active
// configuration. The overridden fields are read only when the listener is
// built, which happens on this goroutine, so a reload can amend them in
// place after Init has published the new configuration.
func (c *ServerCmd) updateCFG() {
if c.Bind != "" {
cfg.C.HTTP.BindAddr = c.Bind
}
if c.Port != 0 {
cfg.C.HTTP.Port = c.Port
}
}
// Run starts the mirror server: configuration, state, crawl loops, and the
// HTTP listener, then handles signals until shutdown.
func (c *ServerCmd) Run() error {
// main installed the configuration; the server additionally needs the
// domains and mounts the sync commands do without.
c.updateCFG()
if err := cfg.C.RequireServer(); err != nil {
return err
}
// Load persisted crawl state.
if err := state.Load(); err != nil {
return err
}
// Start the background loops and the listener.
loopCtx, loopCancel := context.WithCancel(context.Background())
defer loopCancel()
go state.S.FlushLoop(loopCtx)
crawlDone := make(chan struct{})
go func() {
defer close(crawlDone)
server.CrawlLoop(loopCtx)
}()
// stopCrawls cancels the loops and waits for in-flight crawls so their
// outcomes are recorded before the final state flush. The crawl loop
// must return first: a scheduler pass in flight can still dispatch
// crawls, and a WaitGroup Add racing Wait is a misuse panic.
stopCrawls := func() {
loopCancel()
<-crawlDone
server.WaitCrawls()
}
httpServer := server.NewServer()
if err := httpServer.Start(); err != nil {
return err
}
// Attach to the service manager when not run interactively, so a stop
// request arrives on stopChan, then report readiness now that the
// listener is bound.
if !service.Interactive() {
svc, err := new(ServiceCmd).service()
if err != nil {
return err
}
go svc.Run()
}
_, _ = daemon.SdNotify(false, daemon.SdNotifyReady)
// Handle signals: reload on SIGHUP, shut down on SIGINT/SIGTERM or on a
// stop from the service manager.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
SigLoop:
for {
select {
case <-stopChan:
break SigLoop
case sig := <-signals:
if sig != syscall.SIGHUP {
break SigLoop
}
log.Info("Reloading configuration.")
if err := cfg.InitServer(flags.ConfigPath); err != nil {
log.WithError(err).Error("Unable to reload configuration; keeping previous values.")
continue
}
applyGlobalFlags()
c.updateCFG()
if err := httpServer.Reload(); err != nil {
// Exiting on a failed rebind must still flush crawl state.
stopCrawls()
if saveErr := state.S.Save(); saveErr != nil {
log.WithError(saveErr).Error("Failed to save state while shutting down.")
}
return err
}
}
}
// Shut down: stop the listener, stop the loops, then flush state.
log.Info("Shutting down.")
httpServer.Shutdown()
stopCrawls()
return state.S.Save()
}

125
service_cmd.go Normal file
View file

@ -0,0 +1,125 @@
package main
import (
"fmt"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/kardianos/service"
)
// systemdScript is the unit template used when installing the service. It
// replaces the library default to run as a notify service with automatic
// restart, so systemd only considers the mirror started once the listener
// is bound.
const systemdScript = `[Unit]
Description={{Description}}
ConditionFileIsExecutable={{Path | cmdEscape}}
{{range Dependencies}}{{.}}
{{end}}StartLimitIntervalSec=500
StartLimitBurst=5
[Service]
Type=notify
ExecStart={{Path | cmdEscape}}{{range Arguments}} {{. | cmd}}{{end}}
{{if ChRoot}}RootDirectory={{ChRoot | cmd}}
{{end}}{{if WorkingDirectory}}WorkingDirectory={{WorkingDirectory | cmdEscape}}
{{end}}{{if UserName}}User={{UserName}}
{{end}}{{if ReloadSignal}}ExecReload=/bin/kill -{{ReloadSignal}} "$MAINPID"
{{end}}{{if PIDFile}}PIDFile={{PIDFile | cmd}}
{{end}}{{if OutputFileSupport}}StandardOutput=file:{{LogDirectory}}/{{Name}}.out
StandardError=file:{{LogDirectory}}/{{Name}}.err
{{end}}{{if LimitNOFILE}}LimitNOFILE={{LimitNOFILE}}
{{end}}{{if Restart}}Restart={{Restart}}
{{end}}{{if SuccessExitStatus}}SuccessExitStatus={{SuccessExitStatus}}
{{end}}RestartSec=5
EnvironmentFile=-/etc/sysconfig/{{Name}}
{{range EnvVars}}{{.}}
{{end}}[Install]
WantedBy=multi-user.target
`
// ServiceAction lists the accepted service actions, in the order the Run
// switch handles them.
var ServiceAction = []string{"start", "stop", "status", "restart", "install", "uninstall"}
// ServiceCmd manages the mirror server as a system service.
type ServiceCmd struct {
Action string `arg:"" enum:"${serviceActions}" help:"${serviceActions}" required:""`
}
// action returns the requested service action.
func (s *ServiceCmd) action() string {
return s.Action
}
// Run performs the requested action against the installed service.
func (s *ServiceCmd) Run() (err error) {
svc, err := s.service()
if err != nil {
return err
}
switch s.action() {
case ServiceAction[0]:
err = svc.Start()
case ServiceAction[1]:
err = svc.Stop()
case ServiceAction[2]:
var status service.Status
status, err = svc.Status()
if err == nil {
switch status {
case service.StatusRunning:
fmt.Println("Service is running.")
case service.StatusStopped:
fmt.Println("Service is stopped.")
default:
fmt.Println("Service is in an unknown state.")
}
}
case ServiceAction[3]:
err = svc.Restart()
case ServiceAction[4]:
err = svc.Install()
case ServiceAction[5]:
err = svc.Uninstall()
}
if err != nil {
return err
}
// Status already printed its own result.
if s.action() != ServiceAction[2] {
fmt.Println("Command executed successfully.")
}
return
}
// service builds the service definition shared by the management actions
// and by the server when it is started by the service manager.
func (s *ServiceCmd) service() (service.Service, error) {
svcConfig := &service.Config{
Name: cfg.Name,
DisplayName: cfg.DisplayName,
Description: cfg.Description,
Arguments: []string{"server"},
Dependencies: []string{"After=network.target"},
Option: service.KeyValue{
"SystemdScript": systemdScript,
"ReloadSignal": "HUP",
"Restart": "always",
},
}
return service.New(s, svcConfig)
}
// Start satisfies service.Interface. The server is already running in the
// foreground by the time the supervisor attaches.
func (s *ServiceCmd) Start(svc service.Service) error {
return nil
}
// Stop satisfies service.Interface, signalling the server's shutdown.
func (s *ServiceCmd) Stop(svc service.Service) error {
stopChan <- struct{}{}
return nil
}

236
state/state.go Normal file
View file

@ -0,0 +1,236 @@
// Package state persists the mirror server's per-resource crawl tracking
// so discovered repositories survive restarts.
package state
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"gopkg.in/yaml.v3"
)
// Entry tracks one resource: a discovered or configured repository, or a
// generic cached file.
type Entry struct {
// Kind is the repository format, or "generic" for plain files.
Kind string `yaml:"kind"`
// Path is the request path of the resource (a deb entry uses the
// suite directory).
Path string `yaml:"path"`
// Root is the request path prefix covering every file the resource
// owns; requests below it keep the resource alive.
Root string `yaml:"root,omitempty"`
LastRequested time.Time `yaml:"last_requested,omitempty"`
LastCrawled time.Time `yaml:"last_crawled,omitempty"`
NextCrawl time.Time `yaml:"next_crawl,omitempty"`
LastSeenInInventory time.Time `yaml:"last_seen_in_inventory,omitempty"`
LastError string `yaml:"last_error,omitempty"`
}
// file is the on-disk YAML layout.
type file struct {
Entries map[string]*Entry `yaml:"entries"`
}
// Store is the in-memory state with dirty tracking for periodic flushes.
type Store struct {
mu sync.Mutex
file file
dirty bool
}
// S is the active store, installed by Load.
var S *Store
// Load reads the state file, treating a missing or empty file as a fresh
// store. Other read failures abort startup so a state file the process
// cannot read is never silently replaced.
func Load() error {
s := &Store{file: file{Entries: map[string]*Entry{}}}
name := cfg.C.StatePath
data, err := os.ReadFile(name)
switch {
case err == nil && len(data) > 0:
if err := yaml.Unmarshal(data, &s.file); err != nil {
return err
}
// A null or absent entries mapping unmarshals over the initialized
// map with a nil one, so the map is restored before any write.
if s.file.Entries == nil {
s.file.Entries = map[string]*Entry{}
}
// Drop null entries a hand-edited file may carry, as lookups
// dereference the stored pointer.
for key, e := range s.file.Entries {
if e == nil {
delete(s.file.Entries, key)
}
}
case err != nil && !errors.Is(err, fs.ErrNotExist):
return fmt.Errorf("read state %s: %w", name, err)
}
S = s
return nil
}
// MarkRequested records a client request for a resource, creating it if
// needed. It returns a copy of the entry and whether the resource was
// already tracked, so a caller can tell a registration this request created
// from one an earlier request established.
func (s *Store) MarkRequested(kind, key, path, root string, now time.Time) (Entry, bool) {
s.mu.Lock()
defer s.mu.Unlock()
e, tracked := s.file.Entries[key]
if !tracked {
e = &Entry{}
s.file.Entries[key] = e
}
e.Kind = kind
e.Path = path
e.Root = root
e.LastRequested = now
s.dirty = true
return *e, tracked
}
// MarkSeenInInventory records that a configured mount still advertises the
// resource, shielding it from eviction.
func (s *Store) MarkSeenInInventory(kind, key, path, root string, now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
e := s.file.Entries[key]
if e == nil {
e = &Entry{}
s.file.Entries[key] = e
}
e.Kind = kind
e.Path = path
e.Root = root
e.LastSeenInInventory = now
s.dirty = true
}
// MarkCrawled records a crawl outcome, scheduling the next attempt sooner
// after failures.
func (s *Store) MarkCrawled(key string, now time.Time, crawlErr error) {
s.mu.Lock()
defer s.mu.Unlock()
e := s.file.Entries[key]
if e == nil {
return
}
e.LastCrawled = now
if crawlErr != nil {
e.LastError = crawlErr.Error()
e.NextCrawl = now.Add(cfg.C.FailureRetryInterval)
} else {
e.LastError = ""
e.NextCrawl = now.Add(cfg.C.RepoCrawlInterval)
}
s.dirty = true
}
// Delete removes a resource from tracking.
func (s *Store) Delete(key string) {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.file.Entries[key]; ok {
delete(s.file.Entries, key)
s.dirty = true
}
}
// Entry returns a copy of a tracked resource.
func (s *Store) Entry(key string) (Entry, bool) {
s.mu.Lock()
defer s.mu.Unlock()
e, ok := s.file.Entries[key]
if !ok {
return Entry{}, false
}
return *e, true
}
// Snapshot returns a copy of every tracked resource.
func (s *Store) Snapshot() map[string]Entry {
s.mu.Lock()
defer s.mu.Unlock()
out := make(map[string]Entry, len(s.file.Entries))
for key, e := range s.file.Entries {
out[key] = *e
}
return out
}
// TouchRepoMembers refreshes the last-requested time of every repository
// whose root covers a request path, keeping repositories alive while their
// files are fetched. It reports whether any repository matched.
func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool {
s.mu.Lock()
defer s.mu.Unlock()
matched := false
for _, e := range s.file.Entries {
if e.Kind == "generic" || e.Root == "" {
continue
}
under := reqPath == e.Root || e.Root == "/" ||
(len(reqPath) > len(e.Root) && reqPath[:len(e.Root)] == e.Root && reqPath[len(e.Root)] == '/')
if !under {
continue
}
e.LastRequested = now
matched = true
s.dirty = true
}
return matched
}
// Save writes the state atomically when it has changed.
func (s *Store) Save() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.dirty {
return nil
}
data, err := yaml.Marshal(s.file)
if err != nil {
return err
}
name := cfg.C.StatePath
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
return err
}
tmp := name + ".tmp"
if err := os.WriteFile(tmp, data, 0644); err != nil {
return err
}
if err := os.Rename(tmp, name); err != nil {
return err
}
s.dirty = false
return nil
}
// FlushLoop saves periodically and once more at shutdown.
func (s *Store) FlushLoop(ctx context.Context) {
t := time.NewTicker(cfg.C.StateFlushInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
_ = s.Save()
return
case <-t.C:
_ = s.Save()
// Re-arm from the active config so a reload takes effect.
t.Reset(cfg.C.StateFlushInterval)
}
}
}

93
state/state_test.go Normal file
View file

@ -0,0 +1,93 @@
package state
import (
"os"
"path/filepath"
"testing"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
)
// initConfig installs a minimal configuration with the given state path.
func initConfig(t *testing.T, statePath string) {
t.Helper()
conf := `
state_path: ` + statePath + `
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://example.com"}
`
name := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(name, []byte(conf), 0644); err != nil {
t.Fatal(err)
}
if err := cfg.Init(name); err != nil {
t.Fatal(err)
}
}
// TestLoadMissing verifies a missing state file yields a fresh store.
func TestLoadMissing(t *testing.T) {
initConfig(t, filepath.Join(t.TempDir(), "state.yaml"))
if err := Load(); err != nil {
t.Fatal(err)
}
if len(S.Snapshot()) != 0 {
t.Error("fresh store is not empty")
}
}
// TestLoadUnreadable verifies read failures other than a missing file are
// reported to the caller.
func TestLoadUnreadable(t *testing.T) {
initConfig(t, t.TempDir())
if err := Load(); err == nil {
t.Fatal("expected an error for an unreadable state path")
}
}
// TestLoadNullEntryMap verifies a state file whose entries mapping is null
// yields a writable store.
func TestLoadNullEntryMap(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "state.yaml")
if err := os.WriteFile(statePath, []byte("entries:\n"), 0644); err != nil {
t.Fatal(err)
}
initConfig(t, statePath)
if err := Load(); err != nil {
t.Fatal(err)
}
S.MarkRequested("rpm", "rpm:/repo", "/repo", "/repo", time.Now())
if _, ok := S.Entry("rpm:/repo"); !ok {
t.Error("entry not recorded after loading a null entries mapping")
}
}
// TestLoadNullEntries verifies hand-edited null entries are dropped at load
// so later lookups only see valid entries.
func TestLoadNullEntries(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "state.yaml")
content := `
entries:
rpm:/repo: null
deb:/suite:
kind: deb
path: /suite
`
if err := os.WriteFile(statePath, []byte(content), 0644); err != nil {
t.Fatal(err)
}
initConfig(t, statePath)
if err := Load(); err != nil {
t.Fatal(err)
}
if _, ok := S.Entry("rpm:/repo"); ok {
t.Error("null entry survived load")
}
if _, ok := S.Entry("deb:/suite"); !ok {
t.Error("valid entry lost on load")
}
S.TouchRepoMembers("/repo/repodata/repomd.xml", time.Now())
}

233
sync_cmd.go Normal file
View file

@ -0,0 +1,233 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"os/signal"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/mirror"
)
// SyncArgs holds the flags and positional arguments shared by every
// repository type.
type SyncArgs struct {
Trim int `help:"Number of leading URL path components to drop when building the destination path." default:"0"`
Flat bool `help:"Place repositories directly in the destination directory without copying the URL path."`
Discover bool `help:"Treat each URL as a directory index and crawl it for repositories."`
Depth int `help:"Maximum directory depth crawled in discovery mode." default:"5"`
Workers int `help:"Number of concurrent download workers; defaults to the configured crawler workers."`
Verify bool `help:"Re-verify checksums of files that already exist locally."`
Prune bool `help:"Delete local files that are no longer part of the repository."`
Exclude []string `help:"Glob of directory or file names discovery never crawls; repeatable, and matched against the path below the crawled URL when it contains a slash, which a leading slash anchors to that URL (e.g. sles, 'yum/docker*', /docker)."`
IncludeFile []string `help:"Glob of loose files found outside repositories that discovery mirrors as well; repeatable, and matched by path on the same rules as --exclude (e.g. '*.rpm', 'RPM-GPG-KEY-*', '/*.rpm')."`
PruneGrace time.Duration `help:"Keep files that left the repository for this long before pruning them (e.g. 72h); defaults to the configured crawler prune grace."`
DryRun bool `help:"Report what would be downloaded or pruned without changing the repository."`
Missing string `help:"How to handle a package the repository metadata lists but the upstream does not serve: fail the repository, retry it across runs before accepting it as gone, or ignore it; defaults to the configured crawler mode." enum:",fail,retry,ignore" default:""`
MissingRetries int `help:"Number of consecutive runs a file must be missing upstream before --missing=retry accepts it as gone; defaults to the configured crawler value." default:"-1"`
Args []string `arg:"" name:"url" help:"Repository or mirrorlist URLs followed by the destination directory."`
}
// options validates the shared arguments and builds the mirror options.
// The types limit which repository formats the run accepts; an empty list
// accepts every supported format.
func (s *SyncArgs) options(types []mirror.RepoType) (*mirror.Options, error) {
if len(s.Args) < 2 {
return nil, errors.New("expected at least one repository URL and a destination directory")
}
urls := s.Args[:len(s.Args)-1]
for _, raw := range urls {
u, err := url.Parse(raw)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, fmt.Errorf("invalid repository URL %q", raw)
}
}
dest, err := filepath.Abs(s.Args[len(s.Args)-1])
if err != nil {
return nil, err
}
if s.Trim < 0 {
return nil, errors.New("trim must not be negative")
}
if s.Depth < 0 {
return nil, errors.New("depth must not be negative")
}
if s.PruneGrace < 0 {
return nil, errors.New("prune grace must not be negative")
}
// Reject unusable globs up front: a malformed pattern would otherwise
// silently match nothing, which reads as a repository that vanished.
for _, pattern := range slices.Concat(s.Exclude, s.IncludeFile) {
if err := mirror.ValidPattern(pattern); err != nil {
return nil, fmt.Errorf("invalid pattern %q: %w", pattern, err)
}
}
// The patterns describe a crawl, so without one they would do nothing.
if !s.Discover && (len(s.Exclude) > 0 || len(s.IncludeFile) > 0) {
return nil, errors.New("--exclude and --include-file apply to discovery; add --discover")
}
// Unset worker and grace flags fall back to the crawler configuration,
// so a config file tunes the sync commands the same way it tunes the
// server's background crawls.
if s.Workers == 0 {
s.Workers = cfg.C.Crawler.Workers
}
if s.PruneGrace == 0 {
s.PruneGrace = cfg.C.Crawler.PruneGrace
}
if s.Missing == "" {
s.Missing = cfg.C.Crawler.MissingMode
}
if s.MissingRetries < 0 {
s.MissingRetries = cfg.C.Crawler.MissingRetries
}
missing, err := fetch.ParseMissingMode(s.Missing)
if err != nil {
return nil, err
}
opts := &mirror.Options{
Types: types,
URLs: urls,
Destination: dest,
Trim: s.Trim,
Flat: s.Flat,
Discover: s.Discover,
DiscoverDepth: s.Depth,
Exclude: s.Exclude,
IncludeFiles: s.IncludeFile,
Workers: s.Workers,
Verify: s.Verify,
Prune: s.Prune,
PruneGrace: s.PruneGrace,
DryRun: s.DryRun,
Missing: missing,
MissingRetries: s.MissingRetries,
Trace: cfg.C.Trace.Enabled,
TraceInfo: mirror.TraceInfo{
Host: cfg.C.Trace.HostName(),
Maintainer: cfg.C.Trace.Maintainer,
Sponsor: cfg.C.Trace.Sponsor,
Country: cfg.C.Trace.Country,
Location: cfg.C.Trace.Location,
Throughput: cfg.C.Trace.Throughput,
},
}
// A run limited to one format synchronizes every URL as that format,
// rather than identifying each from its metadata.
if len(types) == 1 {
opts.Type = types[0]
}
return opts, nil
}
// run executes a synchronization with signal-aware cancellation, reporting
// what the run transferred once it ends.
func (s *SyncArgs) run(opts *mirror.Options) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
summary, err := mirror.Sync(ctx, opts)
printSummary(os.Stdout, summary)
return err
}
// printSummary reports the run totals as "Field: value" lines, in the style
// of rsync's own statistics, so a calling script can act on what changed by
// reading the output. It is written to standard output rather than through
// the logger, which a configuration is free to send elsewhere or format as
// JSON, and it is reported even for a failed run, as a run that failed part
// way through still changed the destination.
func printSummary(w io.Writer, summary *mirror.Summary) {
if summary == nil {
return
}
fmt.Fprintln(w)
fmt.Fprintf(w, "Number of repositories: %d\n", summary.Repositories)
fmt.Fprintf(w, "Number of failed repositories: %d\n", summary.Failed)
// A dry run transfers metadata to plan against, so reporting that as
// files transferred would read as a change. It reports what a real run
// would have moved instead.
if summary.DryRun {
fmt.Fprintf(w, "Dry run: true\n")
fmt.Fprintf(w, "Number of files to transfer: %d\n", summary.Planned)
fmt.Fprintf(w, "Total file size to transfer: %d bytes\n", summary.PlannedBytes)
} else {
fmt.Fprintf(w, "Number of files transferred: %d\n", summary.Fetched)
fmt.Fprintf(w, "Number of files pruned: %d\n", summary.Pruned)
fmt.Fprintf(w, "Total transferred file size: %d bytes\n", summary.FetchedBytes)
}
fmt.Fprintf(w, "Number of files unchanged: %d\n", summary.Unchanged)
fmt.Fprintf(w, "Number of files missing upstream: %d\n", summary.Missing)
fmt.Fprintf(w, "Repository changed: %t\n", summary.Changed())
}
// DebArgs holds the apt-specific filters, shared by the deb command and the
// type-agnostic sync command, which reaches apt repositories too.
type DebArgs struct {
Component []string `help:"Limit apt synchronization to these components."`
Arch []string `help:"Limit apt synchronization to these architectures; include \"source\" to keep source indexes."`
}
// apply copies the apt filters onto the mirror options.
func (d *DebArgs) apply(opts *mirror.Options) {
opts.Components = d.Component
opts.Architectures = d.Arch
}
// parseTypes converts requested type names into repository types, rejecting
// unsupported ones. An empty request accepts every supported format.
func parseTypes(names []string) ([]mirror.RepoType, error) {
var types []mirror.RepoType
for _, name := range names {
typ := mirror.RepoType(strings.ToLower(strings.TrimSpace(name)))
if !slices.Contains(mirror.AllTypes, typ) {
return nil, fmt.Errorf("unsupported repository type %q; expected one of %s",
name, strings.Join(mirror.TypeNames(), ", "))
}
if !slices.Contains(types, typ) {
types = append(types, typ)
}
}
return types, nil
}
// SyncCmd synchronizes repositories of any supported type. Each repository
// is identified by its own metadata, so one run covers an upstream that
// publishes several formats; --type limits which formats are accepted.
type SyncCmd struct {
Type []string `help:"Repository types to synchronize; repeatable, and defaults to every supported type."`
DebArgs `embed:""`
SyncArgs `embed:""`
}
// Run performs the synchronization.
func (c *SyncCmd) Run() error {
types, err := parseTypes(c.Type)
if err != nil {
return err
}
opts, err := c.options(types)
if err != nil {
return err
}
c.apply(opts)
return c.run(opts)
}

166
sync_cmd_test.go Normal file
View file

@ -0,0 +1,166 @@
package main
import (
"bytes"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/mirror"
)
// summaryFields parses a printed summary back into its fields.
func summaryFields(t *testing.T, out string) map[string]string {
t.Helper()
fields := make(map[string]string)
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
if line == "" {
continue
}
name, value, ok := strings.Cut(line, ": ")
if !ok {
t.Fatalf("summary line %q is not a field", line)
}
fields[name] = value
}
return fields
}
// TestPrintSummary verifies the printed summary reports what a calling script
// keys off, and that a dry run never reports a change.
func TestPrintSummary(t *testing.T) {
tests := []struct {
name string
summary *mirror.Summary
want map[string]string
absent []string
}{
{
name: "a run that transferred files reports a change",
summary: &mirror.Summary{
Repositories: 2,
Fetched: 7,
FetchedBytes: 4096,
Unchanged: 31,
Pruned: 2,
},
want: map[string]string{
"Number of repositories": "2",
"Number of failed repositories": "0",
"Number of files transferred": "7",
"Number of files pruned": "2",
"Total transferred file size": "4096 bytes",
"Number of files unchanged": "31",
"Repository changed": "true",
},
absent: []string{"Dry run", "Number of files to transfer"},
},
{
name: "an idle run reports no change",
summary: &mirror.Summary{Repositories: 1, Unchanged: 12},
want: map[string]string{
"Number of files transferred": "0",
"Number of files pruned": "0",
"Repository changed": "false",
},
},
{
name: "pruning alone is a change",
summary: &mirror.Summary{Repositories: 1, Pruned: 1},
want: map[string]string{
"Number of files pruned": "1",
"Repository changed": "true",
},
},
{
name: "a failed run still reports its totals",
summary: &mirror.Summary{Repositories: 3, Failed: 1, Fetched: 4},
want: map[string]string{
"Number of repositories": "3",
"Number of failed repositories": "1",
"Repository changed": "true",
},
},
{
name: "a dry run reports the work it planned but no change",
summary: &mirror.Summary{
Repositories: 1,
DryRun: true,
Fetched: 4,
Planned: 9,
PlannedBytes: 8192,
},
want: map[string]string{
"Dry run": "true",
"Number of files to transfer": "9",
"Total file size to transfer": "8192 bytes",
"Repository changed": "false",
},
// The metadata a dry run fetches must never be reported as
// files it transferred.
absent: []string{"Number of files transferred", "Number of files pruned"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var out bytes.Buffer
printSummary(&out, tc.summary)
got := summaryFields(t, out.String())
for name, want := range tc.want {
if got[name] != want {
t.Errorf("%s = %q, want %q", name, got[name], want)
}
}
for _, name := range tc.absent {
if _, ok := got[name]; ok {
t.Errorf("%s was reported when it does not apply", name)
}
}
})
}
}
// TestPrintSummaryNone verifies a run with no summary to report prints
// nothing, rather than an empty set of fields a script would misread.
func TestPrintSummaryNone(t *testing.T) {
var out bytes.Buffer
printSummary(&out, nil)
if out.Len() != 0 {
t.Errorf("printed %q without a summary", out.String())
}
}
// TestSyncArgsMissing verifies missing file handling falls back to the
// crawler configuration and that the flags override it.
func TestSyncArgsMissing(t *testing.T) {
dest := t.TempDir()
// An unset flag pair takes the configured defaults, which is what a run
// without a config file gets from the built-in ones.
args := SyncArgs{Args: []string{"https://example.com/repo", dest}, MissingRetries: -1}
opts, err := args.options(nil)
if err != nil {
t.Fatal(err)
}
if opts.Missing != fetch.MissingRetry || opts.MissingRetries != 3 {
t.Errorf("missing = %q after %d runs, want retry after 3", opts.Missing, opts.MissingRetries)
}
// Explicit flags win, including a zero retry budget, which accepts an
// absence the first time it is seen.
args = SyncArgs{Args: []string{"https://example.com/repo", dest}, Missing: "ignore", MissingRetries: 0}
opts, err = args.options(nil)
if err != nil {
t.Fatal(err)
}
if opts.Missing != fetch.MissingIgnore || opts.MissingRetries != 0 {
t.Errorf("missing = %q after %d runs, want ignore after 0", opts.Missing, opts.MissingRetries)
}
// An unsupported mode is rejected rather than silently ignored.
args = SyncArgs{Args: []string{"https://example.com/repo", dest}, Missing: "skip", MissingRetries: -1}
if _, err := args.options(nil); err == nil {
t.Error("an unsupported missing mode was accepted")
}
}