GRMrGecko/repo-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

first commit
Some checks are pending
Go package / build (push) Waiting to run
Details

Browse source
This commit is contained in:
James Coleman 2026-07-27 14:16:32 -05:00
commit b9cc35b1a1
53 changed files with 8033 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
.github/workflows/release.yaml vendored Normal file
View file

@ -0,0 +1,31 @@
on:
release:
types: [created]
permissions:
contents: write
packages: write
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v7
with:
fetch-depth: 0
-
name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
-
name: Run GoReleaser
uses: goreleaser/goreleaser-action@v7
with:
distribution: goreleaser
version: '~> v2'
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

21
.github/workflows/test_golang.yaml vendored Normal file
View file

@ -0,0 +1,21 @@
name: Go package
on: [push]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build
run: go build -v ./...
- name: Test
run: go test -v ./...

7
.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
/repo-sync
/dist/
*.exe
*.test
*.out
*.prof

61
.goreleaser.yaml Normal file
View file

@ -0,0 +1,61 @@
# GoReleaser config for repo-sync.
# https://goreleaser.com
#
# CGO is disabled so the binary is fully static (no glibc dependency) and runs
# unmodified across modern Linux distributions.
version: 2
project_name: repo-sync
before:
hooks:
- go mod tidy
- go test ./...
builds:
- id: repo-sync
main: .
binary: repo-sync
env:
- CGO_ENABLED=0
flags:
- -trimpath
ldflags:
- -s -w
- -X github.com/grmrgecko/repo-sync/config.Version={{ .Version }}
- -X github.com/grmrgecko/repo-sync/config.Commit={{ .ShortCommit }}
- -X github.com/grmrgecko/repo-sync/config.Date={{ .Date }}
- -X github.com/grmrgecko/repo-sync/config.Mode=release
goos:
- linux
goarch:
- "386"
- amd64
- arm
- arm64
- ppc64le
goarm:
- "6"
archives:
- id: default
formats: [tar.gz]
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
files:
- README.md
- LICENSE
checksum:
name_template: "checksums.txt"
snapshot:
version_template: "{{ incpatch .Version }}-snapshot"
changelog:
use: git
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
- "^chore:"

19
LICENSE Normal file
View file

@ -0,0 +1,19 @@
Copyright (c) 2026 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

37
Makefile Normal file
View file

@ -0,0 +1,37 @@
BINARY := repo-sync
MODULE := github.com/grmrgecko/repo-sync/config
# VERSION is the single source of truth for the version string. COMMIT and DATE
# are derived from git and the build clock.
VERSION ?= $(shell cat VERSION 2>/dev/null || echo dev)
COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null)
DATE := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
LDFLAGS := -s -w -X $(MODULE).Version=$(VERSION) -X $(MODULE).Commit=$(COMMIT) -X $(MODULE).Date=$(DATE)
.PHONY: all build test vet fmt snapshot release clean
all: test build
## build: native static binary into dist/
build:
CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(BINARY) .
## test: run the unit tests
test:
go test ./...
vet:
go vet ./...
fmt:
gofmt -w .
## snapshot: local GoReleaser build without publishing (artifacts in dist/)
snapshot:
goreleaser release --snapshot --clean
## release: full GoReleaser release (CI runs this on a tag)
release:
goreleaser release --clean
clean:
rm -rf dist

84
README.md Normal file
View file

@ -0,0 +1,84 @@
# repo-sync
A universal Linux package repository synchronization tool. It mirrors remote
repositories into a local directory tree, preserving the upstream layout so
the result can be served directly to package managers.
Supported repository types:
- **rpm** — yum/dnf repositories (`repodata/repomd.xml`), including
plain-text mirrorlist and metalink URLs with failover between mirrors.
- **deb** — apt repositories, both standard (`dists/<suite>` with a shared
`pool/`) and flat layouts, including `Acquire-By-Hash` population.
- **arch** — pacman repositories (`<name>.db`), including detached package
signatures and companion metadata (`.files`, `.db.tar.gz`, `.links.tar.gz`).
- **apk** — Alpine Linux repositories (`APKINDEX.tar.gz` with packages
beside it).
## Usage
```
repo-sync <type> [flags] <url> [<url> ...] <destination-directory>
```
The last argument is always the destination directory; every argument before
it is a repository or mirrorlist URL, synchronized one after the other.
```sh
# Mirror one repository. The URL path is copied below the destination, so
# this produces ./mirror/repos/CentOS/7/EA4/.
repo-sync rpm https://example.com/repos/CentOS/7/EA4/ ./mirror
# Trim the first two path components: ./mirror/7/EA4/.
repo-sync rpm --trim 2 https://example.com/repos/CentOS/7/EA4/ ./mirror
# No path copying at all: the repository lands directly in ./mirror.
repo-sync rpm --flat https://example.com/repos/CentOS/7/EA4/ ./mirror
# Multiple repositories in one run.
repo-sync rpm https://example.com/repos/a/ https://example.com/repos/b/ ./mirror
# Crawl directory listings for repositories, at most 4 levels deep.
repo-sync rpm --discover --depth 4 https://example.com/repos/ ./mirror
# A pacman repository. The database name is discovered from the directory
# listing, or by probing URL path segments when listings are disabled.
repo-sync arch https://example.com/archlinux/core/os/x86_64/ ./mirror
# An Alpine repository is one arch directory; discovery syncs every arch
# of a release/repo tree in one run.
repo-sync apk https://dl-cdn.alpinelinux.org/alpine/v3.24/community/x86_64/ ./mirror
repo-sync apk --discover --depth 1 https://dl-cdn.alpinelinux.org/alpine/v3.24/community/ ./mirror
# A Fedora-style metalink URL; mirrors are used in preference order. The
# mirrors' paths differ, so --flat keeps the destination stable.
repo-sync rpm --flat 'https://mirrors.fedoraproject.org/metalink?repo=epel-9&arch=x86_64' ./mirror/epel9
# An apt suite. Pool files resolve against the archive root, producing
# ./mirror/debian/dists/bookworm/ and ./mirror/debian/pool/.
repo-sync deb https://deb.example.com/debian/dists/bookworm ./mirror
# Limit an apt mirror to specific components and architectures. Include
# "source" as an architecture to keep source indexes.
repo-sync deb --component main --arch amd64,source https://deb.example.com/debian/dists/bookworm ./mirror
```
## Building
```sh
make
```
The build stamps the binary with the contents of the `VERSION` file plus the
git commit and build date, shown by `repo-sync --version`. A plain
`go build` also works but reports the version as `dev`.
## Testing
```sh
make test
```
The suite is hermetic: fixture repositories for every supported format
(rpm, deb, arch, apk) are generated in temp directories and served over
local HTTP, so no network access is required.

1
VERSION Normal file
View file

@ -0,0 +1 @@
0.1.0

109
config.example.yaml Normal file
View file

@ -0,0 +1,109 @@
# Example repo-sync mirror server configuration.
#
# The server proxies and caches upstream package mirrors. Repositories are
# discovered from client requests: fetching a repository's entry-point
# metadata (repodata/repomd.xml, dists/<suite>/InRelease, <name>.db,
# APKINDEX.tar.gz) registers it for full synchronization and periodic
# refreshing. Repositories listed under a mount's `repos` are kept
# synchronized regardless of traffic.
# state_path persists discovered repositories across restarts. When unset
# it defaults to state.yaml next to the config file (./config.yaml keeps
# state in ./state.yaml), or /etc/repo-sync/state.yaml without one.
state_path: /etc/repo-sync/state.yaml
state_flush_interval: 30s
# repo_crawl_interval is how often known repositories are refreshed while
# they keep being requested. failure_retry_interval schedules the next
# attempt after a failed crawl. crawl_check_interval is the scheduler tick.
repo_crawl_interval: 4h
failure_retry_interval: 1h
crawl_check_interval: 30m
# generic_max_age bounds how stale a plain cached file (anything outside a
# repository) may be before it is revalidated upstream. Upstream 404s are
# remembered for negative_cache_ttl and transient errors for
# transient_error_ttl so failures are not amplified per client.
generic_max_age: 6h
negative_cache_ttl: 1h
transient_error_ttl: 1m
http:
bind_addr: ""
port: 8080
read_header_timeout: 10s
read_timeout: 5m
write_timeout: 30m
idle_timeout: 2m
# directory_indexes enables directory listings: upstream index pages are
# passed through, and when no catch-all "/" mount exists the root serves
# a generated index of the mounts and their configured repositories.
# When false, directory requests answer with a static notice page.
directory_indexes: true
log:
# Severity threshold: debug, info, warn, error.
level: info
# Formatter: console (text) or json.
type: console
# Outputs may include "console" (stderr) or explicit file paths.
outputs:
- console
# domains map the request Host header to a filesystem tree. Exactly one
# domain has the online role: its tree is the writable cache that misses
# are fetched into and crawls write to. Offline domains are read-only
# published trees that fall back to the online tree for anything missing,
# and are promoted out-of-band (e.g. by snapshot tooling).
domains:
- domain: mirror.example.com
role: online
root: /var/lib/repo-sync/online
- domain: stage.mirror.example.com
role: offline
root: /var/lib/repo-sync/stage
# mounts map base request paths onto upstream mirrors; the longest matching
# path wins, and "/" is allowed as a catch-all. A request for
# /almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml resolves through the
# /almalinux mount and registers that path as an RPM repository. Each mount
# may also pin repositories that stay synchronized without client traffic.
mounts:
- path: /almalinux
upstream: https://mirror.example.com/almalinux
repos:
- path: /almalinux/9/BaseOS/x86_64/os
type: rpm
- path: /centos
upstream: https://vault.centos.org
- path: /ubuntu
upstream: https://mirror.example.com/ubuntu
- path: /archlinux
upstream: https://mirror.example.com/archlinux
- path: /alpine
upstream: https://dl-cdn.alpinelinux.org/alpine
crawler:
# workers is the number of download workers inside a single crawl;
# concurrent_crawls bounds how many repository crawls run at once.
workers: 4
concurrent_crawls: 4
# request_timeout bounds how long an upstream may take to start
# responding to a request (response header timeout).
request_timeout: 2m
# user_agent overrides the default repo-sync/<version>.
user_agent: ""
# prune_grace keeps files that left a repository for this long before
# crawl-time pruning removes them.
prune_grace: 24h
# Backoff steps applied once a repository stops being requested. The
# first tier is repo_crawl_interval; each entry extends the refresh
# interval as the resource ages, and once the total budget elapses the
# resource is evicted from the online cache.
refresh_schedule:
- 12h
- 24h
- 48h
- 72h
- 96h
- 120h

380
config/config.go Normal file
View file

@ -0,0 +1,380 @@
package config
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/go-playground/validator/v10"
"github.com/spf13/viper"
)
// Domain roles: the online domain is the writable upstream cache, offline
// domains are read-only published trees that fall back to the online tree.
const (
RoleOnline = "online"
RoleOffline = "offline"
)
// HTTPConfig configures the mirror server listener.
type HTTPConfig struct {
BindAddr string `mapstructure:"bind_addr" yaml:"bind_addr"`
Port uint `mapstructure:"port" yaml:"port" validate:"gt=0,lte=65535"`
ReadHeaderTimeout time.Duration `mapstructure:"read_header_timeout" yaml:"read_header_timeout" validate:"gt=0"`
ReadTimeout time.Duration `mapstructure:"read_timeout" yaml:"read_timeout" validate:"gt=0"`
WriteTimeout time.Duration `mapstructure:"write_timeout" yaml:"write_timeout" validate:"gt=0"`
IdleTimeout time.Duration `mapstructure:"idle_timeout" yaml:"idle_timeout" validate:"gt=0"`
// DirectoryIndexes enables directory listings: upstream index pages are
// passed through, and the root serves a generated index of the mounts
// when no catch-all mount covers it. When disabled, every directory
// request answers with a static notice page instead.
DirectoryIndexes bool `mapstructure:"directory_indexes" yaml:"directory_indexes"`
}
// ListenAddr renders the bind address and port for net.Listen, bracketing
// IPv6 addresses as required.
func (h HTTPConfig) ListenAddr() string {
return net.JoinHostPort(h.BindAddr, strconv.Itoa(int(h.Port)))
}
// LogConfig configures the server's logging output.
type LogConfig struct {
Level string `mapstructure:"level" yaml:"level" validate:"oneof=debug info warn error"`
Type string `mapstructure:"type" yaml:"type" validate:"oneof=console json"`
Outputs []string `mapstructure:"outputs" yaml:"outputs" validate:"required,min=1,dive,required"`
}
// DomainConfig maps a request Host header to a filesystem root and role.
type DomainConfig struct {
Domain string `mapstructure:"domain" yaml:"domain" validate:"required"`
Root string `mapstructure:"root" yaml:"root" validate:"required"`
Role string `mapstructure:"role" yaml:"role" validate:"required,oneof=online offline"`
}
// MountRepoConfig is a repository kept synchronized regardless of client
// traffic, identified by its request path and format.
type MountRepoConfig struct {
Path string `mapstructure:"path" yaml:"path" validate:"required"`
Type string `mapstructure:"type" yaml:"type" validate:"required,oneof=rpm deb arch apk"`
}
// MountConfig maps a base request path onto an upstream mirror URL.
type MountConfig struct {
Path string `mapstructure:"path" yaml:"path" validate:"required"`
Upstream string `mapstructure:"upstream" yaml:"upstream" validate:"required,url"`
Repos []MountRepoConfig `mapstructure:"repos" yaml:"repos" validate:"omitempty,dive"`
}
// CrawlerConfig tunes the background synchronization behavior.
type CrawlerConfig struct {
Workers int `mapstructure:"workers" yaml:"workers" validate:"gt=0"`
ConcurrentCrawls int `mapstructure:"concurrent_crawls" yaml:"concurrent_crawls" validate:"gt=0"`
RequestTimeout time.Duration `mapstructure:"request_timeout" yaml:"request_timeout" validate:"gt=0"`
UserAgent string `mapstructure:"user_agent" yaml:"user_agent"`
PruneGrace time.Duration `mapstructure:"prune_grace" yaml:"prune_grace" validate:"gte=0"`
RefreshSchedule []time.Duration `mapstructure:"refresh_schedule" yaml:"refresh_schedule" validate:"required,min=1,dive,gt=0"`
}
// Config is the mirror server configuration.
type Config struct {
HTTP HTTPConfig `mapstructure:"http" yaml:"http"`
Log LogConfig `mapstructure:"log" yaml:"log"`
StatePath string `mapstructure:"state_path" yaml:"state_path" validate:"required"`
StateFlushInterval time.Duration `mapstructure:"state_flush_interval" yaml:"state_flush_interval" validate:"gt=0"`
RepoCrawlInterval time.Duration `mapstructure:"repo_crawl_interval" yaml:"repo_crawl_interval" validate:"gt=0"`
FailureRetryInterval time.Duration `mapstructure:"failure_retry_interval" yaml:"failure_retry_interval" validate:"gt=0"`
CrawlCheckInterval time.Duration `mapstructure:"crawl_check_interval" yaml:"crawl_check_interval" validate:"gt=0"`
GenericMaxAge time.Duration `mapstructure:"generic_max_age" yaml:"generic_max_age" validate:"gt=0"`
NegativeCacheTTL time.Duration `mapstructure:"negative_cache_ttl" yaml:"negative_cache_ttl" validate:"gt=0"`
TransientErrorTTL time.Duration `mapstructure:"transient_error_ttl" yaml:"transient_error_ttl" validate:"gt=0"`
Domains []DomainConfig `mapstructure:"domains" yaml:"domains" validate:"required,min=1,dive"`
Mounts []MountConfig `mapstructure:"mounts" yaml:"mounts" validate:"required,min=1,dive"`
Crawler CrawlerConfig `mapstructure:"crawler" yaml:"crawler"`
domainsByName map[string]int
onlineIndex int
mountOrder []int
}
// C is the active server configuration, replaced whole on reload. It is
// atomic because a SIGHUP reload swaps it while requests are being served.
var C atomic.Pointer[Config]
// Load reads and validates the configuration without installing it, so the
// caller can apply overrides before the config becomes visible to serving
// goroutines.
func Load(configPath string) (*Config, error) {
v := viper.New()
setDefaults(v)
file := findConfig(configPath)
if file != "" {
v.SetConfigFile(file)
if err := v.ReadInConfig(); err != nil {
return nil, fmt.Errorf("read config %s: %w", file, err)
}
}
c := &Config{}
if err := v.Unmarshal(c); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
// Default the state file next to the config file, so ./config.yaml
// keeps state in ./state.yaml, or to the system path without one.
if c.StatePath == "" {
if file != "" {
c.StatePath = filepath.Join(filepath.Dir(file), "state.yaml")
} else {
c.StatePath = "/etc/repo-sync/state.yaml"
}
}
if err := c.finalize(); err != nil {
return nil, err
}
return c, nil
}
// Init loads and validates the configuration, then installs it as the
// active configuration.
func Init(configPath string) error {
c, err := Load(configPath)
if err != nil {
return err
}
C.Store(c)
return nil
}
// findConfig resolves the configuration file path: the explicit flag, the
// working directory, the user config directory, then the system directory.
func findConfig(explicit string) string {
if explicit != "" {
return explicit
}
candidates := []string{"config.yaml"}
if home, err := os.UserHomeDir(); err == nil {
candidates = append(candidates, filepath.Join(home, ".config", "repo-sync", "config.yaml"))
}
candidates = append(candidates, "/etc/repo-sync/config.yaml")
for _, name := range candidates {
if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() {
return name
}
}
return ""
}
// setDefaults registers every configuration default.
func setDefaults(v *viper.Viper) {
v.SetDefault("state_flush_interval", 30*time.Second)
v.SetDefault("repo_crawl_interval", 4*time.Hour)
v.SetDefault("failure_retry_interval", time.Hour)
v.SetDefault("crawl_check_interval", 30*time.Minute)
v.SetDefault("generic_max_age", 6*time.Hour)
v.SetDefault("negative_cache_ttl", time.Hour)
v.SetDefault("transient_error_ttl", time.Minute)
v.SetDefault("http.port", 8080)
v.SetDefault("http.directory_indexes", true)
v.SetDefault("http.read_header_timeout", 10*time.Second)
v.SetDefault("http.read_timeout", 5*time.Minute)
v.SetDefault("http.write_timeout", 30*time.Minute)
v.SetDefault("http.idle_timeout", 2*time.Minute)
v.SetDefault("log.level", "info")
v.SetDefault("log.type", "console")
v.SetDefault("log.outputs", []string{"console"})
v.SetDefault("crawler.workers", 4)
v.SetDefault("crawler.concurrent_crawls", 4)
v.SetDefault("crawler.request_timeout", 2*time.Minute)
v.SetDefault("crawler.prune_grace", 0)
v.SetDefault("crawler.refresh_schedule", []time.Duration{
12 * time.Hour, 24 * time.Hour, 48 * time.Hour,
72 * time.Hour, 96 * time.Hour, 120 * time.Hour,
})
}
// finalize canonicalizes, validates, and indexes a parsed configuration.
func (c *Config) finalize() error {
c.canonicalize()
if err := validateConfig(c); err != nil {
return err
}
if err := c.indexDomains(); err != nil {
return err
}
return c.indexMounts()
}
// canonicalize normalizes user-supplied values before validation.
func (c *Config) canonicalize() {
c.StatePath = expandHome(strings.TrimSpace(c.StatePath))
for i := range c.Domains {
c.Domains[i].Domain = strings.ToLower(strings.TrimSpace(c.Domains[i].Domain))
c.Domains[i].Role = strings.ToLower(strings.TrimSpace(c.Domains[i].Role))
c.Domains[i].Root = expandHome(strings.TrimSpace(c.Domains[i].Root))
}
for i := range c.Mounts {
c.Mounts[i].Path = cleanRequestPath(c.Mounts[i].Path)
c.Mounts[i].Upstream = strings.TrimRight(strings.TrimSpace(c.Mounts[i].Upstream), "/")
for j := range c.Mounts[i].Repos {
c.Mounts[i].Repos[j].Path = cleanRequestPath(c.Mounts[i].Repos[j].Path)
c.Mounts[i].Repos[j].Type = strings.ToLower(strings.TrimSpace(c.Mounts[i].Repos[j].Type))
}
}
}
// cleanRequestPath normalizes a request path to a single leading slash with
// no trailing slash.
func cleanRequestPath(p string) string {
return path.Clean("/" + strings.Trim(strings.TrimSpace(p), "/"))
}
// expandHome resolves a leading ~ against the user's home directory.
func expandHome(p string) string {
if p == "~" || strings.HasPrefix(p, "~/") {
if home, err := os.UserHomeDir(); err == nil {
return filepath.Join(home, strings.TrimPrefix(p, "~"))
}
}
return p
}
// validateConfig applies the declarative validation rules, reporting
// failures by their YAML key names.
func validateConfig(c *Config) error {
validate := validator.New()
validate.RegisterTagNameFunc(func(field reflect.StructField) string {
name := strings.SplitN(field.Tag.Get("mapstructure"), ",", 2)[0]
if name == "-" {
return ""
}
return name
})
err := validate.Struct(c)
if err == nil {
return nil
}
var verrs validator.ValidationErrors
if !errors.As(err, &verrs) {
return err
}
var fields []string
for _, verr := range verrs {
fields = append(fields, fmt.Sprintf("%s (%s)", strings.ToLower(verr.Namespace()), verr.Tag()))
}
return fmt.Errorf("invalid configuration: %s", strings.Join(fields, ", "))
}
// indexDomains builds the host lookup and enforces exactly one online
// domain.
func (c *Config) indexDomains() error {
c.domainsByName = map[string]int{}
c.onlineIndex = -1
for i := range c.Domains {
d := c.Domains[i]
if _, ok := c.domainsByName[d.Domain]; ok {
return fmt.Errorf("domain %s is configured more than once", d.Domain)
}
c.domainsByName[d.Domain] = i
if d.Role == RoleOnline {
if c.onlineIndex != -1 {
return fmt.Errorf("only one online domain may be configured")
}
c.onlineIndex = i
}
}
if c.onlineIndex == -1 {
return fmt.Errorf("one online domain must be configured")
}
return nil
}
// indexMounts validates mount uniqueness and upstream URLs, orders mounts
// longest path first for prefix lookup, and checks configured repositories
// sit under their mount.
func (c *Config) indexMounts() error {
seen := map[string]bool{}
for i := range c.Mounts {
m := &c.Mounts[i]
if seen[m.Path] {
return fmt.Errorf("mount path %s is configured more than once", m.Path)
}
seen[m.Path] = true
u, err := url.Parse(m.Upstream)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("mount %s upstream must include scheme and host", m.Path)
}
// Request URLs are built by joining a relative path onto the
// upstream; a query or fragment would swallow that path, so reject
// them rather than compose a malformed URL later.
if u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("mount %s upstream must not contain a query or fragment", m.Path)
}
for _, repo := range m.Repos {
if !pathUnder(repo.Path, m.Path) {
return fmt.Errorf("repository %s is not under mount %s", repo.Path, m.Path)
}
}
}
c.mountOrder = make([]int, len(c.Mounts))
for i := range c.mountOrder {
c.mountOrder[i] = i
}
sort.Slice(c.mountOrder, func(a, b int) bool {
return len(c.Mounts[c.mountOrder[a]].Path) > len(c.Mounts[c.mountOrder[b]].Path)
})
return nil
}
// pathUnder reports whether a request path equals or sits below a base
// path.
func pathUnder(p, base string) bool {
if base == "/" {
return true
}
return p == base || strings.HasPrefix(p, base+"/")
}
// Domain looks up the configuration for a request Host header. The port is
// stripped and IPv6 literals are unbracketed before the lookup.
func (c *Config) Domain(host string) (DomainConfig, bool) {
host = strings.ToLower(strings.TrimSpace(host))
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
host = strings.Trim(host, "[]")
i, ok := c.domainsByName[host]
if !ok {
return DomainConfig{}, false
}
return c.Domains[i], true
}
// OnlineDomain returns the single writable cache domain.
func (c *Config) OnlineDomain() DomainConfig {
return c.Domains[c.onlineIndex]
}
// MountFor finds the longest mount matching a request path.
func (c *Config) MountFor(reqPath string) (MountConfig, bool) {
for _, i := range c.mountOrder {
if pathUnder(reqPath, c.Mounts[i].Path) {
return c.Mounts[i], true
}
}
return MountConfig{}, false
}

126
config/config_test.go Normal file
View file

@ -0,0 +1,126 @@
package config
import (
"os"
"path/filepath"
"testing"
)
// writeConfig writes a temp config file and returns its path.
func writeConfig(t *testing.T, content string) string {
t.Helper()
name := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(name, []byte(content), 0644); err != nil {
t.Fatal(err)
}
return name
}
// TestInitExample verifies the shipped example configuration is valid.
func TestInitExample(t *testing.T) {
if err := Init("../config.example.yaml"); err != nil {
t.Fatal(err)
}
if _, ok := C.Load().Domain("mirror.example.com:443"); !ok {
t.Error("online domain not resolvable with a port")
}
if C.Load().OnlineDomain().Domain != "mirror.example.com" {
t.Errorf("online domain = %q", C.Load().OnlineDomain().Domain)
}
mount, ok := C.Load().MountFor("/almalinux/9/BaseOS/x86_64/os")
if !ok || mount.Path != "/almalinux" {
t.Errorf("mount = %+v, %v", mount, ok)
}
}
// TestStatePathDefault verifies an unset state path lands next to the
// config file.
func TestStatePathDefault(t *testing.T) {
path := writeConfig(t, `
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://example.com"}
`)
if err := Init(path); err != nil {
t.Fatal(err)
}
want := filepath.Join(filepath.Dir(path), "state.yaml")
if C.Load().StatePath != want {
t.Errorf("StatePath = %q, want %q", C.Load().StatePath, want)
}
}
// TestMountLongestPrefix verifies the most specific mount wins.
func TestMountLongestPrefix(t *testing.T) {
path := writeConfig(t, `
state_path: /tmp/state.yaml
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://fallback.example.com"}
- {path: /centos, upstream: "https://centos.example.com"}
- {path: /centos/vault, upstream: "https://vault.example.com"}
`)
if err := Init(path); err != nil {
t.Fatal(err)
}
cases := map[string]string{
"/centos/vault/7/os": "https://vault.example.com",
"/centos/8/BaseOS": "https://centos.example.com",
"/debian/dists/testy": "https://fallback.example.com",
}
for reqPath, want := range cases {
mount, ok := C.Load().MountFor(reqPath)
if !ok || mount.Upstream != want {
t.Errorf("MountFor(%q) = %q, want %q", reqPath, mount.Upstream, want)
}
}
}
// TestInitRejections verifies invalid configurations fail with clear
// errors.
func TestInitRejections(t *testing.T) {
cases := map[string]string{
"two online domains": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
- {domain: b.test, role: online, root: /tmp/b}
mounts:
- {path: /, upstream: "https://example.com"}
`,
"no online domain": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: offline, root: /tmp/a}
mounts:
- {path: /, upstream: "https://example.com"}
`,
"repo outside mount": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
mounts:
- path: /centos
upstream: "https://example.com"
repos:
- {path: /debian/dists/x, type: deb}
`,
"bad repo type": `
state_path: /tmp/state.yaml
domains:
- {domain: a.test, role: online, root: /tmp/a}
mounts:
- path: /centos
upstream: "https://example.com"
repos:
- {path: /centos/7/os, type: gentoo}
`,
}
for name, content := range cases {
if err := Init(writeConfig(t, content)); err == nil {
t.Errorf("%s: configuration unexpectedly accepted", name)
}
}
}

19
config/info.go Normal file
View file

@ -0,0 +1,19 @@
// Package config loads and validates the mirror server configuration,
// publishes it for reload-aware readers, configures logging, and carries
// the application identity and build identifiers.
package config
// Build identifiers populated at build time via -ldflags.
var (
Version = "dev"
Commit = ""
Date = ""
Mode = "dev"
)
// Application identifiers used by the CLI and the upstream user agent.
const (
Name = "repo-sync"
DisplayName = "Repository Sync"
Description = "Synchronize Linux package repositories into a local directory tree."
)

56
config/log.go Normal file
View file

@ -0,0 +1,56 @@
package config
import (
"io"
"os"
log "github.com/sirupsen/logrus"
)
// logFiles holds the files opened by the previous SetupLogging call so a
// reload can close them once the new outputs are installed.
var logFiles []*os.File
// SetupLogging applies the configured log level, formatter, and outputs to
// the global logger. Unknown output paths are treated as files opened for
// appending; failures fall back to stderr so the server always logs.
func SetupLogging(lc LogConfig) {
level, err := log.ParseLevel(lc.Level)
if err != nil {
level = log.InfoLevel
}
log.SetLevel(level)
if lc.Type == "json" {
log.SetFormatter(&log.JSONFormatter{})
} else {
log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
}
var writers []io.Writer
var files []*os.File
for _, output := range lc.Outputs {
if output == "console" {
writers = append(writers, os.Stderr)
continue
}
f, err := os.OpenFile(expandHome(output), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.WithError(err).WithField("output", output).Warn("Unable to open log output; skipping.")
continue
}
writers = append(writers, f)
files = append(files, f)
}
if len(writers) == 0 {
writers = append(writers, os.Stderr)
}
log.SetOutput(io.MultiWriter(writers...))
// Close the previous reload's files only after the new writer is
// installed; the logger serializes writes, so none remain in flight.
for _, f := range logFiles {
f.Close()
}
logFiles = files
}

795
fetch/fetch.go Normal file
View file

@ -0,0 +1,795 @@
// Package fetch downloads repository files over HTTP with mirror failover,
// verifying published sizes and checksums, staging updates so a live tree
// is never served a partially updated repository, and pruning files the
// upstream no longer lists.
package fetch
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"hash/fnv"
"io"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/klauspost/compress/zstd"
log "github.com/sirupsen/logrus"
"github.com/ulikunitz/xz"
)
// client is the shared HTTP client used for all repository fetches. It is
// atomic because a SIGHUP reload swaps it while requests are in flight.
var client atomic.Pointer[http.Client]
// headerTimeout bounds how long an upstream may take to start responding,
// stored as nanoseconds. It is atomic because a reload may change it while
// requests are in flight.
var headerTimeout atomic.Int64
func init() {
headerTimeout.Store(int64(30 * time.Second))
// Build a client immediately so the package is safe to use without an
// explicit Reload.
Reload()
}
// SetRequestTimeout replaces the upstream response header timeout applied
// by the next Reload. Non-positive durations keep the current value.
func SetRequestTimeout(d time.Duration) {
if d > 0 {
headerTimeout.Store(int64(d))
}
}
// Reload rebuilds the shared HTTP client. A header timeout is used instead
// of a whole-request timeout so large package downloads are not cut short.
func Reload() {
old := client.Load()
client.Store(&http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
ResponseHeaderTimeout: time.Duration(headerTimeout.Load()),
},
})
// Release the replaced transport's idle pool; in-flight requests on it
// finish undisturbed.
if old != nil {
old.CloseIdleConnections()
}
}
// ErrNotFound marks an upstream file that does not exist, letting callers
// treat missing optional files differently from transport failures.
var ErrNotFound = errors.New("upstream file not found")
// ErrNotModified marks a conditional request the upstream answered with
// 304, meaning the local copy is already current.
var ErrNotModified = errors.New("upstream file not modified")
// StagedSuffix is appended to staged downloads until they are promoted.
const StagedSuffix = ".staged"
// PartialSuffix is appended to in-progress downloads. Partials of files
// with published checksums survive failures so the next run can resume.
const PartialSuffix = ".partial"
// LockFileName is the flock target guarding a destination directory.
const LockFileName = ".repo-sync.lock"
// PruneStateName is the prune grace bookkeeping file kept at the root of a
// pruned tree.
const PruneStateName = ".repo-sync-prune.json"
// Expect describes the size and checksums a fetched file must match. A size
// of -1 means the size is unknown, and Sums maps lowercase algorithm names
// to hex digests.
type Expect struct {
Size int64
Sums map[string]string
}
// MakeExpect builds an Expect from a size and a single checksum, treating
// non-positive sizes and empty checksums as unknown.
func MakeExpect(size int64, algo, digest string) *Expect {
e := &Expect{Size: size, Sums: map[string]string{}}
if size <= 0 {
e.Size = -1
}
if algo != "" && digest != "" {
e.Sums[strings.ToLower(algo)] = strings.ToLower(digest)
}
return e
}
// hashOrder lists checksum algorithms from most to least preferred.
var hashOrder = []string{"sha512", "sha384", "sha256", "sha224", "sha1", "sha", "md5", "md5sum"}
// newHasher returns a hash implementation for a checksum algorithm name as
// found in repository metadata, or nil when the algorithm is unknown.
func newHasher(algo string) hash.Hash {
switch strings.ToLower(algo) {
case "md5", "md5sum":
return md5.New()
case "sha", "sha1":
return sha1.New()
case "sha224":
return sha256.New224()
case "sha256":
return sha256.New()
case "sha384":
return sha512.New384()
case "sha512":
return sha512.New()
}
return nil
}
// bestSum picks the strongest checksum available in the expectation.
func (e *Expect) bestSum() (string, string) {
for _, algo := range hashOrder {
if digest, ok := e.Sums[algo]; ok && digest != "" {
return algo, digest
}
}
return "", ""
}
// FileState reports what File did with a file. It is only meaningful when
// the accompanying error is nil.
type FileState int
const (
// FileMissing marks an optional file the upstream does not serve.
FileMissing FileState = iota
// FileUnchanged marks an existing local file that matched expectations.
FileUnchanged
// FileFetched marks a file downloaded into its final location.
FileFetched
// FileStaged marks a file downloaded next to its final location for a
// later promote.
FileStaged
)
// verifyLocal reports whether an existing regular file already matches the
// expected size and, when checkHash is set, the expected checksum.
func verifyLocal(name string, want *Expect, checkHash bool) bool {
info, err := os.Stat(name)
if err != nil || !info.Mode().IsRegular() {
return false
}
if want.Size >= 0 && info.Size() != want.Size {
return false
}
if !checkHash {
return true
}
algo, digest := want.bestSum()
if digest == "" {
return true
}
sum, err := hashFile(name, algo)
if err != nil {
return false
}
return sum == digest
}
// hashFile computes the named checksum of a file, returned as lowercase hex.
func hashFile(name, algo string) (string, error) {
h := newHasher(algo)
if h == nil {
return "", fmt.Errorf("unsupported checksum algorithm %q", algo)
}
f, err := os.Open(name)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// fileStripes bounds the destination lock table. Downloads to the same
// destination are serialized by stripe; a collision between unrelated
// destinations only delays a download briefly.
const fileStripes = 512
// fileLocks serializes downloads by destination path. The mirror server's
// on-demand fetches and background crawls can target the same file
// concurrently, and their partial downloads must not interleave.
var fileLocks [fileStripes]sync.Mutex
// lockFile holds the lock for a destination path and returns its unlock
// function.
func lockFile(dst string) func() {
h := fnv.New32a()
_, _ = io.WriteString(h, dst)
m := &fileLocks[h.Sum32()%fileStripes]
m.Lock()
return m.Unlock
}
// File downloads reqPath from the source into dst, verifying size and
// checksum when known. Existing files that already match are left alone.
// With stage set the download is written next to dst for a later promote so
// readers of a live tree never see a partially updated repository.
func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting bool) (FileState, error) {
// Serialize writers to this destination so concurrent callers, such
// as the mirror server's on-demand fetches and background crawls,
// cannot interleave partial downloads of the same file.
unlock := lockFile(dst)
defer unlock()
// Clear any staged leftover from an interrupted run so a later promote
// can never publish stale content.
if stage {
_ = os.Remove(dst + StagedSuffix)
}
// Skip files that already match the expected content.
if want != nil && verifyLocal(dst, want, verifyExisting) {
Stats.Unchanged.Add(1)
return FileUnchanged, nil
}
// Checksummed files with a known size can resume a failed download's
// partial file with a ranged request.
var resumeFrom int64
if want != nil && want.Size > 0 {
if _, digest := want.bestSum(); digest != "" {
if info, err := os.Stat(dst + PartialSuffix); err == nil && info.Size() > 0 && info.Size() < want.Size {
resumeFrom = info.Size()
}
}
}
for {
state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom)
if err != nil && resumeFrom > 0 && !errors.Is(err, ErrNotFound) && ctx.Err() == nil {
// The partial may not be a prefix of the current upstream file;
// retry once from scratch.
log.WithError(err).WithField("path", reqPath).Debug("Resumed download failed; retrying in full.")
_ = os.Remove(dst + PartialSuffix)
resumeFrom = 0
continue
}
return state, err
}
}
// download performs one transfer attempt for File, appending to the
// partial file when resuming from a prior failure.
func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64) (FileState, error) {
// Files without published checksums cannot be verified locally, so an
// existing copy is revalidated with a conditional request instead of
// being re-downloaded every run.
var modifiedSince time.Time
if want == nil {
if info, err := os.Stat(dst); err == nil && info.Mode().IsRegular() {
modifiedSince = info.ModTime()
}
}
// Request the file, failing over between mirrors as needed.
resp, err := src.Get(ctx, reqPath, GetOptions{ModifiedSince: modifiedSince, RangeFrom: resumeFrom})
if errors.Is(err, ErrNotModified) {
Stats.Unchanged.Add(1)
log.WithField("path", reqPath).Debug("Kept unchanged repository file.")
return FileUnchanged, nil
}
if err != nil {
return FileMissing, err
}
defer resp.Body.Close()
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return FileMissing, err
}
var hasher hash.Hash
var algo, digest string
if want != nil {
algo, digest = want.bestSum()
if digest != "" {
hasher = newHasher(algo)
}
}
// Partials are only kept for files whose content the next attempt can
// verify by checksum.
keepPartial := digest != "" && want.Size > 0
// A 206 continues the partial file, whose existing bytes must feed the
// hash first; any other success replaces it from the start.
partial := dst + PartialSuffix
resumed := resumeFrom > 0 && resp.StatusCode == http.StatusPartialContent
openFlags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
if resumed {
openFlags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
if hasher != nil {
pf, err := os.Open(partial)
if err != nil {
return FileMissing, err
}
_, err = io.Copy(hasher, pf)
pf.Close()
if err != nil {
return FileMissing, err
}
}
log.WithFields(log.Fields{"path": reqPath, "offset": resumeFrom}).Debug("Resuming partial download.")
}
// Stream the body into the partial file so a crash never leaves an
// incomplete file at the final location.
f, err := os.OpenFile(partial, openFlags, 0644)
if err != nil {
return FileMissing, err
}
w := io.Writer(f)
if hasher != nil {
w = io.MultiWriter(f, hasher)
}
written, copyErr := io.Copy(w, resp.Body)
closeErr := f.Close()
if copyErr != nil {
if !keepPartial {
_ = os.Remove(partial)
}
return FileMissing, fmt.Errorf("download %s: %w", reqPath, copyErr)
}
if closeErr != nil {
if !keepPartial {
_ = os.Remove(partial)
}
return FileMissing, closeErr
}
// Verify the transfer against the published size and checksum. A
// mismatch invalidates the partial file entirely.
total := written
if resumed {
total += resumeFrom
}
if want != nil && want.Size >= 0 && total != want.Size {
_ = os.Remove(partial)
return FileMissing, fmt.Errorf("download %s: size %d does not match expected %d", reqPath, total, want.Size)
}
if hasher != nil {
if sum := hex.EncodeToString(hasher.Sum(nil)); sum != digest {
_ = os.Remove(partial)
return FileMissing, fmt.Errorf("download %s: %s checksum %s does not match expected %s", reqPath, algo, sum, digest)
}
}
// Move the download into place, or next to it when staging.
final := dst
state := FileFetched
if stage {
final = dst + StagedSuffix
state = FileStaged
}
if err := os.Rename(partial, final); err != nil {
_ = os.Remove(partial)
return FileMissing, err
}
// Carry the upstream modification time so later runs can revalidate
// with conditional requests. Promotion preserves it.
if lm, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
_ = os.Chtimes(final, time.Now(), lm)
}
Stats.Fetched.Add(1)
Stats.FetchedBytes.Add(written)
log.WithField("path", reqPath).Debug("Fetched repository file.")
return state, nil
}
// PlanJobs records what a real run would download without transferring
// anything, still marking every destination as part of the repository so a
// dry-run prune report stays accurate. Optional files the upstream may not
// serve are counted, so the result is an upper bound.
func PlanJobs(jobs []Job, verifyExisting bool, keep *KeepSet) {
for _, job := range jobs {
keep.Add(job.Dst)
if job.Want != nil && verifyLocal(job.Dst, job.Want, verifyExisting) {
Stats.Unchanged.Add(1)
continue
}
Stats.Planned.Add(1)
if job.Want != nil && job.Want.Size > 0 {
Stats.PlannedBytes.Add(job.Want.Size)
}
log.WithField("path", job.ReqPath).Debug("Would fetch repository file.")
}
}
// RemoveStale deletes a local file the upstream no longer serves, so an
// outdated copy is never served beside refreshed metadata. Missing files
// are not an error.
func RemoveStale(name string) {
if err := os.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
log.WithError(err).WithField("path", name).Warn("Failed to remove stale file.")
}
}
// StagedOrFinal returns the staged copy of dst when one exists, falling
// back to the promoted file for runs where the upstream was unchanged.
func StagedOrFinal(dst string) string {
staged := dst + StagedSuffix
if _, err := os.Stat(staged); err == nil {
return staged
}
return dst
}
// PromoteStaged moves a staged download into its final location. Missing
// staged files are not an error, as unchanged files are never staged.
func PromoteStaged(dst string) error {
staged := dst + StagedSuffix
if _, err := os.Stat(staged); errors.Is(err, fs.ErrNotExist) {
return nil
}
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
return os.Rename(staged, dst)
}
// PromoteOrDiscard finishes a staged file: promoted normally, or removed in
// a dry run so no published repository state changes.
func PromoteOrDiscard(dst string, dryRun bool) error {
if dryRun {
_ = os.Remove(dst + StagedSuffix)
return nil
}
return PromoteStaged(dst)
}
// Job describes one file for Many to download.
type Job struct {
ReqPath string
Dst string
Want *Expect
Optional bool
Stage bool
}
// Many downloads jobs with a fixed worker pool, recording each job's
// resulting state. The first hard failure aborts the remaining work.
func Many(ctx context.Context, src *Source, jobs []Job, workers int, verifyExisting bool, keep *KeepSet) ([]FileState, error) {
states := make([]FileState, len(jobs))
indexes := make(chan int)
errs := make(chan error, 1)
var wg sync.WaitGroup
if workers < 1 {
workers = 1
}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for idx := range indexes {
job := jobs[idx]
state, err := File(ctx, src, job.ReqPath, job.Dst, job.Want, job.Stage, verifyExisting)
if errors.Is(err, ErrNotFound) && job.Optional {
log.WithField("path", job.ReqPath).Debug("Skipped file the upstream does not serve.")
continue
}
if err != nil {
select {
case errs <- err:
default:
}
return
}
states[idx] = state
keep.Add(job.Dst)
}
}()
}
for idx := range jobs {
select {
case err := <-errs:
close(indexes)
wg.Wait()
return states, err
case <-ctx.Done():
close(indexes)
wg.Wait()
return states, ctx.Err()
case indexes <- idx:
}
}
close(indexes)
wg.Wait()
select {
case err := <-errs:
return states, err
default:
return states, nil
}
}
// KeepSet tracks the files that belong to the current sync so pruning can
// remove everything else. A nil KeepSet ignores all operations.
type KeepSet struct {
mu sync.Mutex
paths map[string]bool
}
// NewKeepSet returns a tracking set when pruning is enabled, or nil so all
// tracking becomes a no-op.
func NewKeepSet(enabled bool) *KeepSet {
if !enabled {
return nil
}
return &KeepSet{paths: map[string]bool{}}
}
// Add records a local file as part of the repository.
func (k *KeepSet) Add(name string) {
if k == nil {
return
}
abs, err := filepath.Abs(name)
if err != nil {
return
}
k.mu.Lock()
k.paths[abs] = true
k.mu.Unlock()
}
// has reports whether a local file is part of the repository.
func (k *KeepSet) has(name string) bool {
if k == nil {
return false
}
abs, err := filepath.Abs(name)
if err != nil {
return false
}
k.mu.Lock()
defer k.mu.Unlock()
return k.paths[abs]
}
// loadPruneState reads the grace bookkeeping for a pruned tree, mapping
// root-relative paths to when they were first seen unreferenced.
func loadPruneState(root string) map[string]time.Time {
state := map[string]time.Time{}
data, err := os.ReadFile(filepath.Join(root, PruneStateName))
if err != nil {
return state
}
if err := json.Unmarshal(data, &state); err != nil {
log.WithError(err).WithField("path", root).Warn("Failed to parse prune state; starting fresh.")
return map[string]time.Time{}
}
return state
}
// savePruneState persists the grace bookkeeping, or removes it when no
// files are waiting out their grace period.
func savePruneState(root string, state map[string]time.Time) {
name := filepath.Join(root, PruneStateName)
if len(state) == 0 {
_ = os.Remove(name)
return
}
data, err := json.Marshal(state)
if err == nil {
err = os.WriteFile(name, data, 0644)
}
if err != nil {
log.WithError(err).WithField("path", name).Warn("Failed to save prune state.")
}
}
// PruneTree removes files under root that are not part of the current sync,
// then clears out any directories left empty. A grace period defers
// deletion until a file has been unreferenced across runs for that long,
// so clients holding earlier metadata keep working. Failures are logged
// and do not fail the sync.
func PruneTree(root string, keep *KeepSet, grace time.Duration, dryRun bool) {
if keep == nil {
return
}
var state map[string]time.Time
if grace > 0 {
state = loadPruneState(root)
}
now := time.Now()
var dirs []string
err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if name != root {
dirs = append(dirs, name)
}
return nil
}
base := filepath.Base(name)
if base == PruneStateName || base == LockFileName {
return nil
}
rel, relErr := filepath.Rel(root, name)
if relErr != nil {
return nil
}
if keep.has(name) {
delete(state, rel)
return nil
}
// Internal work files are removed regardless of grace; anything
// else waits out the grace period across runs first.
internal := strings.HasSuffix(base, StagedSuffix) || strings.HasSuffix(base, PartialSuffix)
if grace > 0 && !internal {
first, ok := state[rel]
if !ok {
// A dry run records nothing, so report the pending grace
// entry instead of staying silent about the file.
if dryRun {
log.WithField("path", name).Info("Stale file would enter the prune grace period.")
return nil
}
state[rel] = now
log.WithField("path", name).Debug("Stale file entered the prune grace period.")
return nil
}
if now.Sub(first) < grace {
return nil
}
}
if dryRun {
Stats.Pruned.Add(1)
log.WithField("path", name).Info("Would prune stale file.")
return nil
}
if err := os.Remove(name); err != nil {
log.WithError(err).WithField("path", name).Warn("Failed to prune stale file.")
return nil
}
delete(state, rel)
Stats.Pruned.Add(1)
log.WithField("path", name).Debug("Pruned stale file.")
return nil
})
if err != nil {
log.WithError(err).WithField("path", root).Warn("Failed to walk repository for pruning.")
}
// Drop bookkeeping for files that no longer exist, then persist it.
if grace > 0 && !dryRun {
for rel := range state {
if _, err := os.Stat(filepath.Join(root, rel)); errors.Is(err, fs.ErrNotExist) {
delete(state, rel)
}
}
savePruneState(root, state)
}
if dryRun {
return
}
// Remove now-empty directories deepest first. Separator count is the
// true depth, so a parent is always removed after its children even
// when a sibling's child has a longer name.
sep := string(os.PathSeparator)
sort.Slice(dirs, func(i, j int) bool {
return strings.Count(dirs[i], sep) > strings.Count(dirs[j], sep)
})
for _, dir := range dirs {
_ = os.Remove(dir)
}
}
// Decompressor wraps a reader with the decompressor matching the file name
// extension, returning an optional close function for the wrapper. Staged
// suffixes are ignored so staged indexes can be parsed before promotion.
func Decompressor(r io.Reader, filename string) (io.Reader, func(), error) {
filename = strings.TrimSuffix(filename, StagedSuffix)
switch {
case strings.HasSuffix(filename, ".gz"):
gz, err := gzip.NewReader(r)
if err != nil {
return nil, nil, err
}
return gz, func() { _ = gz.Close() }, nil
case strings.HasSuffix(filename, ".bz2"):
return bzip2.NewReader(r), nil, nil
case strings.HasSuffix(filename, ".xz"):
xr, err := xz.NewReader(r)
if err != nil {
return nil, nil, err
}
return xr, nil, nil
case strings.HasSuffix(filename, ".zst"):
zr, err := zstd.NewReader(r)
if err != nil {
return nil, nil, err
}
return zr, zr.Close, nil
default:
return r, nil, nil
}
}
// SniffDecompressor wraps a reader with the decompressor matching its
// leading magic bytes, for formats whose file names carry no extension.
func SniffDecompressor(r io.Reader) (io.Reader, func(), error) {
br := bufio.NewReader(r)
magic, err := br.Peek(6)
if err != nil && !errors.Is(err, io.EOF) {
return nil, nil, err
}
switch {
case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b:
gz, err := gzip.NewReader(br)
if err != nil {
return nil, nil, err
}
return gz, func() { _ = gz.Close() }, nil
case len(magic) >= 3 && magic[0] == 'B' && magic[1] == 'Z' && magic[2] == 'h':
return bzip2.NewReader(br), nil, nil
case len(magic) >= 6 && bytes.Equal(magic[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
xr, err := xz.NewReader(br)
if err != nil {
return nil, nil, err
}
return xr, nil, nil
case len(magic) >= 4 && bytes.Equal(magic[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}):
zr, err := zstd.NewReader(br)
if err != nil {
return nil, nil, err
}
return zr, zr.Close, nil
default:
return br, nil, nil
}
}
// LocalJoin resolves a repository-relative path under root, rejecting paths
// that would escape the destination tree.
func LocalJoin(root, rel string) (string, error) {
clean := path.Clean("/" + strings.ReplaceAll(rel, "\\", "/"))
dst := filepath.Join(root, filepath.FromSlash(clean))
rootClean := filepath.Clean(root)
prefix := rootClean + string(os.PathSeparator)
if rootClean == string(os.PathSeparator) {
prefix = rootClean
}
if dst != rootClean && !strings.HasPrefix(dst, prefix) {
return "", fmt.Errorf("path %q escapes destination %q", rel, root)
}
return dst, nil
}

212
fetch/fetch_test.go Normal file
View file

@ -0,0 +1,212 @@
package fetch
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// resumeFixture serves a distinct-content file and records the Range
// headers of incoming requests.
func resumeFixture(t *testing.T, stripRange bool) (*Source, []byte, *[]string) {
t.Helper()
content := make([]byte, 3000)
for i := range content {
content[i] = byte(i % 251)
}
www := t.TempDir()
testrepos.WriteFile(t, filepath.Join(www, "big.bin"), content)
var mu sync.Mutex
ranges := &[]string{}
fs := http.FileServer(http.Dir(www))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
*ranges = append(*ranges, r.Header.Get("Range"))
mu.Unlock()
if stripRange {
r.Header.Del("Range")
}
fs.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
return NewSource([]string{srv.URL}), content, ranges
}
// TestFetchFileResume verifies a valid partial file is completed with a
// ranged request.
func TestFetchFileResume(t *testing.T) {
src, content, ranges := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, content[:1000])
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, content) {
t.Error("resumed content mismatch")
}
if len(*ranges) != 1 || (*ranges)[0] != "bytes=1000-" {
t.Errorf("ranges = %v, want one bytes=1000- request", *ranges)
}
}
// TestFetchFileResumeIgnored verifies a server that ignores ranges still
// produces a correct full download.
func TestFetchFileResumeIgnored(t *testing.T) {
src, content, _ := resumeFixture(t, true)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, content[:1000])
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, _ := os.ReadFile(dst)
if !bytes.Equal(got, content) {
t.Error("content mismatch after ignored range")
}
}
// TestFetchFileResumeBadPartial verifies a partial that is not a prefix of
// the upstream file falls back to a full download.
func TestFetchFileResumeBadPartial(t *testing.T) {
src, content, ranges := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
testrepos.WriteFile(t, dst+PartialSuffix, bytes.Repeat([]byte("X"), 1000))
state, err := File(context.Background(), src, "big.bin", dst, want, false, false)
if err != nil || state != FileFetched {
t.Fatalf("fetch = %v, %v", state, err)
}
got, _ := os.ReadFile(dst)
if !bytes.Equal(got, content) {
t.Error("content mismatch after bad partial")
}
if len(*ranges) != 2 || (*ranges)[1] != "" {
t.Errorf("ranges = %v, want a ranged then a full request", *ranges)
}
}
// TestFetchFileConditional verifies files without published checksums are
// revalidated with conditional requests: unchanged upstreams are kept, and
// changed upstreams are fetched again.
func TestFetchFileConditional(t *testing.T) {
www := t.TempDir()
upstream := filepath.Join(www, "meta.tar.gz")
testrepos.WriteFile(t, upstream, []byte("metadata"))
srv := testrepos.ServeDir(t, www)
src := NewSource([]string{srv.URL})
dst := filepath.Join(t.TempDir(), "meta.tar.gz")
state, err := File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileStaged {
t.Fatalf("first fetch = %v, %v", state, err)
}
if err := PromoteStaged(dst); err != nil {
t.Fatal(err)
}
// An unchanged upstream answers the conditional request with 304.
state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileUnchanged {
t.Errorf("unchanged fetch = %v, %v", state, err)
}
// A newer upstream file must be fetched again. The modification time
// is pushed forward because Last-Modified has one-second resolution.
testrepos.WriteFile(t, upstream, []byte("metadata-v2"))
future := time.Now().Add(2 * time.Second)
if err := os.Chtimes(upstream, future, future); err != nil {
t.Fatal(err)
}
state, err = File(context.Background(), src, "meta.tar.gz", dst, nil, true, false)
if err != nil || state != FileStaged {
t.Fatalf("changed fetch = %v, %v", state, err)
}
got, err := os.ReadFile(dst + StagedSuffix)
if err != nil {
t.Fatal(err)
}
if string(got) != "metadata-v2" {
t.Errorf("staged content = %q", got)
}
}
// TestFetchFileConcurrent verifies concurrent downloads to the same
// destination are serialized: no caller's partial download can interleave
// with another's.
func TestFetchFileConcurrent(t *testing.T) {
src, content, _ := resumeFixture(t, false)
want := &Expect{Size: int64(len(content)), Sums: map[string]string{"sha256": testrepos.SHA256Hex(content)}}
dst := filepath.Join(t.TempDir(), "big.bin")
var wg sync.WaitGroup
errs := make(chan error, 8)
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := File(context.Background(), src, "big.bin", dst, want, false, true); err != nil {
errs <- err
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
got, err := os.ReadFile(dst)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, content) {
t.Error("content mismatch after concurrent downloads")
}
}
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
Reload()
os.Exit(m.Run())
}
// TestLocalJoin verifies traversal attempts cannot escape the destination.
func TestLocalJoin(t *testing.T) {
root := t.TempDir()
got, err := LocalJoin(root, "../../etc/passwd")
if err != nil {
t.Fatal(err)
}
want := filepath.Join(root, "etc", "passwd")
if got != want {
t.Errorf("got %q, want %q", got, want)
}
if _, err := LocalJoin(root, "a/../b/c.rpm"); err != nil {
t.Errorf("clean relative path rejected: %v", err)
}
got, err = LocalJoin("/", "etc/passwd")
if err != nil {
t.Fatalf("filesystem root destination rejected: %v", err)
}
if got != "/etc/passwd" {
t.Errorf("got %q, want %q", got, "/etc/passwd")
}
}

258
fetch/source.go Normal file
View file

@ -0,0 +1,258 @@
package fetch
import (
"bufio"
"bytes"
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
log "github.com/sirupsen/logrus"
)
// Source is the set of base URLs a repository can be fetched from. Direct
// repositories have a single base; mirrorlists provide several with
// failover between them.
type Source struct {
// mu guards bases and active.
mu sync.Mutex
bases []string
active int
}
// NewSource builds a Source from base URLs, normalizing trailing slashes.
func NewSource(bases []string) *Source {
s := &Source{}
for _, base := range bases {
s.bases = append(s.bases, strings.TrimRight(base, "/"))
}
return s
}
// Rebase trims a common relative suffix off every base URL so later fetches
// resolve against the parent directory. Bases without the suffix are kept
// unchanged.
func (s *Source) Rebase(suffix string) {
suffix = "/" + strings.Trim(suffix, "/")
s.mu.Lock()
defer s.mu.Unlock()
for i, base := range s.bases {
if strings.HasSuffix(base, suffix) {
s.bases[i] = strings.TrimSuffix(base, suffix)
} else {
log.WithField("mirror", base).Warn("Mirror URL does not share the expected path suffix; using it unchanged.")
}
}
}
// GetOptions adjusts a single Source request. A non-zero ModifiedSince
// adds a conditional header, surfacing a 304 as ErrNotModified. A positive
// RangeFrom requests the remainder of a partially downloaded file.
type GetOptions struct {
ModifiedSince time.Time
RangeFrom int64
}
// Get requests a repository-relative path, failing over to the next mirror
// on transport or server errors. A 404 from the responding mirror is
// treated as authoritative so optional files do not trigger failover.
func (s *Source) Get(ctx context.Context, reqPath string, o GetOptions) (*http.Response, error) {
// Snapshot the bases so a concurrent Rebase cannot race the loop.
s.mu.Lock()
start := s.active
bases := append([]string(nil), s.bases...)
s.mu.Unlock()
if len(bases) == 0 {
return nil, errors.New("source has no mirror URLs")
}
var lastErr error
for i := 0; i < len(bases); i++ {
idx := (start + i) % len(bases)
target := bases[idx] + "/" + strings.TrimPrefix(reqPath, "/")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
if !o.ModifiedSince.IsZero() {
req.Header.Set("If-Modified-Since", o.ModifiedSince.UTC().Format(http.TimeFormat))
}
if o.RangeFrom > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", o.RangeFrom))
}
resp, err := client.Load().Do(req)
if err != nil {
lastErr = fmt.Errorf("fetch %s: %w", target, err)
log.WithError(err).WithField("url", target).Debug("Mirror request failed.")
continue
}
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
resp.Body.Close()
return nil, fmt.Errorf("fetch %s: %w", target, ErrNotFound)
}
if resp.StatusCode == http.StatusNotModified && !o.ModifiedSince.IsZero() {
resp.Body.Close()
return nil, fmt.Errorf("fetch %s: %w", target, ErrNotModified)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
resp.Body.Close()
lastErr = fmt.Errorf("fetch %s: status %d", target, resp.StatusCode)
log.WithField("url", target).WithField("status", resp.StatusCode).Debug("Mirror request failed.")
continue
}
if idx != start {
s.mu.Lock()
s.active = idx
s.mu.Unlock()
}
return resp, nil
}
return nil, lastErr
}
// ReadURL retrieves a raw URL fully into memory, limited to 4 MiB. It is
// used for probing mirrorlists, whose URLs may carry query strings that a
// Source cannot join paths onto.
func ReadURL(ctx context.Context, rawURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
resp, err := client.Load().Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
return nil, fmt.Errorf("fetch %s: %w", rawURL, ErrNotFound)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("fetch %s: status %d", rawURL, resp.StatusCode)
}
// Read one byte past the cap so an oversized document can be detected
// and reported rather than silently truncated into a bad parse.
const maxSize = 4 << 20
data, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1))
if err != nil {
return nil, err
}
if len(data) > maxSize {
log.WithField("url", rawURL).Warn("Response exceeds the read limit and was truncated.")
data = data[:maxSize]
}
return data, nil
}
// ParseMirrorlist extracts mirror base URLs from a mirrorlist body,
// skipping comments and lines that are not absolute HTTP URLs.
func ParseMirrorlist(body []byte) []string {
seen := map[string]bool{}
var bases []string
scanner := bufio.NewScanner(bytes.NewReader(body))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
u, err := url.Parse(line)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
continue
}
base := strings.TrimRight(line, "/")
if seen[base] {
continue
}
seen[base] = true
bases = append(bases, base)
}
return bases
}
// ParseMetalink extracts mirror base URLs from a metalink document's
// repomd.xml entries, keeping the document's preference order.
func ParseMetalink(body []byte) []string {
const repomdSuffix = "/repodata/repomd.xml"
var ml struct {
Files []struct {
Name string `xml:"name,attr"`
URLs []struct {
Value string `xml:",chardata"`
} `xml:"resources>url"`
} `xml:"files>file"`
}
if xml.Unmarshal(body, &ml) != nil {
return nil
}
seen := map[string]bool{}
var bases []string
for _, file := range ml.Files {
if file.Name != "repomd.xml" {
continue
}
for _, u := range file.URLs {
v := strings.TrimSpace(u.Value)
if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") {
continue
}
base := strings.TrimSuffix(v, repomdSuffix)
if base == v {
continue
}
base = strings.TrimRight(base, "/")
if seen[base] {
continue
}
seen[base] = true
bases = append(bases, base)
}
}
return bases
}
// userAgent is sent with every upstream request; the main package sets it
// to include the build version. It is atomic because a reload may change
// it while requests are in flight.
var userAgent atomic.Value
func init() {
userAgent.Store("repo-sync")
}
// SetUserAgent replaces the user agent sent with every upstream request.
func SetUserAgent(ua string) {
userAgent.Store(ua)
}
// agent returns the current user agent string.
func agent() string {
return userAgent.Load().(string)
}
// Get performs a plain GET of a raw URL with the mirror user agent, used
// for directory listings and other non-repository requests.
func Get(ctx context.Context, rawURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", agent())
return client.Load().Do(req)
}
// Bases returns a copy of the mirror base URLs, primarily for tests and
// diagnostics.
func (s *Source) Bases() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.bases...)
}

97
fetch/source_test.go Normal file
View file

@ -0,0 +1,97 @@
package fetch
import (
"context"
"errors"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseMirrorlist verifies URL extraction, comment skipping, and
// deduplication.
func TestParseMirrorlist(t *testing.T) {
body := "# comment\n\nhttp://mirror1.example.com/repo/\nhttps://mirror2.example.com/repo\nhttp://mirror1.example.com/repo\n/bare/path\nnot a url\n"
bases := ParseMirrorlist([]byte(body))
want := []string{"http://mirror1.example.com/repo", "https://mirror2.example.com/repo"}
if len(bases) != len(want) {
t.Fatalf("got %v, want %v", bases, want)
}
for i := range want {
if bases[i] != want[i] {
t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i])
}
}
}
// TestSourceFailover verifies a dead first mirror fails over to a working
// one and that missing files surface the not-found sentinel.
func TestSourceFailover(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
src := NewSource([]string{"http://127.0.0.1:1/el9", srv.URL + "/el9"})
resp, err := src.Get(context.Background(), "repodata/repomd.xml", GetOptions{})
if err != nil {
t.Fatal("failover did not reach the working mirror:", err)
}
resp.Body.Close()
if src.active != 1 {
t.Errorf("active mirror = %d, want 1", src.active)
}
if _, err := src.Get(context.Background(), "no/such/file", GetOptions{}); !errors.Is(err, ErrNotFound) {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
// TestParseMetalink verifies base URL extraction from metalink documents.
func TestParseMetalink(t *testing.T) {
body := `<?xml version="1.0"?><metalink xmlns="urn:ietf:params:xml:ns:metalink">` +
`<files><file name="repomd.xml"><resources>` +
`<url protocol="https" type="https">https://mirror1.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="rsync">rsync://mirror2.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="http">http://mirror3.example.com/fedora/repodata/repomd.xml</url>` +
`<url protocol="https">https://mirror1.example.com/fedora/repodata/repomd.xml</url>` +
`</resources></file></files></metalink>`
bases := ParseMetalink([]byte(body))
want := []string{"https://mirror1.example.com/fedora", "http://mirror3.example.com/fedora"}
if len(bases) != len(want) {
t.Fatalf("got %v, want %v", bases, want)
}
for i := range want {
if bases[i] != want[i] {
t.Errorf("bases[%d] = %q, want %q", i, bases[i], want[i])
}
}
if ParseMetalink([]byte("not xml at all")) != nil {
t.Error("non-XML input produced bases")
}
}
// TestSourceNoBases verifies a Source with no mirrors reports an error and
// no response.
func TestSourceNoBases(t *testing.T) {
src := NewSource(nil)
resp, err := src.Get(context.Background(), "anything", GetOptions{})
if err == nil {
t.Fatal("expected an error for a source without bases")
}
if resp != nil {
t.Error("response must be nil on error")
}
}
// TestSourceRebase verifies suffix trimming for suite-relative sources.
func TestSourceRebase(t *testing.T) {
src := NewSource([]string{"http://example.com/debian/dists/test", "http://other.example.com/mirror"})
src.Rebase("dists/test")
if src.bases[0] != "http://example.com/debian" {
t.Errorf("bases[0] = %q", src.bases[0])
}
if src.bases[1] != "http://other.example.com/mirror" {
t.Errorf("bases[1] changed unexpectedly: %q", src.bases[1])
}
}

51
fetch/stats.go Normal file
View file

@ -0,0 +1,51 @@
package fetch
import (
"fmt"
"sync/atomic"
)
// counters accumulates transfer activity for one repository's summary log
// line. Counters are atomic because download workers run concurrently.
type counters struct {
Fetched atomic.Int64
FetchedBytes atomic.Int64
Unchanged atomic.Int64
Pruned atomic.Int64
Planned atomic.Int64
PlannedBytes atomic.Int64
}
// Stats collects transfer activity for the repository currently
// synchronizing. The CLI synchronizes repositories sequentially, so one
// package-level collector suffices there. The server's concurrent crawls
// interleave into these counters, so their values are only meaningful in
// the sequential path; the server must not report them.
var Stats = &counters{}
// Reset clears the collector for the next repository.
func (s *counters) Reset() {
s.Fetched.Store(0)
s.FetchedBytes.Store(0)
s.Unchanged.Store(0)
s.Pruned.Store(0)
s.Planned.Store(0)
s.PlannedBytes.Store(0)
}
// FormatBytes renders a byte count in a human readable unit.
func FormatBytes(n int64) string {
const unit = 1024
const prefixes = "KMGTPE"
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
// Cap the exponent at the last available prefix so an unexpectedly
// large count can never index past the prefix table.
for m := n / unit; m >= unit && exp < len(prefixes)-1; m /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), prefixes[exp])
}

72
flags.go Normal file
View file

@ -0,0 +1,72 @@
package main
import (
"fmt"
"runtime/debug"
"github.com/alecthomas/kong"
cfg "github.com/grmrgecko/repo-sync/config"
)
// VersionFlag prints build information and exits.
type VersionFlag bool
// Decode satisfies kong.MapperValue. The flag is treated as a boolean toggle.
func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil }
// IsBool reports the flag as a boolean for kong's parser.
func (v VersionFlag) IsBool() bool { return true }
// BeforeApply emits version information then exits before the rest of the
// command is executed.
func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error {
fmt.Printf("%s: %s (%s)\n", cfg.Name, cfg.Version, cfg.Mode)
if cfg.Commit != "" {
fmt.Printf(" commit: %s\n", cfg.Commit)
}
if cfg.Date != "" {
fmt.Printf(" built: %s\n", cfg.Date)
}
if cfg.Commit == "" {
if bi, ok := debug.ReadBuildInfo(); ok {
for _, s := range bi.Settings {
switch s.Key {
case "vcs.revision":
fmt.Printf(" commit: %s\n", s.Value)
case "vcs.time":
fmt.Printf(" built: %s\n", s.Value)
}
}
}
}
app.Exit(0)
return nil
}
// Flags is the root command line configuration.
type Flags struct {
LogLevel string `help:"Log level (debug, info, warn, error)." enum:"debug,info,warn,error" default:"info"`
Version VersionFlag `name:"version" help:"Print version information and quit"`
RPM RPMCmd `cmd:"" name:"rpm" help:"Synchronize RPM (yum/dnf) repositories."`
Deb DebCmd `cmd:"" name:"deb" help:"Synchronize DEB (apt) repositories."`
Arch ArchCmd `cmd:"" name:"arch" help:"Synchronize Arch Linux (pacman) repositories."`
Apk ApkCmd `cmd:"" name:"apk" help:"Synchronize Alpine Linux (apk) repositories."`
Server ServerCmd `cmd:"" name:"server" help:"Run the caching mirror server."`
}
// flags holds the parsed command line configuration.
var flags Flags
// ParseFlags parses the command line, exiting with usage information on
// error.
func ParseFlags() *kong.Context {
return kong.Parse(&flags,
kong.Name(cfg.Name),
kong.Description(cfg.Description),
kong.UsageOnError(),
kong.ConfigureHelp(kong.HelpOptions{
Compact: true,
}),
)
}

34
go.mod Normal file
View file

@ -0,0 +1,34 @@
module github.com/grmrgecko/repo-sync
go 1.26.4
require (
github.com/alecthomas/kong v1.16.0
github.com/go-playground/validator/v10 v10.30.3
github.com/klauspost/compress v1.19.1
github.com/sirupsen/logrus v1.9.4
github.com/spf13/viper v1.21.0
github.com/ulikunitz/xz v0.5.16
golang.org/x/net v0.57.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
)

77
go.sum Normal file
View file

@ -0,0 +1,77 @@
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4=
github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8=
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

258
internal/testrepos/testrepos.go Normal file
View file

@ -0,0 +1,258 @@
// Package testrepos builds small fixture repositories for tests across
// the fetch, mirror, and server packages.
package testrepos
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// WriteFile creates a file with parent directories under a fixture tree.
func WriteFile(t *testing.T, name string, data []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(name, data, 0644); err != nil {
t.Fatal(err)
}
}
// GzipBytes compresses data with gzip for index fixtures.
func GzipBytes(t *testing.T, data []byte) []byte {
t.Helper()
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
if _, err := gz.Write(data); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// SHA256Hex returns the hex SHA-256 of data.
func SHA256Hex(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// MD5Hex returns the hex MD5 of data.
func MD5Hex(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
// ServeDir serves a fixture tree over HTTP with directory listings, closing
// the server when the test finishes.
func ServeDir(t *testing.T, dir string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.FileServer(http.Dir(dir)))
t.Cleanup(srv.Close)
return srv
}
// BuildRPMRepo writes a minimal yum repository into dir and returns the
// package contents by file name.
func BuildRPMRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"foo-1.0-1.x86_64.rpm": bytes.Repeat([]byte("foo"), 500),
"bar-2.0-1.x86_64.rpm": bytes.Repeat([]byte("bar"), 700),
}
var pkgXML bytes.Buffer
for name, data := range pkgs {
WriteFile(t, filepath.Join(dir, "Packages", name), data)
fmt.Fprintf(&pkgXML,
`<package type="rpm"><name>%s</name><checksum type="sha256" pkgid="YES">%s</checksum><size package="%d"/><location href="Packages/%s"/></package>`,
name, SHA256Hex(data), len(data), name)
}
primary := []byte(`<?xml version="1.0"?><metadata packages="2">` + pkgXML.String() + `</metadata>`)
primaryGz := GzipBytes(t, primary)
primaryName := SHA256Hex(primaryGz) + "-primary.xml.gz"
WriteFile(t, filepath.Join(dir, "repodata", primaryName), primaryGz)
// A prestodelta index references one delta package under drpms/.
drpm := RPMDelta()
WriteFile(t, filepath.Join(dir, "drpms", "foo-0.9_1.0-1.x86_64.drpm"), drpm)
presto := []byte(fmt.Sprintf(
`<?xml version="1.0"?><prestodelta><newpackage name="foo"><delta oldversion="0.9"><filename>drpms/foo-0.9_1.0-1.x86_64.drpm</filename><size>%d</size><checksum type="sha256">%s</checksum></delta></newpackage></prestodelta>`,
len(drpm), SHA256Hex(drpm)))
prestoGz := GzipBytes(t, presto)
prestoName := SHA256Hex(prestoGz) + "-prestodelta.xml.gz"
WriteFile(t, filepath.Join(dir, "repodata", prestoName), prestoGz)
repomd := fmt.Sprintf(
`<?xml version="1.0"?><repomd><data type="primary"><checksum type="sha256">%s</checksum><location href="repodata/%s"/><size>%d</size></data><data type="prestodelta"><checksum type="sha256">%s</checksum><location href="repodata/%s"/><size>%d</size></data></repomd>`,
SHA256Hex(primaryGz), primaryName, len(primaryGz),
SHA256Hex(prestoGz), prestoName, len(prestoGz))
WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml"), []byte(repomd))
WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.asc"), []byte("fake signature"))
return pkgs
}
// RPMDelta is the delta package content used by BuildRPMRepo.
func RPMDelta() []byte {
return bytes.Repeat([]byte("drpm"), 100)
}
// debPackagesStanza renders one binary package stanza for a Packages index.
func debPackagesStanza(name, version, poolPath string, data []byte) string {
return fmt.Sprintf(
"Package: %s\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSize: %d\nMD5sum: %s\nSHA256: %s\nDescription: Test package\n",
name, version, poolPath, len(data), MD5Hex(data), SHA256Hex(data))
}
// BuildDebRepo writes a minimal structured apt repository with main and
// universe components into dir and returns the pool file contents by
// archive-relative path.
func BuildDebRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pool := map[string][]byte{
"pool/main/h/hello/hello_1.0_amd64.deb": bytes.Repeat([]byte("hello"), 300),
"pool/universe/e/extra/extra_2.0_amd64.deb": bytes.Repeat([]byte("extra"), 400),
}
for p, data := range pool {
WriteFile(t, filepath.Join(dir, filepath.FromSlash(p)), data)
}
indexes := map[string][]byte{
"main/binary-amd64/Packages": []byte(debPackagesStanza(
"hello", "1.0", "pool/main/h/hello/hello_1.0_amd64.deb", pool["pool/main/h/hello/hello_1.0_amd64.deb"])),
"universe/binary-amd64/Packages": []byte(debPackagesStanza(
"extra", "2.0", "pool/universe/e/extra/extra_2.0_amd64.deb", pool["pool/universe/e/extra/extra_2.0_amd64.deb"])),
}
// The Release file lists both the uncompressed and gzip variants, but
// only the gzip variant is served, matching real archives.
release := "Origin: Test\nSuite: test\nCodename: test\nArchitectures: amd64\nComponents: main universe\nAcquire-By-Hash: yes\n"
listed := map[string][]byte{}
for p, data := range indexes {
gz := GzipBytes(t, data)
WriteFile(t, filepath.Join(dir, "dists", "test", filepath.FromSlash(p)+".gz"), gz)
listed[p] = data
listed[p+".gz"] = gz
}
for field, hasher := range map[string]func([]byte) string{"MD5Sum": MD5Hex, "SHA256": SHA256Hex} {
release += field + ":\n"
for p, data := range listed {
release += fmt.Sprintf(" %s %d %s\n", hasher(data), len(data), p)
}
}
WriteFile(t, filepath.Join(dir, "dists", "test", "Release"), []byte(release))
return pool
}
// BuildArchRepo writes a minimal pacman repository named name into dir and
// returns the package contents by file name.
func BuildArchRepo(t *testing.T, dir, name string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"zlib-1.3-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("zlib"), 300),
"jq-1.7-1-x86_64.pkg.tar.zst": bytes.Repeat([]byte("jq"), 400),
}
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
tw := tar.NewWriter(gz)
for fname, data := range pkgs {
WriteFile(t, filepath.Join(dir, fname), data)
WriteFile(t, filepath.Join(dir, fname+".sig"), []byte("signature"))
desc := fmt.Sprintf("%%FILENAME%%\n%s\n\n%%NAME%%\n%s\n\n%%CSIZE%%\n%d\n\n%%SHA256SUM%%\n%s\n",
fname, strings.SplitN(fname, "-", 2)[0], len(data), SHA256Hex(data))
hdr := &tar.Header{
Name: strings.TrimSuffix(fname, "-x86_64.pkg.tar.zst") + "/desc",
Mode: 0644,
Size: int64(len(desc)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(desc)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
WriteFile(t, filepath.Join(dir, name+".db"), buf.Bytes())
WriteFile(t, filepath.Join(dir, name+".files"), buf.Bytes())
return pkgs
}
// BuildApkRepo writes a minimal Alpine repository into dir and returns the
// package contents by file name. The index replicates apk's real layout: a
// signature tar segment with its end-of-archive blocks cut off and the
// index tar as separate gzip streams, concatenated.
func BuildApkRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
pkgs := map[string][]byte{
"musl-1.2.5-r1.apk": bytes.Repeat([]byte("musl"), 300),
"zlib-1.3-r2.apk": bytes.Repeat([]byte("zlib"), 200),
}
index := ""
for fname, data := range pkgs {
name := strings.SplitN(fname, "-", 2)[0]
version := strings.TrimSuffix(strings.TrimPrefix(fname, name+"-"), ".apk")
index += fmt.Sprintf("C:Q1notarealpullchecksum=\nP:%s\nV:%s\nA:x86_64\nS:%d\n\n", name, version, len(data))
WriteFile(t, filepath.Join(dir, fname), data)
}
// tarBytes renders entries into an uncompressed tar archive.
tarBytes := func(entries map[string]string) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for name, content := range entries {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(content))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(content)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// The signature segment loses its two zero end-of-archive blocks so
// the tar stream continues into the index segment.
sig := tarBytes(map[string]string{".SIGN.RSA.test.rsa.pub": "signature"})
sig = sig[:len(sig)-1024]
idx := tarBytes(map[string]string{"DESCRIPTION": "test repo", "APKINDEX": index})
combined := append(GzipBytes(t, sig), GzipBytes(t, idx)...)
WriteFile(t, filepath.Join(dir, "APKINDEX.tar.gz"), combined)
return pkgs
}
// BuildFlatDebRepo writes a minimal flat apt repository into dir and
// returns the package contents by file name.
func BuildFlatDebRepo(t *testing.T, dir string) map[string][]byte {
t.Helper()
data := bytes.Repeat([]byte("flat"), 250)
WriteFile(t, filepath.Join(dir, "hello_3.0_amd64.deb"), data)
packages := []byte(debPackagesStanza("hello", "3.0", "hello_3.0_amd64.deb", data))
packagesGz := GzipBytes(t, packages)
WriteFile(t, filepath.Join(dir, "Packages.gz"), packagesGz)
release := "Architectures: amd64\nSHA256:\n"
for p, d := range map[string][]byte{"Packages": packages, "Packages.gz": packagesGz} {
release += fmt.Sprintf(" %s %d %s\n", SHA256Hex(d), len(d), p)
}
WriteFile(t, filepath.Join(dir, "Release"), []byte(release))
return map[string][]byte{"hello_3.0_amd64.deb": data}
}

29
main.go Normal file
View file

@ -0,0 +1,29 @@
package main
import (
"fmt"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// main parses the command line, configures logging, and dispatches to the
// selected command.
func main() {
ctx := ParseFlags()
setupLogging(flags.LogLevel)
fetch.SetUserAgent(fmt.Sprintf("%s/%s", cfg.Name, cfg.Version))
err := ctx.Run()
ctx.FatalIfErrorf(err)
}
// setupLogging configures the global logger from the command line level.
func setupLogging(level string) {
parsed, err := log.ParseLevel(level)
if err != nil {
parsed = log.InfoLevel
}
log.SetLevel(parsed)
log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
}

169
mirror/apk.go Normal file
View file

@ -0,0 +1,169 @@
package mirror
import (
"archive/tar"
"bufio"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// APKIndexName is the fixed name of an Alpine repository index.
const APKIndexName = "APKINDEX.tar.gz"
// apkPackage is one package entry parsed from an APKINDEX. Alpine's C:
// pull checksum only covers a package's control segment, so mirroring can
// verify downloads by size alone.
type apkPackage struct {
name string
version string
size int64
}
// filename returns the package file name as served beside the index.
func (p *apkPackage) filename() string {
return p.name + "-" + p.version + ".apk"
}
// expect converts the package's metadata into a download expectation.
func (p *apkPackage) expect() *fetch.Expect {
e := &fetch.Expect{Size: p.size}
if p.size <= 0 {
e.Size = -1
}
return e
}
// syncApk synchronizes one Alpine repository from src into destDir.
func syncApk(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
// Stage the index so a live tree keeps a consistent view until the
// packages it references are in place.
idxDst, err := fetch.LocalJoin(destDir, APKIndexName)
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, APKIndexName, idxDst, nil, true, false); err != nil {
return fmt.Errorf("fetch %s: %w", APKIndexName, err)
}
pkgs, err := readApkIndex(fetch.StagedOrFinal(idxDst))
if err != nil {
return err
}
// Download the packages, which live beside the index.
var jobs []fetch.Job
for _, pkg := range pkgs {
dst, err := fetch.LocalJoin(destDir, pkg.filename())
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename(), Dst: dst, Want: pkg.expect()})
}
log.WithField("packages", len(pkgs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// Promote the index last so the published metadata chain is complete.
if err := fetch.PromoteOrDiscard(idxDst, opts.DryRun); err != nil {
return err
}
keep.Add(idxDst)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
return nil
}
// readApkIndex parses an APKINDEX.tar.gz archive. The file is a signature
// segment and an index segment as concatenated gzip streams, which the
// multistream gzip reader and tar reader walk through transparently.
func readApkIndex(filename string) ([]apkPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
defer gz.Close()
tr := tar.NewReader(gz)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
if hdr.Name != "APKINDEX" {
continue
}
pkgs, err := parseApkIndex(tr)
if err != nil {
return nil, fmt.Errorf("parse apk index %s: %w", filename, err)
}
return pkgs, nil
}
return nil, fmt.Errorf("no APKINDEX entry in %s", filename)
}
// parseApkIndex reads APKINDEX stanzas, collecting each package's name,
// version, and file size from the P:, V:, and S: fields.
func parseApkIndex(r io.Reader) ([]apkPackage, error) {
var pkgs []apkPackage
var cur apkPackage
// flush records the current stanza when it is complete.
flush := func() {
if cur.name != "" && cur.version != "" {
pkgs = append(pkgs, cur)
}
cur = apkPackage{}
}
scanner := bufio.NewScanner(r)
scanner.Buffer(make([]byte, 0, 256*1024), 256*1024)
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == "" {
flush()
continue
}
field, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
switch field {
case "P":
cur.name = value
case "V":
cur.version = value
case "S":
cur.size, _ = strconv.ParseInt(value, 10, 64)
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
flush()
return pkgs, nil
}

100
mirror/apk_test.go Normal file
View file

@ -0,0 +1,100 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseApkIndex verifies stanza splitting and field extraction.
func TestParseApkIndex(t *testing.T) {
index := "C:Q1abc=\nP:musl\nV:1.2.5-r1\nA:x86_64\nS:1200\n\nC:Q1def=\nP:zlib\nV:1.3-r2\nS:800\n"
pkgs, err := parseApkIndex(strings.NewReader(index))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Fatalf("got %d packages, want 2", len(pkgs))
}
if pkgs[0].filename() != "musl-1.2.5-r1.apk" || pkgs[0].size != 1200 {
t.Errorf("first package = %+v", pkgs[0])
}
if pkgs[1].filename() != "zlib-1.3-r2.apk" || pkgs[1].size != 800 {
t.Errorf("second package = %+v", pkgs[1])
}
}
// TestReadApkIndex verifies parsing through the signature segment of a
// multistream index archive.
func TestReadApkIndex(t *testing.T) {
dir := t.TempDir()
testrepos.BuildApkRepo(t, dir)
pkgs, err := readApkIndex(filepath.Join(dir, APKIndexName))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Errorf("got %d packages, want 2", len(pkgs))
}
}
// TestSyncApk verifies a full synchronization mirrors packages and the
// index with no staged leftovers.
func TestSyncApk(t *testing.T) {
www := t.TempDir()
pkgs := testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "community", "x86_64"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/alpine/v3.24/community/x86_64", opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "alpine", "v3.24", "community", "x86_64")
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(local, APKIndexName)); err != nil {
t.Error("index not promoted:", err)
}
staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncApkPrune verifies stale packages are removed when pruning.
func TestSyncApkPrune(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "repo"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoApk, Destination: dest, Workers: 2, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "repo", "old-1.0-r0.apk")
if err := os.WriteFile(stale, []byte("old"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/repo", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale package survived pruning")
}
}

258
mirror/arch.go Normal file
View file

@ -0,0 +1,258 @@
package mirror
import (
"archive/tar"
"bufio"
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"sort"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// archPackage is one package entry parsed from a pacman database.
type archPackage struct {
filename string
size int64
sums map[string]string
}
// expect converts the package's metadata into a download expectation.
func (p *archPackage) expect() *fetch.Expect {
e := &fetch.Expect{Size: p.size, Sums: p.sums}
if p.size <= 0 {
e.Size = -1
}
return e
}
// archExtras lists the companion metadata files mirrored beside the
// database, as suffixes appended to the repository name. All are optional;
// which ones exist varies by repository.
var archExtras = []string{
".files",
".db.tar.gz",
".files.tar.gz",
".links.tar.gz",
".db.sig",
".db.tar.gz.sig",
".files.sig",
".files.tar.gz.sig",
".links.tar.gz.sig",
}
// syncArch synchronizes one pacman repository from src into destDir.
func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
// Determine the database name, which matches the repository rather
// than any fixed path.
name, err := archDBName(ctx, src, repoURL)
if err != nil {
return err
}
log.WithField("database", name+".db").Debug("Resolved pacman database.")
// Stage the database so a live tree keeps a consistent view until the
// packages it references are in place.
dbDst, err := fetch.LocalJoin(destDir, name+".db")
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, name+".db", dbDst, nil, true, false); err != nil {
return fmt.Errorf("fetch %s.db: %w", name, err)
}
pkgs, err := readArchDB(fetch.StagedOrFinal(dbDst))
if err != nil {
return err
}
// Download packages with their detached signatures. Signatures have no
// published checksums, so an existing file is trusted as-is.
var jobs []fetch.Job
for _, pkg := range pkgs {
dst, err := fetch.LocalJoin(destDir, pkg.filename)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect()})
jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: &fetch.Expect{Size: -1}, Optional: true})
}
log.WithField("packages", len(pkgs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// Stage the companion metadata files that exist upstream. A dry run
// only counts them, as none are needed for planning.
var extraJobs []fetch.Job
for _, suffix := range archExtras {
dst, err := fetch.LocalJoin(destDir, name+suffix)
if err != nil {
return err
}
extraJobs = append(extraJobs, fetch.Job{ReqPath: name + suffix, Dst: dst, Optional: true, Stage: true})
}
if opts.DryRun {
fetch.PlanJobs(extraJobs, opts.Verify, keep)
} else {
states, err := fetch.Many(ctx, src, extraJobs, opts.Workers, opts.Verify, keep)
if err != nil {
return fmt.Errorf("fetch repository metadata: %w", err)
}
for i, job := range extraJobs {
if states[i] == fetch.FileMissing {
// The upstream dropped the file; drop the local copy so a
// stale companion or signature is never served.
fetch.RemoveStale(job.Dst)
continue
}
if err := fetch.PromoteStaged(job.Dst); err != nil {
return err
}
}
}
// Promote the database last so the published metadata chain is
// complete.
if err := fetch.PromoteOrDiscard(dbDst, opts.DryRun); err != nil {
return err
}
keep.Add(dbDst)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
return nil
}
// archDBName finds the pacman database name for a repository, preferring
// the directory listing and falling back to probing names derived from the
// URL path, deepest segment first.
func archDBName(ctx context.Context, src *fetch.Source, repoURL string) (string, error) {
if listing, err := listDir(ctx, strings.TrimRight(repoURL, "/")+"/"); err == nil {
var names []string
for f := range listing.files {
if strings.HasSuffix(f, ".db") {
names = append(names, strings.TrimSuffix(f, ".db"))
}
}
sort.Strings(names)
if len(names) > 1 {
log.WithField("databases", names).Warn("Multiple pacman databases found; using the first.")
}
if len(names) > 0 {
return names[0], nil
}
}
// Listings can be disabled; a repository at .../core/os/x86_64 is
// probed as x86_64.db, os.db, then core.db.
u, err := url.Parse(repoURL)
if err != nil {
return "", err
}
var segs []string
for _, seg := range strings.Split(u.Path, "/") {
if seg != "" {
segs = append(segs, seg)
}
}
for i := len(segs) - 1; i >= 0; i-- {
resp, err := src.Get(ctx, segs[i]+".db", fetch.GetOptions{})
if err == nil {
resp.Body.Close()
return segs[i], nil
}
if !errors.Is(err, fetch.ErrNotFound) {
return "", err
}
}
return "", fmt.Errorf("no pacman database found at %s", repoURL)
}
// readArchDB parses a pacman database archive and collects each package's
// file name, size, and checksums from its desc entry.
func readArchDB(filename string) ([]archPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
// The database carries no compression extension, so the format is
// sniffed from its leading bytes.
r, closeFn, err := fetch.SniffDecompressor(f)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
tr := tar.NewReader(r)
var pkgs []archPackage
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse pacman database %s: %w", filename, err)
}
if hdr.FileInfo().IsDir() || path.Base(hdr.Name) != "desc" {
continue
}
data, err := io.ReadAll(io.LimitReader(tr, 1<<20))
if err != nil {
return nil, fmt.Errorf("read desc entry %s: %w", hdr.Name, err)
}
fields := parseDesc(data)
if fields["FILENAME"] == "" {
continue
}
size, _ := strconv.ParseInt(fields["CSIZE"], 10, 64)
sums := map[string]string{}
if v := fields["SHA256SUM"]; v != "" {
sums["sha256"] = strings.ToLower(v)
}
if v := fields["MD5SUM"]; v != "" {
sums["md5"] = strings.ToLower(v)
}
pkgs = append(pkgs, archPackage{filename: fields["FILENAME"], size: size, sums: sums})
}
return pkgs, nil
}
// parseDesc extracts the first value of each %FIELD% block from a pacman
// desc entry.
func parseDesc(data []byte) map[string]string {
fields := map[string]string{}
field := ""
scanner := bufio.NewScanner(strings.NewReader(string(data)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
switch {
case line == "":
field = ""
case strings.HasPrefix(line, "%") && strings.HasSuffix(line, "%"):
field = strings.Trim(line, "%")
case field != "":
if _, ok := fields[field]; !ok {
fields[field] = line
}
}
}
return fields
}

166
mirror/arch_test.go Normal file
View file

@ -0,0 +1,166 @@
package mirror
import (
"archive/tar"
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseDesc verifies field extraction from a pacman desc entry.
func TestParseDesc(t *testing.T) {
desc := "%FILENAME%\nzlib-1.3-1-x86_64.pkg.tar.zst\n\n%CSIZE%\n1200\n\n%SHA256SUM%\nabc123\n\n%DESC%\nFirst line\nSecond line\n"
fields := parseDesc([]byte(desc))
if fields["FILENAME"] != "zlib-1.3-1-x86_64.pkg.tar.zst" {
t.Errorf("FILENAME = %q", fields["FILENAME"])
}
if fields["CSIZE"] != "1200" {
t.Errorf("CSIZE = %q", fields["CSIZE"])
}
if fields["SHA256SUM"] != "abc123" {
t.Errorf("SHA256SUM = %q", fields["SHA256SUM"])
}
if fields["DESC"] != "First line" {
t.Errorf("DESC = %q, want first value only", fields["DESC"])
}
}
// TestReadArchDB verifies database parsing for both compressed and plain
// tar archives.
func TestReadArchDB(t *testing.T) {
dir := t.TempDir()
testrepos.BuildArchRepo(t, dir, "core")
pkgs, err := readArchDB(filepath.Join(dir, "core.db"))
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 2 {
t.Fatalf("got %d packages, want 2", len(pkgs))
}
for _, pkg := range pkgs {
if pkg.filename == "" || pkg.size <= 0 || pkg.sums["sha256"] == "" {
t.Errorf("incomplete package entry: %+v", pkg)
}
}
// A plain uncompressed tar database must parse through the sniffer.
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
desc := "%FILENAME%\nplain-1.0-1-any.pkg.tar.zst\n\n%CSIZE%\n10\n\n%SHA256SUM%\nabc\n"
if err := tw.WriteHeader(&tar.Header{Name: "plain-1.0-1/desc", Mode: 0644, Size: int64(len(desc))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(desc)); err != nil {
t.Fatal(err)
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
plain := filepath.Join(dir, "plain.db")
if err := os.WriteFile(plain, buf.Bytes(), 0644); err != nil {
t.Fatal(err)
}
pkgs, err = readArchDB(plain)
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 1 || pkgs[0].filename != "plain-1.0-1-any.pkg.tar.zst" {
t.Errorf("plain tar parse = %+v", pkgs)
}
}
// TestSyncArch verifies a full synchronization mirrors packages,
// signatures, and metadata with no staged leftovers.
func TestSyncArch(t *testing.T) {
www := t.TempDir()
pkgs := testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoArch, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/core/os/x86_64", opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "core", "os", "x86_64")
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
if _, err := os.Stat(filepath.Join(local, name+".sig")); err != nil {
t.Errorf("signature for %s missing: %v", name, err)
}
}
for _, name := range []string{"core.db", "core.files"} {
if _, err := os.Stat(filepath.Join(local, name)); err != nil {
t.Errorf("%s not promoted: %v", name, err)
}
}
staged, _ := filepath.Glob(filepath.Join(local, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncArchPrune verifies stale files are removed when pruning.
func TestSyncArchPrune(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "core"), "core")
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoArch, Destination: dest, Workers: 2, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/core", opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "core", "old-0.9-1-x86_64.pkg.tar.zst")
if err := os.WriteFile(stale, []byte("old"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/core", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale package survived pruning")
}
}
// TestArchDBNameFallback verifies the database name is probed from URL
// path segments when directory listings are unavailable.
func TestArchDBNameFallback(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "custom", "os", "x86_64"), "custom")
// Serve files but refuse directory listings to force the fallback.
fs := http.FileServer(http.Dir(www))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
http.NotFound(w, r)
return
}
fs.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
src := fetch.NewSource([]string{srv.URL + "/custom/os/x86_64"})
name, err := archDBName(context.Background(), src, srv.URL+"/custom/os/x86_64")
if err != nil {
t.Fatal(err)
}
if name != "custom" {
t.Errorf("name = %q, want custom", name)
}
}

100
mirror/control.go Normal file
View file

@ -0,0 +1,100 @@
package mirror
import (
"bufio"
"io"
"strings"
)
// stanza is one RFC 822 style field block from an apt control file.
type stanza map[string]string
// Get returns the first matching field, trying each name in order to cope
// with casing differences between repositories.
func (s stanza) Get(names ...string) string {
for _, name := range names {
if v, ok := s[name]; ok {
return v
}
}
return ""
}
// controlReader reads stanzas from an apt control file.
type controlReader struct {
br *bufio.Reader
}
// newControlReader wraps a reader for stanza-by-stanza parsing.
func newControlReader(r io.Reader) *controlReader {
return &controlReader{br: bufio.NewReaderSize(r, 64*1024)}
}
// readStanza parses the next stanza, returning io.EOF when the input is
// exhausted. Continuation lines are joined with newlines so multi-line
// checksum fields keep one entry per line.
func (r *controlReader) readStanza() (stanza, error) {
st := stanza{}
var last string
for {
line, err := r.br.ReadString('\n')
if err != nil && err != io.EOF {
return nil, err
}
done := err == io.EOF && line == ""
line = strings.TrimRight(line, "\r\n")
switch {
case done || strings.TrimSpace(line) == "":
if len(st) > 0 {
return st, nil
}
if done {
return nil, io.EOF
}
case line[0] == ' ' || line[0] == '\t':
if last != "" {
st[last] += "\n" + strings.TrimSpace(line)
}
default:
name, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
last = name
st[name] = strings.TrimSpace(value)
}
}
}
// stripClearsign extracts the signed payload from a clearsigned InRelease
// document, returning the input unchanged when it is not clearsigned.
func stripClearsign(data []byte) []byte {
const begin = "-----BEGIN PGP SIGNED MESSAGE-----"
const sig = "-----BEGIN PGP SIGNATURE-----"
text := strings.ReplaceAll(string(data), "\r\n", "\n")
idx := strings.Index(text, begin)
if idx < 0 {
return data
}
text = text[idx+len(begin):]
// Skip the armor headers up to the first blank line. Without one the
// document is malformed, so treat it as not clearsigned rather than
// returning the armor headers as payload.
i := strings.Index(text, "\n\n")
if i < 0 {
return data
}
text = text[i+2:]
if i := strings.Index(text, sig); i >= 0 {
text = text[:i]
}
// Reverse the dash escaping applied by the signer.
var b strings.Builder
for _, line := range strings.Split(text, "\n") {
b.WriteString(strings.TrimPrefix(line, "- "))
b.WriteString("\n")
}
return []byte(b.String())
}

70
mirror/control_test.go Normal file
View file

@ -0,0 +1,70 @@
package mirror
import (
"errors"
"io"
"strings"
"testing"
)
// TestReadStanza verifies stanza splitting, continuation joining, and EOF
// handling without a trailing newline.
func TestReadStanza(t *testing.T) {
input := "Package: hello\nSHA256:\n abc 10 path/one\n def 20 path/two\n\nPackage: world\nSize: 42"
cr := newControlReader(strings.NewReader(input))
st, err := cr.readStanza()
if err != nil {
t.Fatal(err)
}
if st.Get("Package") != "hello" {
t.Errorf("Package = %q, want hello", st.Get("Package"))
}
if want := "\nabc 10 path/one\ndef 20 path/two"; st.Get("SHA256") != want {
t.Errorf("SHA256 = %q, want %q", st.Get("SHA256"), want)
}
st, err = cr.readStanza()
if err != nil {
t.Fatal(err)
}
if st.Get("Package") != "world" || st.Get("Size") != "42" {
t.Errorf("second stanza = %v", st)
}
if _, err = cr.readStanza(); !errors.Is(err, io.EOF) {
t.Errorf("expected EOF, got %v", err)
}
}
// TestStanzaGet verifies fallback field names cover casing differences.
func TestStanzaGet(t *testing.T) {
st := stanza{"MD5Sum": "abc"}
if got := st.Get("MD5sum", "MD5Sum"); got != "abc" {
t.Errorf("got %q, want abc", got)
}
if got := st.Get("Missing"); got != "" {
t.Errorf("got %q, want empty", got)
}
}
// TestStripClearsign verifies payload extraction and dash unescaping from a
// clearsigned document, and passthrough of unsigned input.
func TestStripClearsign(t *testing.T) {
signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nOrigin: Test\n- Dashed: line\n-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n"
got := string(stripClearsign([]byte(signed)))
if !strings.Contains(got, "Origin: Test\n") {
t.Errorf("payload missing Origin: %q", got)
}
if !strings.Contains(got, "Dashed: line") || strings.Contains(got, "- Dashed") {
t.Errorf("dash escaping not reversed: %q", got)
}
if strings.Contains(got, "PGP") {
t.Errorf("armor not removed: %q", got)
}
plain := "Origin: Test\n"
if string(stripClearsign([]byte(plain))) != plain {
t.Error("unsigned input was modified")
}
}

526
mirror/deb.go Normal file
View file

@ -0,0 +1,526 @@
package mirror
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// debFile is one file referenced by repository metadata with its expected
// size and checksums, keyed by lowercase algorithm name.
type debFile struct {
path string
size int64
sums map[string]string
}
// expect converts the file's metadata into a download expectation.
func (f *debFile) expect() *fetch.Expect {
e := &fetch.Expect{Size: f.size, Sums: f.sums}
if f.size <= 0 {
e.Size = -1
}
return e
}
// debRelease is the parsed subset of an apt Release file needed to mirror
// a suite.
type debRelease struct {
components []string
architectures []string
acquireByHash bool
files map[string]*debFile
}
// releaseSumFields maps Release checksum block names to algorithm names,
// including the MD5sum casing some repositories use.
var releaseSumFields = map[string]string{
"MD5Sum": "md5sum",
"MD5sum": "md5sum",
"SHA1": "sha1",
"SHA256": "sha256",
"SHA512": "sha512",
}
// byHashDirs maps checksum algorithm names to the by-hash directory names
// apt uses.
var byHashDirs = map[string]string{
"md5sum": "MD5Sum",
"sha1": "SHA1",
"sha256": "SHA256",
"sha512": "SHA512",
}
// parseRelease extracts the fields the sync needs from a Release or
// InRelease document.
func parseRelease(data []byte) (*debRelease, error) {
st, err := newControlReader(bytes.NewReader(stripClearsign(data))).readStanza()
if err != nil {
return nil, fmt.Errorf("parse release file: %w", err)
}
rel := &debRelease{files: map[string]*debFile{}}
rel.components = strings.Fields(st.Get("Components"))
rel.architectures = strings.Fields(st.Get("Architectures"))
rel.acquireByHash = strings.EqualFold(st.Get("Acquire-By-Hash"), "yes")
// Merge each checksum block into one record per file path.
for field, algo := range releaseSumFields {
for _, line := range strings.Split(st.Get(field), "\n") {
parts := strings.Fields(line)
if len(parts) != 3 {
continue
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
continue
}
f := rel.files[parts[2]]
if f == nil {
f = &debFile{path: parts[2], sums: map[string]string{}}
rel.files[parts[2]] = f
}
f.size = size
f.sums[algo] = strings.ToLower(parts[0])
}
}
return rel, nil
}
// syncDeb synchronizes one apt suite or flat repository from repoURL.
func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
// Split the URL into the archive root and the suite path. Repositories
// without a dists directory are flat, with everything relative to the
// repository URL itself.
u, err := url.Parse(repoURL)
if err != nil {
return err
}
suiteRel := ""
rootURL := repoURL
if idx := strings.Index(u.Path, "/dists/"); idx >= 0 {
suiteRel = strings.Trim(u.Path[idx+1:], "/")
root := *u
root.Path = u.Path[:idx]
rootURL = root.String()
src.Rebase(suiteRel)
}
destRoot, err := opts.repoDest(rootURL)
if err != nil {
return err
}
destSuite := destRoot
suitePrefix := ""
if suiteRel != "" {
if destSuite, err = fetch.LocalJoin(destRoot, suiteRel); err != nil {
return err
}
suitePrefix = suiteRel + "/"
}
// Stage the release files; they are promoted last so a live tree keeps
// a consistent metadata chain.
releaseNames := []string{"InRelease", "Release", "Release.gpg"}
releaseStates := make([]fetch.FileState, len(releaseNames))
for i, name := range releaseNames {
dst, err := fetch.LocalJoin(destSuite, name)
if err != nil {
return err
}
state, err := fetch.File(ctx, src, suitePrefix+name, dst, nil, true, false)
if err != nil {
if errors.Is(err, fetch.ErrNotFound) {
// The upstream dropped this release file; drop the local
// copy so outdated metadata is never served.
if !opts.DryRun {
fetch.RemoveStale(dst)
}
continue
}
return fmt.Errorf("fetch %s: %w", name, err)
}
releaseStates[i] = state
}
// Parse the strongest release document available, which may be the
// promoted copy when the upstream reported it unchanged.
var relData []byte
switch {
case releaseStates[0] != fetch.FileMissing:
relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "InRelease")))
case releaseStates[1] != fetch.FileMissing:
relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "Release")))
default:
return errors.New("repository serves neither InRelease nor Release")
}
if err != nil {
return err
}
rel, err := parseRelease(relData)
if err != nil {
return err
}
// Stage every index file the release lists, applying component and
// architecture filters. Missing variants are normal; repositories list
// checksums for files they do not serve. A dry run only fetches the
// package and source indexes needed to plan the pool.
var indexJobs, plannedJobs []fetch.Job
for _, f := range sortedFiles(rel.files) {
// Releases commonly list their own release files; those are
// already staged above and must not be fetched twice.
if slices.Contains(releaseNames, f.path) {
continue
}
if !includeIndexFile(f.path, opts.Components, opts.Architectures) {
continue
}
dst, err := fetch.LocalJoin(destSuite, f.path)
if err != nil {
return err
}
job := fetch.Job{
ReqPath: suitePrefix + f.path,
Dst: dst,
Want: f.expect(),
Optional: true,
Stage: true,
}
if opts.DryRun {
base, _ := splitCompressExt(f.path)
if name := path.Base(base); name != "Packages" && name != "Sources" {
plannedJobs = append(plannedJobs, job)
continue
}
}
indexJobs = append(indexJobs, job)
}
fetch.PlanJobs(plannedJobs, opts.Verify, keep)
states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, opts.Verify, keep)
if err != nil {
return fmt.Errorf("fetch suite indexes: %w", err)
}
// Pick one local compression variant of each package and source index
// to parse for pool file references.
variants := map[string]string{}
variantExt := map[string]string{}
prefOrder := map[string]int{".gz": 0, ".xz": 1, ".zst": 2, ".bz2": 3, "": 4}
for i, job := range indexJobs {
if states[i] == fetch.FileMissing {
continue
}
relPath := strings.TrimPrefix(job.ReqPath, suitePrefix)
base, ext := splitCompressExt(relPath)
if name := path.Base(base); name != "Packages" && name != "Sources" {
continue
}
local := job.Dst
if states[i] == fetch.FileStaged {
local = job.Dst + fetch.StagedSuffix
}
if cur, ok := variantExt[base]; !ok || prefOrder[ext] < prefOrder[cur] {
variantExt[base] = ext
variants[base] = local
}
}
// Collect the pool files referenced by the chosen indexes.
pool := map[string]*debFile{}
for _, base := range sortedKeys(variants) {
files, err := parseIndexFile(variants[base], path.Base(base))
if err != nil {
return fmt.Errorf("parse %s: %w", base, err)
}
for _, f := range files {
if _, ok := pool[f.path]; !ok {
pool[f.path] = f
}
}
}
// Download pool files relative to the archive root.
poolJobs := make([]fetch.Job, 0, len(pool))
for _, f := range sortedFiles(pool) {
dst, err := fetch.LocalJoin(destRoot, f.path)
if err != nil {
return err
}
poolJobs = append(poolJobs, fetch.Job{ReqPath: f.path, Dst: dst, Want: f.expect()})
}
log.WithField("files", len(poolJobs)).Info("Synchronizing pool files.")
if opts.DryRun {
fetch.PlanJobs(poolJobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, opts.Verify, keep); err != nil {
return fmt.Errorf("fetch pool files: %w", err)
}
// Promote the staged indexes now the pool files they reference exist.
for i, job := range indexJobs {
if states[i] == fetch.FileMissing {
continue
}
if err := fetch.PromoteOrDiscard(job.Dst, opts.DryRun); err != nil {
return err
}
if rel.acquireByHash && !opts.DryRun {
addByHash(job.Dst, strings.TrimPrefix(job.ReqPath, suitePrefix), rel, keep)
}
}
// Promote the release files last to complete the metadata chain.
for _, name := range []string{"Release.gpg", "Release", "InRelease"} {
dst, err := fetch.LocalJoin(destSuite, name)
if err != nil {
return err
}
if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil {
return err
}
if _, err := os.Stat(dst); err == nil {
keep.Add(dst)
}
}
// Prune only the suite directory for structured repositories; the pool
// can be shared with other suites of the same archive.
if opts.Prune {
if suiteRel == "" {
fetch.PruneTree(destRoot, keep, opts.PruneGrace, opts.DryRun)
} else {
log.Info("Pruning is limited to the suite directory; shared pool files are kept.")
fetch.PruneTree(destSuite, keep, opts.PruneGrace, opts.DryRun)
}
}
return nil
}
// addByHash links an index file into its directory's by-hash tree for each
// checksum the release lists, matching apt's Acquire-By-Hash layout.
// Failures are logged and do not fail the sync.
func addByHash(dst, relPath string, rel *debRelease, keep *fetch.KeepSet) {
f := rel.files[relPath]
if f == nil || !strings.Contains(relPath, "/") {
return
}
dir := filepath.Dir(dst)
for algo, digest := range f.sums {
sub, ok := byHashDirs[algo]
if !ok {
continue
}
link := filepath.Join(dir, "by-hash", sub, digest)
if err := os.MkdirAll(filepath.Dir(link), 0755); err != nil {
log.WithError(err).WithField("path", link).Warn("Failed to create by-hash directory.")
return
}
if _, err := os.Stat(link); err == nil {
keep.Add(link)
continue
}
if err := os.Link(dst, link); err != nil {
// Fall back to copying when hard links are not supported.
if err := copyFile(dst, link); err != nil {
log.WithError(err).WithField("path", link).Warn("Failed to create by-hash entry.")
continue
}
}
keep.Add(link)
}
}
// copyFile duplicates src to dst.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
_ = os.Remove(dst)
return err
}
return out.Close()
}
// parseIndexFile reads a Packages or Sources index and returns the pool
// files it references, with paths relative to the archive root.
func parseIndexFile(filename, kind string) ([]*debFile, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
cr := newControlReader(r)
var out []*debFile
for {
st, err := cr.readStanza()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, err
}
if kind == "Sources" {
out = append(out, sourceFiles(st)...)
continue
}
name := st.Get("Filename")
if name == "" {
continue
}
size, _ := strconv.ParseInt(st.Get("Size"), 10, 64)
sums := map[string]string{}
if v := st.Get("MD5sum", "MD5Sum"); v != "" {
sums["md5sum"] = strings.ToLower(v)
}
for _, algo := range []string{"SHA1", "SHA256", "SHA512"} {
if v := st.Get(algo); v != "" {
sums[strings.ToLower(algo)] = strings.ToLower(v)
}
}
out = append(out, &debFile{path: path.Clean(name), size: size, sums: sums})
}
return out, nil
}
// sourceSumFields maps source stanza checksum list fields to algorithm
// names; Files is the historical MD5 list.
var sourceSumFields = map[string]string{
"Files": "md5sum",
"Checksums-Sha1": "sha1",
"Checksums-Sha256": "sha256",
"Checksums-Sha512": "sha512",
}
// sourceFiles extracts the files of one source package stanza, merging the
// per-algorithm checksum lists by file name.
func sourceFiles(st stanza) []*debFile {
dir := st.Get("Directory")
byName := map[string]*debFile{}
var order []string
for field, algo := range sourceSumFields {
for _, line := range strings.Split(st.Get(field), "\n") {
parts := strings.Fields(line)
if len(parts) != 3 {
continue
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
continue
}
name := parts[2]
f := byName[name]
if f == nil {
f = &debFile{path: path.Join(dir, name), sums: map[string]string{}}
byName[name] = f
order = append(order, name)
}
f.size = size
f.sums[algo] = strings.ToLower(parts[0])
}
}
out := make([]*debFile, 0, len(byName))
for _, name := range order {
out = append(out, byName[name])
}
return out
}
// includeIndexFile applies the component and architecture filters to a
// release-listed file path.
func includeIndexFile(p string, comps, archs []string) bool {
if len(comps) > 0 {
if segs := strings.Split(p, "/"); len(segs) > 1 && !slices.Contains(comps, segs[0]) {
return false
}
}
if len(archs) > 0 {
if arch := indexArch(p); arch != "" && !slices.Contains(archs, arch) {
return false
}
}
return true
}
// indexArch extracts the architecture a release-listed file applies to, or
// an empty string when the file is architecture independent.
func indexArch(p string) string {
for _, seg := range strings.Split(p, "/") {
if arch, ok := strings.CutPrefix(seg, "binary-"); ok {
return arch
}
if arch, ok := strings.CutPrefix(seg, "installer-"); ok {
return arch
}
if seg == "source" {
return "source"
}
}
base, _ := splitCompressExt(path.Base(p))
if arch, ok := strings.CutPrefix(base, "Contents-udeb-"); ok {
return arch
}
if arch, ok := strings.CutPrefix(base, "Contents-"); ok {
return arch
}
return ""
}
// splitCompressExt splits a known compression extension off a path.
func splitCompressExt(p string) (string, string) {
for _, ext := range []string{".gz", ".xz", ".bz2", ".zst"} {
if strings.HasSuffix(p, ext) {
return strings.TrimSuffix(p, ext), ext
}
}
return p, ""
}
// sortedFiles returns metadata file records in a stable path order.
func sortedFiles(files map[string]*debFile) []*debFile {
out := make([]*debFile, 0, len(files))
for _, f := range files {
out = append(out, f)
}
sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path })
return out
}
// sortedKeys returns a map's keys in a stable order.
func sortedKeys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}

203
mirror/deb_test.go Normal file
View file

@ -0,0 +1,203 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestParseRelease verifies field extraction and checksum block merging.
func TestParseRelease(t *testing.T) {
release := "Origin: Test\nArchitectures: amd64 arm64\nComponents: main universe\nAcquire-By-Hash: yes\n" +
"MD5Sum:\n aaa 10 main/binary-amd64/Packages\n" +
"SHA256:\n bbb 10 main/binary-amd64/Packages\n ccc 20 main/binary-amd64/Packages.gz\n"
rel, err := parseRelease([]byte(release))
if err != nil {
t.Fatal(err)
}
if len(rel.components) != 2 || rel.components[0] != "main" {
t.Errorf("components = %v", rel.components)
}
if len(rel.architectures) != 2 {
t.Errorf("architectures = %v", rel.architectures)
}
if !rel.acquireByHash {
t.Error("acquireByHash not detected")
}
f := rel.files["main/binary-amd64/Packages"]
if f == nil || f.size != 10 || f.sums["md5sum"] != "aaa" || f.sums["sha256"] != "bbb" {
t.Errorf("merged file = %+v", f)
}
if rel.files["main/binary-amd64/Packages.gz"] == nil {
t.Error("gz variant missing")
}
}
// TestIndexArch verifies architecture extraction from release file paths.
func TestIndexArch(t *testing.T) {
cases := map[string]string{
"main/binary-amd64/Packages.gz": "amd64",
"main/debian-installer/binary-i386/Packages": "i386",
"main/installer-armhf/current/images/SHA256SUMS": "armhf",
"main/source/Sources.gz": "source",
"Contents-s390x.gz": "s390x",
"main/Contents-udeb-riscv64.gz": "riscv64",
"main/i18n/Translation-en.gz": "",
"main/dep11/icons-64x64.tar.gz": "",
"Release": "",
}
for p, want := range cases {
if got := indexArch(p); got != want {
t.Errorf("indexArch(%q) = %q, want %q", p, got, want)
}
}
}
// TestIncludeIndexFile verifies component and architecture filtering.
func TestIncludeIndexFile(t *testing.T) {
cases := []struct {
path string
comps []string
archs []string
want bool
}{
{"main/binary-amd64/Packages.gz", nil, nil, true},
{"universe/binary-amd64/Packages.gz", []string{"main"}, nil, false},
{"main/binary-i386/Packages.gz", []string{"main"}, []string{"amd64"}, false},
{"main/binary-amd64/Packages.gz", []string{"main"}, []string{"amd64"}, true},
{"main/i18n/Translation-en.gz", []string{"main"}, []string{"amd64"}, true},
{"Contents-arm64.gz", []string{"main"}, []string{"amd64"}, false},
{"main/source/Sources.gz", nil, []string{"amd64"}, false},
{"main/source/Sources.gz", nil, []string{"amd64", "source"}, true},
}
for _, c := range cases {
if got := includeIndexFile(c.path, c.comps, c.archs); got != c.want {
t.Errorf("includeIndexFile(%q, %v, %v) = %v, want %v", c.path, c.comps, c.archs, got, c.want)
}
}
}
// TestSourceFiles verifies checksum list merging and directory joining for
// source package stanzas.
func TestSourceFiles(t *testing.T) {
st := stanza{
"Directory": "pool/main/h/hello",
"Files": "\naaa 100 hello_1.0.dsc\nbbb 200 hello_1.0.tar.gz",
"Checksums-Sha256": "\nccc 100 hello_1.0.dsc\nddd 200 hello_1.0.tar.gz",
}
files := sourceFiles(st)
if len(files) != 2 {
t.Fatalf("got %d files, want 2", len(files))
}
byPath := map[string]*debFile{}
for _, f := range files {
byPath[f.path] = f
}
dsc := byPath["pool/main/h/hello/hello_1.0.dsc"]
if dsc == nil || dsc.size != 100 || dsc.sums["md5sum"] != "aaa" || dsc.sums["sha256"] != "ccc" {
t.Errorf("dsc = %+v", dsc)
}
}
// TestSyncDebStructured verifies a dists-layout synchronization: pool files
// at the archive root, promoted indexes, by-hash entries, and no staged
// leftovers.
func TestSyncDebStructured(t *testing.T) {
www := t.TempDir()
pool := testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "debian")
for p, data := range pool {
got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(p)))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("pool file %s content mismatch", p)
}
}
suite := filepath.Join(root, "dists", "test")
if _, err := os.Stat(filepath.Join(suite, "Release")); err != nil {
t.Error("Release not promoted:", err)
}
if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages.gz")); err != nil {
t.Error("Packages.gz missing:", err)
}
// The uncompressed variant is listed in the Release file but not
// served, so it must be skipped without failing the sync.
if _, err := os.Stat(filepath.Join(suite, "main", "binary-amd64", "Packages")); err == nil {
t.Error("unserved uncompressed Packages unexpectedly present")
}
byHash, err := filepath.Glob(filepath.Join(suite, "main", "binary-amd64", "by-hash", "SHA256", "*"))
if err != nil || len(byHash) == 0 {
t.Error("by-hash entries missing")
}
staged, _ := filepath.Glob(filepath.Join(suite, "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncDebComponentFilter verifies filtered components are not mirrored.
func TestSyncDebComponentFilter(t *testing.T) {
www := t.TempDir()
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2, Components: []string{"main"}}
if err := syncOne(context.Background(), srv.URL+"/debian/dists/test", opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "debian")
if _, err := os.Stat(filepath.Join(root, "pool", "main", "h", "hello", "hello_1.0_amd64.deb")); err != nil {
t.Error("main pool file missing:", err)
}
if _, err := os.Stat(filepath.Join(root, "pool", "universe")); err == nil {
t.Error("filtered universe pool files present")
}
if _, err := os.Stat(filepath.Join(root, "dists", "test", "universe")); err == nil {
t.Error("filtered universe indexes present")
}
}
// TestSyncDebFlat verifies a repository without a dists directory mirrors
// relative to the repository URL itself.
func TestSyncDebFlat(t *testing.T) {
www := t.TempDir()
pool := testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoDeb, Destination: dest, Workers: 2}
if err := syncOne(context.Background(), srv.URL+"/flat", opts); err != nil {
t.Fatal(err)
}
root := filepath.Join(dest, "flat")
for name, data := range pool {
got, err := os.ReadFile(filepath.Join(root, name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(root, "Release")); err != nil {
t.Error("Release not promoted:", err)
}
}

178
mirror/discover.go Normal file
View file

@ -0,0 +1,178 @@
package mirror
import (
"context"
"fmt"
"io"
"net/url"
"slices"
"sort"
"strings"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
"golang.org/x/net/html"
)
// dirListing holds the entries parsed from an HTML directory index.
type dirListing struct {
dirs []string
files map[string]bool
}
// discoverRepos crawls directory listings under baseURL looking for
// repositories of the given type, descending at most maxDepth levels.
func discoverRepos(ctx context.Context, baseURL string, typ RepoType, maxDepth int) ([]string, error) {
type node struct {
url string
depth int
}
start := strings.TrimRight(baseURL, "/") + "/"
queue := []node{{url: start}}
visited := map[string]bool{start: true}
var found []string
for len(queue) > 0 {
if err := ctx.Err(); err != nil {
return nil, err
}
n := queue[0]
queue = queue[1:]
listing, err := listDir(ctx, n.url)
if err != nil {
log.WithError(err).WithField("url", n.url).Warn("Failed to list directory during discovery.")
continue
}
// A directory containing repository metadata is synchronized as a
// whole; its children are not crawled further.
switch typ {
case RepoRPM:
if slices.Contains(listing.dirs, "repodata") {
found = append(found, strings.TrimRight(n.url, "/"))
continue
}
case RepoDeb:
if listing.files["InRelease"] || listing.files["Release"] {
found = append(found, strings.TrimRight(n.url, "/"))
continue
}
case RepoArch:
if hasArchDB(listing.files) {
found = append(found, strings.TrimRight(n.url, "/"))
continue
}
case RepoApk:
if listing.files[APKIndexName] {
found = append(found, strings.TrimRight(n.url, "/"))
continue
}
}
if n.depth >= maxDepth {
continue
}
for _, dir := range listing.dirs {
child := n.url + url.PathEscape(dir) + "/"
if visited[child] {
continue
}
visited[child] = true
queue = append(queue, node{url: child, depth: n.depth + 1})
}
}
sort.Strings(found)
return found, nil
}
// hasArchDB reports whether a directory listing contains a pacman database.
func hasArchDB(files map[string]bool) bool {
for name := range files {
if strings.HasSuffix(name, ".db") {
return true
}
}
return false
}
// listDir fetches an HTML directory index and extracts its immediate child
// directories and files.
func listDir(ctx context.Context, dirURL string) (*dirListing, error) {
resp, err := fetch.Get(ctx, dirURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("list %s: status %d", dirURL, resp.StatusCode)
}
base, err := url.Parse(dirURL)
if err != nil {
return nil, err
}
listing := &dirListing{files: map[string]bool{}}
seenDirs := map[string]bool{}
tok := html.NewTokenizer(io.LimitReader(resp.Body, 8<<20))
for {
tt := tok.Next()
if tt == html.ErrorToken {
break
}
if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
continue
}
name, hasAttr := tok.TagName()
if string(name) != "a" || !hasAttr {
continue
}
var href string
for {
key, val, more := tok.TagAttr()
if string(key) == "href" {
href = string(val)
}
if !more {
break
}
}
entry, isDir := childEntry(base, href)
if entry == "" {
continue
}
if isDir {
if !seenDirs[entry] {
seenDirs[entry] = true
listing.dirs = append(listing.dirs, entry)
}
} else {
listing.files[entry] = true
}
}
return listing, nil
}
// childEntry resolves an anchor href against the directory URL, returning
// the entry name when it is an immediate child of that directory.
func childEntry(base *url.URL, href string) (string, bool) {
if href == "" || strings.HasPrefix(href, "#") {
return "", false
}
ref, err := url.Parse(href)
if err != nil || ref.RawQuery != "" || ref.Fragment != "" {
return "", false
}
resolved := base.ResolveReference(ref)
if resolved.Scheme != base.Scheme || resolved.Host != base.Host {
return "", false
}
rel := strings.TrimPrefix(resolved.Path, base.Path)
if rel == "" || rel == resolved.Path {
return "", false
}
isDir := strings.HasSuffix(rel, "/")
rel = strings.TrimSuffix(rel, "/")
if rel == "" || strings.Contains(rel, "/") {
return "", false
}
return rel, isDir
}

138
mirror/discover_test.go Normal file
View file

@ -0,0 +1,138 @@
package mirror
import (
"context"
"net/url"
"path/filepath"
"testing"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestChildEntry verifies anchor resolution against a directory URL.
func TestChildEntry(t *testing.T) {
base, _ := url.Parse("http://example.com/repos/")
cases := []struct {
href string
want string
isDir bool
}{
{"el9/", "el9", true},
{"file.rpm", "file.rpm", false},
{"/repos/el8/", "el8", true},
{"../", "", false},
{"?C=N;O=D", "", false},
{"http://other.example.com/repos/x/", "", false},
{"/outside/", "", false},
{"deep/nested/", "", false},
{"#anchor", "", false},
}
for _, c := range cases {
got, isDir := childEntry(base, c.href)
if got != c.want || isDir != c.isDir {
t.Errorf("childEntry(%q) = %q,%v, want %q,%v", c.href, got, isDir, c.want, c.isDir)
}
}
}
// TestDiscoverRepos verifies RPM repositories are found through directory
// listings and that the depth limit is honored.
func TestDiscoverRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "a", "repo1"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "b", "nested", "repo2"))
testrepos.BuildRPMRepo(t, filepath.Join(www, "deep", "x", "y", "z", "repo3"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, RepoRPM, 3)
if err != nil {
t.Fatal(err)
}
want := []string{srv.URL + "/a/repo1", srv.URL + "/b/nested/repo2"}
if len(found) != len(want) {
t.Fatalf("found %v, want %v", found, want)
}
for i := range want {
if found[i] != want[i] {
t.Errorf("found[%d] = %q, want %q", i, found[i], want[i])
}
}
// A deeper limit reaches the third repository.
found, err = discoverRepos(context.Background(), srv.URL, RepoRPM, 6)
if err != nil {
t.Fatal(err)
}
if len(found) != 3 {
t.Errorf("depth 6 found %v, want 3 repositories", found)
}
}
// TestDiscoverArchRepos verifies pacman repositories are recognized by
// their database files.
func TestDiscoverArchRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildArchRepo(t, filepath.Join(www, "core", "os", "x86_64"), "core")
testrepos.BuildArchRepo(t, filepath.Join(www, "extra", "os", "x86_64"), "extra")
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, RepoArch, 4)
if err != nil {
t.Fatal(err)
}
want := []string{srv.URL + "/core/os/x86_64", srv.URL + "/extra/os/x86_64"}
if len(found) != len(want) {
t.Fatalf("found %v, want %v", found, want)
}
for i := range want {
if found[i] != want[i] {
t.Errorf("found[%d] = %q, want %q", i, found[i], want[i])
}
}
}
// TestDiscoverApkRepos verifies Alpine repositories are recognized by
// their index files, matching the arch directories under a release tree.
func TestDiscoverApkRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "x86_64"))
testrepos.BuildApkRepo(t, filepath.Join(www, "v3.24", "community", "aarch64"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL+"/v3.24/community", RepoApk, 1)
if err != nil {
t.Fatal(err)
}
want := []string{srv.URL + "/v3.24/community/aarch64", srv.URL + "/v3.24/community/x86_64"}
if len(found) != len(want) {
t.Fatalf("found %v, want %v", found, want)
}
for i := range want {
if found[i] != want[i] {
t.Errorf("found[%d] = %q, want %q", i, found[i], want[i])
}
}
}
// TestDiscoverDebRepos verifies suite directories are recognized by their
// release files.
func TestDiscoverDebRepos(t *testing.T) {
www := t.TempDir()
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
testrepos.BuildFlatDebRepo(t, filepath.Join(www, "flat"))
srv := testrepos.ServeDir(t, www)
found, err := discoverRepos(context.Background(), srv.URL, RepoDeb, 4)
if err != nil {
t.Fatal(err)
}
want := []string{srv.URL + "/debian/dists/test", srv.URL + "/flat"}
if len(found) != len(want) {
t.Fatalf("found %v, want %v", found, want)
}
for i := range want {
if found[i] != want[i] {
t.Errorf("found[%d] = %q, want %q", i, found[i], want[i])
}
}
}

14
mirror/mirror_test.go Normal file
View file

@ -0,0 +1,14 @@
package mirror
import (
"os"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
)
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
fetch.Reload()
os.Exit(m.Run())
}

91
mirror/options.go Normal file
View file

@ -0,0 +1,91 @@
// Package mirror synchronizes remote package repositories into a local
// directory tree, preserving the upstream layout so the result can be
// served directly to package managers.
package mirror
import (
"net/url"
"path"
"strings"
"time"
"github.com/grmrgecko/repo-sync/fetch"
)
// RepoType identifies the repository format being synchronized.
type RepoType string
const (
// RepoRPM synchronizes yum/dnf style repositories.
RepoRPM RepoType = "rpm"
// RepoDeb synchronizes apt style repositories.
RepoDeb RepoType = "deb"
// RepoArch synchronizes pacman style repositories.
RepoArch RepoType = "arch"
// RepoApk synchronizes Alpine apk style repositories.
RepoApk RepoType = "apk"
)
// Options configures a synchronization run.
type Options struct {
// Type selects the repository format.
Type RepoType
// URLs lists the repository or mirrorlist URLs to synchronize.
URLs []string
// Destination is the local directory repositories are placed under.
Destination string
// Trim drops this many leading URL path components when building the
// destination path.
Trim int
// Flat places repositories directly in Destination without copying the
// URL path.
Flat bool
// Discover crawls each URL's directory listings for repositories.
Discover bool
// DiscoverDepth limits how deep discovery crawls below each URL.
DiscoverDepth int
// Workers is the number of concurrent download workers.
Workers int
// Verify re-verifies checksums of existing local files.
Verify bool
// Prune deletes local files no longer part of the repository.
Prune bool
// PruneGrace defers pruning a file until it has been unreferenced for
// this long across runs.
PruneGrace time.Duration
// DryRun reports what would change without downloading packages or
// altering the published repository.
DryRun bool
// destBase, when set by the mirror server, pins the repository root
// directory directly instead of deriving it from the URL path.
destBase string
// Components limits deb synchronization to these components.
Components []string
// Architectures limits deb synchronization to these architectures.
Architectures []string
}
// repoDest maps a repository URL onto the destination directory according
// to the configured trimming rules.
func (o *Options) repoDest(repoURL string) (string, error) {
if o.destBase != "" {
return o.destBase, nil
}
if o.Flat {
return o.Destination, nil
}
u, err := url.Parse(repoURL)
if err != nil {
return "", err
}
var segs []string
for _, seg := range strings.Split(u.Path, "/") {
if seg != "" {
segs = append(segs, seg)
}
}
if o.Trim >= len(segs) {
return o.Destination, nil
}
return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...))
}

31
mirror/options_test.go Normal file
View file

@ -0,0 +1,31 @@
package mirror
import (
"path/filepath"
"testing"
)
// TestRepoDest verifies URL path copying, trimming, and flat placement.
func TestRepoDest(t *testing.T) {
dest := t.TempDir()
repoURL := "https://example.com/repos/CentOS/7/EA4/"
cases := []struct {
name string
opts Options
want string
}{
{"default", Options{Destination: dest}, filepath.Join(dest, "repos", "CentOS", "7", "EA4")},
{"trim2", Options{Destination: dest, Trim: 2}, filepath.Join(dest, "7", "EA4")},
{"trim-all", Options{Destination: dest, Trim: 10}, dest},
{"flat", Options{Destination: dest, Flat: true}, dest},
}
for _, c := range cases {
got, err := c.opts.repoDest(repoURL)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if got != c.want {
t.Errorf("%s: got %q, want %q", c.name, got, c.want)
}
}
}

90
mirror/resolve.go Normal file
View file

@ -0,0 +1,90 @@
package mirror
import (
"context"
"errors"
"fmt"
"net/url"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// probePaths lists the metadata files that identify a repository of a type.
// Types without fixed-name metadata return nothing.
func probePaths(typ RepoType) []string {
switch typ {
case RepoRPM:
return []string{"repodata/repomd.xml"}
case RepoDeb:
return []string{"InRelease", "Release"}
case RepoApk:
return []string{APKIndexName}
}
return nil
}
// hasQueryOrFragment reports whether a URL carries a query or fragment,
// which makes it unusable as a base for joining repository paths onto.
// Unparsable URLs are reported as plain so the caller still probes them.
func hasQueryOrFragment(rawURL string) bool {
u, err := url.Parse(rawURL)
if err != nil {
return false
}
return u.RawQuery != "" || u.Fragment != ""
}
// resolveSource decides whether repoURL is a repository root or a mirror
// list and returns the source to fetch from, along with the URL whose path
// reflects the repository layout: repoURL itself for a direct repository,
// or the first mirror's URL when repoURL is a mirror list. RPM mirror lists
// may be plain text or metalink documents.
func resolveSource(ctx context.Context, repoURL string, typ RepoType) (*fetch.Source, string, error) {
// Pacman databases are named after the repository, so there is no
// fixed path to probe and pacman mirrorlists are config snippets
// rather than URL lists; the URL is used directly.
direct := fetch.NewSource([]string{repoURL})
probes := probePaths(typ)
if len(probes) == 0 {
return direct, repoURL, nil
}
// Probe for repository metadata directly under the URL. A URL carrying a
// query or fragment cannot have a repository path joined onto it, as the
// path would land inside the query, so skip the probe: such URLs are
// mirrorlist or metalink endpoints and are read as one below.
if !hasQueryOrFragment(repoURL) {
for _, probe := range probes {
resp, err := direct.Get(ctx, probe, fetch.GetOptions{})
if err == nil {
resp.Body.Close()
return direct, repoURL, nil
}
if !errors.Is(err, fetch.ErrNotFound) {
return nil, "", err
}
}
}
// Fall back to reading the URL itself as a mirror list.
body, err := fetch.ReadURL(ctx, repoURL)
if err != nil {
return nil, "", fmt.Errorf("read mirrorlist %s: %w", repoURL, err)
}
kind := "mirrorlist"
var bases []string
if typ == RepoRPM {
if bases = fetch.ParseMetalink(body); len(bases) > 0 {
kind = "metalink"
}
}
if len(bases) == 0 {
bases = fetch.ParseMirrorlist(body)
}
if len(bases) == 0 {
return nil, "", fmt.Errorf("%s is neither a repository nor a mirrorlist", repoURL)
}
log.WithFields(log.Fields{"url": repoURL, "mirrors": len(bases), "kind": kind}).Info("Using mirror list for repository.")
return fetch.NewSource(bases), bases[0], nil
}

123
mirror/resolve_test.go Normal file
View file

@ -0,0 +1,123 @@
package mirror
import (
"context"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// TestResolveSource verifies direct repositories are used as-is and
// mirrorlist bodies expand into multi-mirror sources.
func TestResolveSource(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
// The mirrorlist needs the real server URL, which is only known now
// that the listener is up; files are read per-request.
testrepos.WriteFile(t, filepath.Join(www, "ml.txt"),
[]byte("http://127.0.0.1:1/el9\n"+srv.URL+"/el9\n"))
direct, layoutURL, err := resolveSource(context.Background(), srv.URL+"/el9", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(direct.Bases()) != 1 {
t.Errorf("direct bases = %v", direct.Bases())
}
if layoutURL != srv.URL+"/el9" {
t.Errorf("direct layout URL = %q", layoutURL)
}
ml, layoutURL, err := resolveSource(context.Background(), srv.URL+"/ml.txt", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(ml.Bases()) != 2 {
t.Errorf("mirrorlist bases = %v", ml.Bases())
}
if layoutURL != "http://127.0.0.1:1/el9" {
t.Errorf("mirrorlist layout URL = %q", layoutURL)
}
if _, _, err := resolveSource(context.Background(), srv.URL+"/el9/Packages", RepoRPM); err == nil {
t.Error("expected error for a non-repository URL")
}
}
// TestResolveSourceQueryMirrorlist verifies a mirrorlist URL carrying a
// query string is read as a mirrorlist rather than probed as a repository.
// A repository path cannot be joined onto such a URL: it would land inside
// the query, and endpoints that answer 200 with an error page for an
// unrecognized parameter would then look like a repository.
func TestResolveSourceQueryMirrorlist(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
repo := testrepos.ServeDir(t, www)
var probed []string
list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
probed = append(probed, r.URL.String())
if r.URL.Query().Get("repo") != "os" {
// Several real mirrorlist services answer 200 with a message.
w.Write([]byte("Invalid release/repo/arch combination\n"))
return
}
w.Write([]byte(repo.URL + "/el9\n"))
}))
t.Cleanup(list.Close)
src, layoutURL, err := resolveSource(context.Background(), list.URL+"/?release=9&repo=os", RepoRPM)
if err != nil {
t.Fatal(err)
}
if bases := src.Bases(); len(bases) != 1 || bases[0] != repo.URL+"/el9" {
t.Errorf("bases = %v, want [%s]", bases, repo.URL+"/el9")
}
if layoutURL != repo.URL+"/el9" {
t.Errorf("layout URL = %q, want %q", layoutURL, repo.URL+"/el9")
}
for _, u := range probed {
if strings.Contains(u, "repodata") {
t.Errorf("probe folded a repository path into the query: %s", u)
}
}
}
// TestResolveSourceMetalink verifies metalink documents expand into
// multi-mirror sources for RPM repositories.
func TestResolveSourceMetalink(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
metalink := `<?xml version="1.0"?><metalink><files><file name="repomd.xml"><resources>` +
`<url protocol="http">http://127.0.0.1:1/dead/repodata/repomd.xml</url>` +
`<url protocol="http">` + srv.URL + `/el9/repodata/repomd.xml</url>` +
`</resources></file></files></metalink>`
testrepos.WriteFile(t, filepath.Join(www, "metalink.xml"), []byte(metalink))
src, layoutURL, err := resolveSource(context.Background(), srv.URL+"/metalink.xml", RepoRPM)
if err != nil {
t.Fatal(err)
}
if len(src.Bases()) != 2 || src.Bases()[1] != srv.URL+"/el9" {
t.Errorf("bases = %v", src.Bases())
}
if layoutURL != "http://127.0.0.1:1/dead" {
t.Errorf("layout URL = %q", layoutURL)
}
// The dead first mirror must fail over to the live one.
resp, err := src.Get(context.Background(), "repodata/repomd.xml", fetch.GetOptions{})
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
}

328
mirror/rpm.go Normal file
View file

@ -0,0 +1,328 @@
package mirror
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"os"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// repoMD is the partial XML schema for a yum repomd.xml file. Only the
// fields the sync needs are decoded; unknown elements are ignored so
// upstream additions do not break parsing.
type repoMD struct {
Data []repoData `xml:"data"`
}
// repoData is one metadata file referenced from repomd.xml.
type repoData struct {
Type string `xml:"type,attr"`
Checksum repoChecksum `xml:"checksum"`
Location repoLocation `xml:"location"`
Size int64 `xml:"size"`
}
// repoChecksum is a checksum element with its algorithm type.
type repoChecksum struct {
Type string `xml:"type,attr"`
Value string `xml:",chardata"`
}
// repoLocation captures the relative href of a metadata or package file.
type repoLocation struct {
Href string `xml:"href,attr"`
}
// rpmPackage is the minimal package element decoded from primary metadata,
// carrying the location, checksum, and size needed to mirror the file.
type rpmPackage struct {
Checksum repoChecksum `xml:"checksum"`
Size rpmSize `xml:"size"`
Location repoLocation `xml:"location"`
}
// rpmSize carries the on-disk package size from primary metadata.
type rpmSize struct {
Package int64 `xml:"package,attr"`
}
// syncRPM synchronizes one RPM repository from src into destDir.
func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Options) error {
keep := fetch.NewKeepSet(opts.Prune)
// Stage repomd.xml so clients of a live tree keep a consistent view of
// the old repository until everything else is in place.
repomdDst, err := fetch.LocalJoin(destDir, "repodata/repomd.xml")
if err != nil {
return err
}
if _, err := fetch.File(ctx, src, "repodata/repomd.xml", repomdDst, nil, true, false); err != nil {
return fmt.Errorf("fetch repomd.xml: %w", err)
}
md, err := readRepomd(fetch.StagedOrFinal(repomdDst))
if err != nil {
return err
}
// Download the metadata files referenced by repomd. Their names embed
// content hashes, so they can be written directly into place. A dry run
// only fetches the indexes that must be parsed, staged for discard.
var jobs, planned []fetch.Job
var primary *repoData
var deltaMDs []*repoData
var parseDsts []string
for i, data := range md.Data {
if data.Location.Href == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, data.Location.Href)
if err != nil {
return err
}
job := fetch.Job{
ReqPath: data.Location.Href,
Dst: dst,
Want: fetch.MakeExpect(data.Size, data.Checksum.Type, data.Checksum.Value),
}
parsed := false
switch data.Type {
case "primary":
primary = &md.Data[i]
parsed = true
case "prestodelta", "deltainfo":
deltaMDs = append(deltaMDs, &md.Data[i])
parsed = true
}
if opts.DryRun {
if !parsed {
planned = append(planned, job)
continue
}
job.Stage = true
parseDsts = append(parseDsts, dst)
}
jobs = append(jobs, job)
}
fetch.PlanJobs(planned, opts.Verify, keep)
if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil {
return fmt.Errorf("fetch repository metadata: %w", err)
}
if primary == nil {
return errors.New("repository metadata lists no primary index")
}
// Collect package locations from the primary index, plus any delta
// packages referenced by prestodelta metadata.
primaryPath, err := fetch.LocalJoin(destDir, primary.Location.Href)
if err != nil {
return err
}
pkgs, err := readPrimary(fetch.StagedOrFinal(primaryPath))
if err != nil {
return err
}
jobs = jobs[:0]
for _, pkg := range pkgs {
if pkg.Location.Href == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, pkg.Location.Href)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{
ReqPath: pkg.Location.Href,
Dst: dst,
Want: fetch.MakeExpect(pkg.Size.Package, pkg.Checksum.Type, pkg.Checksum.Value),
})
}
for _, deltaMD := range deltaMDs {
deltaPath, err := fetch.LocalJoin(destDir, deltaMD.Location.Href)
if err != nil {
return err
}
deltas, err := readDeltas(fetch.StagedOrFinal(deltaPath))
if err != nil {
return err
}
for _, delta := range deltas {
if delta.Filename == "" {
continue
}
dst, err := fetch.LocalJoin(destDir, delta.Filename)
if err != nil {
return err
}
jobs = append(jobs, fetch.Job{
ReqPath: delta.Filename,
Dst: dst,
Want: fetch.MakeExpect(delta.Size, delta.Checksum.Type, delta.Checksum.Value),
})
}
}
log.WithField("packages", len(jobs)).Info("Synchronizing packages.")
if opts.DryRun {
fetch.PlanJobs(jobs, opts.Verify, keep)
} else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep); err != nil {
return fmt.Errorf("fetch packages: %w", err)
}
// A dry run discards the indexes it staged for parsing.
for _, dst := range parseDsts {
_ = os.Remove(dst + fetch.StagedSuffix)
}
// Stage the optional signature material so it is published together
// with the repomd.xml it signs, never beside the previous index.
var sigDsts []string
for _, extra := range []string{"repodata/repomd.xml.asc", "repodata/repomd.xml.key"} {
dst, err := fetch.LocalJoin(destDir, extra)
if err != nil {
return err
}
_, err = fetch.File(ctx, src, extra, dst, nil, true, false)
switch {
case err == nil:
sigDsts = append(sigDsts, dst)
case errors.Is(err, fetch.ErrNotFound):
// The upstream dropped the signature; drop the local copy so
// a stale signature is never served beside a new repomd.xml.
if !opts.DryRun {
fetch.RemoveStale(dst)
}
default:
return fmt.Errorf("fetch %s: %w", extra, err)
}
}
// Promote the signatures and then repomd.xml so the published metadata
// chain is complete.
for _, dst := range sigDsts {
if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil {
return err
}
if _, err := os.Stat(dst); err == nil {
keep.Add(dst)
}
}
if err := fetch.PromoteOrDiscard(repomdDst, opts.DryRun); err != nil {
return err
}
keep.Add(repomdDst)
// Remove files that are no longer part of the repository.
if opts.Prune {
fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun)
}
return nil
}
// rpmDelta is one delta package referenced from prestodelta metadata.
type rpmDelta struct {
Filename string `xml:"filename"`
Size int64 `xml:"size"`
Checksum repoChecksum `xml:"checksum"`
}
// readDeltas streams prestodelta metadata and collects each delta package
// reference so drpm files are mirrored alongside full packages.
func readDeltas(filename string) ([]rpmDelta, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
dec := xml.NewDecoder(r)
var deltas []rpmDelta
for {
tok, err := dec.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse delta metadata %s: %w", filename, err)
}
start, ok := tok.(xml.StartElement)
if !ok || start.Name.Local != "delta" {
continue
}
var delta rpmDelta
if err := dec.DecodeElement(&delta, &start); err != nil {
return nil, fmt.Errorf("parse delta entry %s: %w", filename, err)
}
if delta.Filename != "" {
deltas = append(deltas, delta)
}
}
return deltas, nil
}
// readRepomd decodes a repomd.xml file.
func readRepomd(filename string) (*repoMD, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
var md repoMD
if err := xml.NewDecoder(f).Decode(&md); err != nil {
return nil, fmt.Errorf("parse repomd %s: %w", filename, err)
}
return &md, nil
}
// readPrimary streams a primary metadata index and collects each package
// entry without loading the whole document into memory.
func readPrimary(filename string) ([]rpmPackage, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r, closeFn, err := fetch.Decompressor(f, filename)
if err != nil {
return nil, err
}
if closeFn != nil {
defer closeFn()
}
dec := xml.NewDecoder(r)
var pkgs []rpmPackage
for {
tok, err := dec.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("parse primary metadata %s: %w", filename, err)
}
start, ok := tok.(xml.StartElement)
if !ok || start.Name.Local != "package" {
continue
}
var pkg rpmPackage
if err := dec.DecodeElement(&pkg, &start); err != nil {
return nil, fmt.Errorf("parse package entry %s: %w", filename, err)
}
if pkg.Location.Href != "" {
pkgs = append(pkgs, pkg)
}
}
return pkgs, nil
}

171
mirror/rpm_test.go Normal file
View file

@ -0,0 +1,171 @@
package mirror
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// syncRPMFixture builds a served RPM repository, synchronizes it, and
// returns the fixture directory, repository URL, and local repository path.
func syncRPMFixture(t *testing.T, opts *Options) (string, string, string, map[string][]byte) {
t.Helper()
www := t.TempDir()
repoDir := filepath.Join(www, "repos", "el9")
pkgs := testrepos.BuildRPMRepo(t, repoDir)
srv := testrepos.ServeDir(t, www)
repoURL := srv.URL + "/repos/el9"
opts.Type = RepoRPM
if opts.Destination == "" {
opts.Destination = t.TempDir()
}
if opts.Workers == 0 {
opts.Workers = 2
}
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
return repoDir, repoURL, filepath.Join(opts.Destination, "repos", "el9"), pkgs
}
// TestSyncRPM verifies a full synchronization mirrors packages and metadata
// with no staged leftovers.
func TestSyncRPM(t *testing.T) {
_, _, local, pkgs := syncRPMFixture(t, &Options{})
for name, data := range pkgs {
got, err := os.ReadFile(filepath.Join(local, "Packages", name))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data) {
t.Errorf("package %s content mismatch", name)
}
}
if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml")); err != nil {
t.Error("repomd.xml not promoted:", err)
}
if _, err := os.Stat(filepath.Join(local, "repodata", "repomd.xml.asc")); err != nil {
t.Error("repomd.xml.asc not promoted:", err)
}
drpm, err := os.ReadFile(filepath.Join(local, "drpms", "foo-0.9_1.0-1.x86_64.drpm"))
if err != nil {
t.Error("delta package not mirrored:", err)
} else if !bytes.Equal(drpm, testrepos.RPMDelta()) {
t.Error("delta package content mismatch")
}
staged, _ := filepath.Glob(filepath.Join(local, "repodata", "*"+fetch.StagedSuffix))
if len(staged) != 0 {
t.Errorf("staged leftovers remain: %v", staged)
}
}
// TestSyncRPMVerify verifies size-only skipping keeps same-size corruption
// while the verify option repairs it.
func TestSyncRPMVerify(t *testing.T) {
opts := &Options{}
repoDir, repoURL, local, _ := syncRPMFixture(t, opts)
_ = repoDir
// Corrupt a local package without changing its size.
victim := filepath.Join(local, "Packages", "foo-1.0-1.x86_64.rpm")
corrupt := bytes.Repeat([]byte("oof"), 500)
if err := os.WriteFile(victim, corrupt, 0644); err != nil {
t.Fatal(err)
}
// A plain re-run trusts the matching size and keeps the corruption.
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(victim)
if !bytes.Equal(got, corrupt) {
t.Error("size-only run unexpectedly rewrote the file")
}
// A verify run re-hashes local files and repairs the corruption.
opts.Verify = true
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
got, _ = os.ReadFile(victim)
if !bytes.Equal(got, bytes.Repeat([]byte("foo"), 500)) {
t.Error("verify run did not repair the corruption")
}
}
// TestSyncRPMPrune verifies stale files are removed only when pruning is
// enabled.
func TestSyncRPMPrune(t *testing.T) {
opts := &Options{}
_, repoURL, local, _ := syncRPMFixture(t, opts)
stale := filepath.Join(local, "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Error("file pruned without the prune option")
}
opts.Prune = true
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale file survived pruning")
}
}
// TestSyncRPMDroppedSignature verifies a signature the upstream stops
// serving is removed locally instead of lingering beside a new repomd.xml.
func TestSyncRPMDroppedSignature(t *testing.T) {
opts := &Options{}
repoDir, repoURL, local, _ := syncRPMFixture(t, opts)
ascLocal := filepath.Join(local, "repodata", "repomd.xml.asc")
if _, err := os.Stat(ascLocal); err != nil {
t.Fatal("signature not mirrored:", err)
}
if err := os.Remove(filepath.Join(repoDir, "repodata", "repomd.xml.asc")); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), repoURL, opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(ascLocal); !os.IsNotExist(err) {
t.Error("stale signature survived after the upstream dropped it")
}
}
// TestSyncRPMChecksumMismatch verifies a package whose served content does
// not match the published checksum fails the synchronization.
func TestSyncRPMChecksumMismatch(t *testing.T) {
www := t.TempDir()
repoDir := filepath.Join(www, "repos", "el9")
testrepos.BuildRPMRepo(t, repoDir)
srv := testrepos.ServeDir(t, www)
// Replace a served package with same-size different content after the
// metadata was generated.
victim := filepath.Join(repoDir, "Packages", "foo-1.0-1.x86_64.rpm")
if err := os.WriteFile(victim, bytes.Repeat([]byte("fox"), 500), 0644); err != nil {
t.Fatal(err)
}
opts := &Options{Type: RepoRPM, Destination: t.TempDir(), Workers: 2}
err := syncOne(context.Background(), srv.URL+"/repos/el9", opts)
if err == nil || !strings.Contains(err.Error(), "checksum") {
t.Errorf("expected checksum mismatch error, got %v", err)
}
}

177
mirror/sync.go Normal file
View file

@ -0,0 +1,177 @@
package mirror
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
"time"
"github.com/grmrgecko/repo-sync/fetch"
log "github.com/sirupsen/logrus"
)
// lockDestination takes an exclusive lock under the destination directory
// so overlapping runs, such as from cron, cannot race on the same tree.
// The caller closes the returned file to release the lock.
func lockDestination(dest string) (*os.File, error) {
if err := os.MkdirAll(dest, 0755); err != nil {
return nil, err
}
f, err := os.OpenFile(filepath.Join(dest, fetch.LockFileName), os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return nil, err
}
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
f.Close()
return nil, fmt.Errorf("destination %s is locked by another repo-sync run", dest)
}
return f, nil
}
// Sync synchronizes every configured repository URL into the destination,
// continuing after per-repository failures so one bad repository does not
// block the rest.
func Sync(ctx context.Context, opts *Options) error {
fetch.Reload()
// Hold the destination for the whole run; the lock file itself stays
// behind, as removing it would race a waiting run.
lock, err := lockDestination(opts.Destination)
if err != nil {
return err
}
defer lock.Close()
// Expand discovery URLs into concrete repository URLs.
repoURLs := opts.URLs
if opts.Discover {
repoURLs = nil
for _, base := range opts.URLs {
found, err := discoverRepos(ctx, base, opts.Type, opts.DiscoverDepth)
if err != nil {
return fmt.Errorf("discover repositories under %s: %w", base, err)
}
log.WithFields(log.Fields{"url": base, "repositories": len(found)}).Info("Discovered repositories.")
repoURLs = append(repoURLs, found...)
}
}
if len(repoURLs) == 0 {
return errors.New("no repositories to synchronize")
}
// Synchronize each repository in turn.
var failures int
for _, repoURL := range repoURLs {
if err := ctx.Err(); err != nil {
return err
}
log.WithField("url", repoURL).Info("Synchronizing repository.")
fetch.Stats.Reset()
start := time.Now()
if err := syncOne(ctx, repoURL, opts); err != nil {
if errors.Is(err, context.Canceled) {
return err
}
log.WithError(err).WithField("url", repoURL).Error("Repository synchronization failed.")
failures++
continue
}
logSummary(repoURL, opts, time.Since(start))
}
if failures > 0 {
return fmt.Errorf("%d of %d repositories failed to synchronize", failures, len(repoURLs))
}
return nil
}
// logSummary reports one repository's transfer activity after a successful
// synchronization.
func logSummary(repoURL string, opts *Options, elapsed time.Duration) {
fields := log.Fields{
"url": repoURL,
"fetched": fetch.Stats.Fetched.Load(),
"transferred": fetch.FormatBytes(fetch.Stats.FetchedBytes.Load()),
"unchanged": fetch.Stats.Unchanged.Load(),
"duration": elapsed.Round(time.Millisecond).String(),
}
if opts.DryRun {
fields["would_fetch"] = fetch.Stats.Planned.Load()
fields["would_transfer"] = fetch.FormatBytes(fetch.Stats.PlannedBytes.Load())
if opts.Prune {
fields["would_prune"] = fetch.Stats.Pruned.Load()
}
log.WithFields(fields).Info("Dry run complete.")
return
}
if opts.Prune {
fields["pruned"] = fetch.Stats.Pruned.Load()
}
log.WithFields(fields).Info("Repository synchronized.")
}
// syncOne resolves the source for a single repository URL and runs the
// format-specific synchronization. Destination paths derive from the layout
// URL, so a mirror list maps to the first mirror's path rather than the
// list's own URL.
func syncOne(ctx context.Context, repoURL string, opts *Options) error {
src, layoutURL, err := resolveSource(ctx, repoURL, opts.Type)
if err != nil {
return err
}
switch opts.Type {
case RepoRPM:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncRPM(ctx, src, destDir, opts)
case RepoDeb:
return syncDeb(ctx, src, layoutURL, opts)
case RepoArch:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncArch(ctx, src, layoutURL, destDir, opts)
case RepoApk:
destDir, err := opts.repoDest(layoutURL)
if err != nil {
return err
}
log.WithField("destination", destDir).Debug("Resolved repository destination.")
return syncApk(ctx, src, destDir, opts)
default:
return fmt.Errorf("unsupported repository type %q", opts.Type)
}
}
// SyncInto synchronizes one repository into an explicit destination
// directory, bypassing URL-derived destination mapping. It exists for the
// mirror server, whose destinations follow request paths rather than
// upstream URL paths. For deb the destination is the archive root
// directory; for other types it is the repository directory itself.
// Concurrent callers interleave into fetch.Stats, so the counters carry no
// per-repository meaning on this path and are neither reset nor reported.
func SyncInto(ctx context.Context, typ RepoType, src *fetch.Source, repoURL, dest string, opts *Options) error {
switch typ {
case RepoRPM:
return syncRPM(ctx, src, dest, opts)
case RepoDeb:
// Copy before pinning the destination so the caller's Options is
// never mutated, keeping the struct safe to reuse or share.
debOpts := *opts
debOpts.destBase = dest
return syncDeb(ctx, src, repoURL, &debOpts)
case RepoArch:
return syncArch(ctx, src, repoURL, dest, opts)
case RepoApk:
return syncApk(ctx, src, dest, opts)
default:
return fmt.Errorf("unsupported repository type %q", typ)
}
}

158
mirror/sync_test.go Normal file
View file

@ -0,0 +1,158 @@
package mirror
import (
"context"
"encoding/json"
"io/fs"
"os"
"path/filepath"
"testing"
"time"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
)
// countFiles returns the number of regular files under root.
func countFiles(t *testing.T, root string) int {
t.Helper()
count := 0
err := filepath.WalkDir(root, func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
count++
}
return nil
})
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
return count
}
// TestDryRun verifies a dry run leaves the destination unchanged: no
// packages downloaded, no metadata published, and no files pruned.
func TestDryRun(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
// A dry run against an empty destination writes no files at all.
dest := t.TempDir()
opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, DryRun: true, Prune: true}
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
if n := countFiles(t, dest); n != 0 {
t.Errorf("dry run left %d files in the destination", n)
}
// A real run works, and a later dry run with pruning keeps stale files.
opts.DryRun = false
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
stale := filepath.Join(dest, "el9", "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
opts.DryRun = true
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Error("dry run pruned a file:", err)
}
}
// TestPruneGrace verifies stale files wait out the grace period across
// runs before being removed.
func TestPruneGrace(t *testing.T) {
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "el9"))
srv := testrepos.ServeDir(t, www)
dest := t.TempDir()
opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2, Prune: true, PruneGrace: time.Hour}
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
local := filepath.Join(dest, "el9")
stale := filepath.Join(local, "Packages", "stale.rpm")
if err := os.WriteFile(stale, []byte("junk"), 0644); err != nil {
t.Fatal(err)
}
// The first run only starts the grace period.
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Fatal("file pruned before its grace period expired:", err)
}
statePath := filepath.Join(local, fetch.PruneStateName)
if _, err := os.Stat(statePath); err != nil {
t.Fatal("prune state not recorded:", err)
}
// A second run within the grace period still keeps the file.
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err != nil {
t.Fatal("file pruned within its grace period:", err)
}
// Age the recorded timestamp past the grace period; the next run
// prunes the file and clears the bookkeeping.
data, err := os.ReadFile(statePath)
if err != nil {
t.Fatal(err)
}
state := map[string]time.Time{}
if err := json.Unmarshal(data, &state); err != nil {
t.Fatal(err)
}
for rel := range state {
state[rel] = time.Now().Add(-2 * time.Hour)
}
data, err = json.Marshal(state)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatal(err)
}
if err := syncOne(context.Background(), srv.URL+"/el9", opts); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); err == nil {
t.Error("stale file survived past its grace period")
}
if _, err := os.Stat(statePath); err == nil {
t.Error("prune state lingered after all grace periods resolved")
}
}
// TestLockDestination verifies a held destination lock blocks a second run
// and is released on close.
func TestLockDestination(t *testing.T) {
dest := t.TempDir()
lock, err := lockDestination(dest)
if err != nil {
t.Fatal(err)
}
if _, err := lockDestination(dest); err == nil {
t.Error("second lock unexpectedly succeeded")
}
lock.Close()
lock, err = lockDestination(dest)
if err != nil {
t.Error("lock not released on close:", err)
} else {
lock.Close()
}
}

98
server/classify.go Normal file
View file

@ -0,0 +1,98 @@
package server
import (
"path"
"strings"
"github.com/grmrgecko/repo-sync/mirror"
)
// kindGeneric marks request paths that are not repository entry points.
const kindGeneric = "generic"
// resource is a crawlable unit derived from a request path.
type resource struct {
// Kind is a RepoType string, or kindGeneric.
Kind string
// Key uniquely identifies the resource in state and the lock map.
Key string
// Path is the repository request path (the suite directory for deb).
Path string
// Root is the request path prefix covering every file the resource
// owns; deb uses the archive root so pool requests match.
Root string
// ReqPath preserves the original request path.
ReqPath string
}
// classifyRequest maps a request path onto a crawlable resource. Only
// repository entry-point metadata registers a repository; every other path
// is generic and served on demand.
func classifyRequest(reqPath string) resource {
clean := path.Clean("/" + strings.TrimPrefix(reqPath, "/"))
base := path.Base(clean)
dir := path.Dir(clean)
// RPM repositories are identified by any repodata request.
if idx := strings.Index(clean, "/repodata/"); idx >= 0 {
root := clean[:idx]
if root == "" {
root = "/"
}
return resource{
Kind: string(mirror.RepoRPM),
Key: string(mirror.RepoRPM) + ":" + root,
Path: root,
Root: root,
ReqPath: clean,
}
}
// Apt suites are identified by their release files; the archive root
// above dists also covers the pool.
if base == "InRelease" || base == "Release" || base == "Release.gpg" {
root := dir
if idx := strings.Index(clean, "/dists/"); idx >= 0 {
root = clean[:idx]
if root == "" {
root = "/"
}
}
return resource{
Kind: string(mirror.RepoDeb),
Key: string(mirror.RepoDeb) + ":" + dir,
Path: dir,
Root: root,
ReqPath: clean,
}
}
// Pacman repositories are identified by their database file.
if strings.HasSuffix(base, ".db") {
return resource{
Kind: string(mirror.RepoArch),
Key: string(mirror.RepoArch) + ":" + dir,
Path: dir,
Root: dir,
ReqPath: clean,
}
}
// Alpine repositories are identified by their index archive.
if base == mirror.APKIndexName {
return resource{
Kind: string(mirror.RepoApk),
Key: string(mirror.RepoApk) + ":" + dir,
Path: dir,
Root: dir,
ReqPath: clean,
}
}
return resource{
Kind: kindGeneric,
Key: kindGeneric + ":" + clean,
Path: clean,
ReqPath: clean,
}
}

68
server/classify_test.go Normal file
View file

@ -0,0 +1,68 @@
package server
import (
"testing"
"time"
"github.com/grmrgecko/repo-sync/state"
)
// TestClassifyRequest verifies request paths map to the right resource
// kinds, crawl paths, and keep-alive roots.
func TestClassifyRequest(t *testing.T) {
cases := []struct {
reqPath string
kind string
path string
root string
}{
{"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"},
{"/almalinux/9/BaseOS/x86_64/os/repodata/abc-primary.xml.gz", "rpm", "/almalinux/9/BaseOS/x86_64/os", "/almalinux/9/BaseOS/x86_64/os"},
{"/debian/dists/bookworm/InRelease", "deb", "/debian/dists/bookworm", "/debian"},
{"/debian/dists/stable/updates/Release", "deb", "/debian/dists/stable/updates", "/debian"},
{"/flat/Release", "deb", "/flat", "/flat"},
{"/archlinux/core/os/x86_64/core.db", "arch", "/archlinux/core/os/x86_64", "/archlinux/core/os/x86_64"},
{"/alpine/v3.24/main/x86_64/APKINDEX.tar.gz", "apk", "/alpine/v3.24/main/x86_64", "/alpine/v3.24/main/x86_64"},
{"/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", kindGeneric, "/almalinux/9/BaseOS/x86_64/os/Packages/foo.rpm", ""},
{"/notes/README.txt", kindGeneric, "/notes/README.txt", ""},
}
for _, c := range cases {
res := classifyRequest(c.reqPath)
if res.Kind != c.kind || res.Path != c.path || res.Root != c.root {
t.Errorf("classifyRequest(%q) = {%s %s %s}, want {%s %s %s}",
c.reqPath, res.Kind, res.Path, res.Root, c.kind, c.path, c.root)
}
}
}
// TestRefreshTier verifies the tier walk and staleness budget.
func TestRefreshTier(t *testing.T) {
schedule := []time.Duration{4 * time.Hour, 12 * time.Hour, 24 * time.Hour}
if interval, stale := refreshTier(time.Hour, schedule); stale || interval != 4*time.Hour {
t.Errorf("fresh resource tier = %v, %v", interval, stale)
}
if interval, stale := refreshTier(10*time.Hour, schedule); stale || interval != 12*time.Hour {
t.Errorf("aging resource tier = %v, %v", interval, stale)
}
if _, stale := refreshTier(41*time.Hour, schedule); !stale {
t.Error("resource past the budget not stale")
}
}
// TestNextCrawlAt verifies crawl gating for fresh, failed, and deferred
// entries.
func TestNextCrawlAt(t *testing.T) {
now := time.Now()
if !nextCrawlAt(state.Entry{}, time.Hour).IsZero() {
t.Error("never-crawled entry not due immediately")
}
failedAt := now.Add(30 * time.Minute)
failed := state.Entry{LastError: "boom", LastCrawled: now, NextCrawl: failedAt}
if got := nextCrawlAt(failed, time.Hour); !got.Equal(failedAt) {
t.Errorf("failed entry gate = %v, want %v", got, failedAt)
}
ok := state.Entry{LastCrawled: now, NextCrawl: now.Add(10 * time.Minute)}
if got := nextCrawlAt(ok, 2*time.Hour); !got.Equal(now.Add(2 * time.Hour)) {
t.Errorf("tier gate = %v, want %v", got, now.Add(2*time.Hour))
}
}

433
server/crawl_loop.go Normal file
View file

@ -0,0 +1,433 @@
package server
import (
"context"
"fmt"
"os"
"strings"
"sync"
"sync/atomic"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/mirror"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// lockMu guards the per-resource lock table and the in-flight crawl set.
var lockMu sync.Mutex
// resourceLock is one entry in the lock table; refs counts holders and
// waiters so the entry can be dropped once nobody references it.
type resourceLock struct {
mu sync.Mutex
refs int
}
// locks serializes crawls of the same resource across request handlers and
// the scheduled loop.
var locks = map[string]*resourceLock{}
// lockResource acquires the lock for a resource key and returns its unlock
// function. Entries are reference counted and removed when the last holder
// or waiter releases, so evicted resources do not accumulate in the table.
func lockResource(key string) func() {
lockMu.Lock()
l := locks[key]
if l == nil {
l = &resourceLock{}
locks[key] = l
}
l.refs++
lockMu.Unlock()
l.mu.Lock()
return func() {
l.mu.Unlock()
lockMu.Lock()
l.refs--
if l.refs == 0 {
delete(locks, key)
}
lockMu.Unlock()
}
}
// inventoryEvictionGrace shields resources still advertised by a mount's
// configuration from eviction after requests stop.
const inventoryEvictionGrace = 72 * time.Hour
// inflight records resources with a crawl queued or running so concurrent
// dispatches from the request handlers and the scheduler collapse into one
// crawl. Guarded by lockMu.
var inflight = map[string]bool{}
// crawlWG tracks dispatched crawl goroutines so shutdown can wait for them
// before the final state flush records their outcomes.
var crawlWG sync.WaitGroup
// crawlCtx is the context dispatched crawls run under. CrawlLoop installs
// its own so shutdown cancels crawls; the background default covers tests
// and requests arriving before the loop starts.
var crawlCtx atomic.Value
func init() {
crawlCtx.Store(context.Background())
}
// Crawl slots bound how many crawls run at once; slotCond re-reads the
// configured limit on every wake so a reload takes effect.
var (
slotMu sync.Mutex
slotCond = sync.NewCond(&slotMu)
slotsInUse int
)
// acquireCrawlSlot blocks until a crawl slot is free under the configured
// concurrency limit.
func acquireCrawlSlot() {
slotMu.Lock()
for slotsInUse >= cfg.C.Load().Crawler.ConcurrentCrawls {
slotCond.Wait()
}
slotsInUse++
slotMu.Unlock()
}
// releaseCrawlSlot frees a crawl slot and wakes the waiters.
func releaseCrawlSlot() {
slotMu.Lock()
slotsInUse--
slotMu.Unlock()
slotCond.Broadcast()
}
// startCrawl dispatches an asynchronous crawl of one resource, collapsing
// duplicate dispatches and bounding concurrency. Failures are logged; the
// retry gating lives in the schedulers.
func startCrawl(res resource) {
lockMu.Lock()
if inflight[res.Key] {
lockMu.Unlock()
return
}
inflight[res.Key] = true
lockMu.Unlock()
crawlWG.Add(1)
go func() {
defer crawlWG.Done()
defer func() {
lockMu.Lock()
delete(inflight, res.Key)
lockMu.Unlock()
}()
acquireCrawlSlot()
defer releaseCrawlSlot()
ctx := crawlCtx.Load().(context.Context)
if ctx.Err() != nil {
return
}
if err := crawlResource(ctx, res); err != nil {
log.WithError(err).WithField("key", res.Key).Error("Repository crawl failed.")
}
}()
}
// WaitCrawls blocks until every dispatched crawl has finished. It is called
// at shutdown after the crawl context is canceled and before the final
// state flush.
func WaitCrawls() {
crawlWG.Wait()
}
// crawlResource synchronizes one classified resource under its lock. Deb
// resources also hold an archive-root lock so suites sharing a pool never
// download the same file concurrently.
func crawlResource(ctx context.Context, res resource) error {
if res.Kind == string(mirror.RepoDeb) && res.Root != res.Path {
defer lockResource("deb-root:" + res.Root)()
}
defer lockResource(res.Key)()
err := dispatchCrawl(ctx, res)
state.S.MarkCrawled(res.Key, time.Now(), err)
return err
}
// pathBelow reports whether a request path sits strictly below a base path.
func pathBelow(p, base string) bool {
if p == base {
return false
}
if base == "/" {
return true
}
return strings.HasPrefix(p, base+"/")
}
// prunableTree reports whether a crawl may prune its destination tree. Every
// sync prunes the repository directory, which for deb is the suite directory
// and otherwise the repository's own path. A crawl's keep set covers only
// that one repository, so a tree that also holds another mount's content or
// another tracked repository must not be pruned: the neighbour's files are
// unreferenced there and would be deleted. This matters most for a
// repository sitting at the mirror root, whose tree is the whole cache.
func prunableTree(conf *cfg.Config, res resource) bool {
for _, mount := range conf.Mounts {
if pathBelow(mount.Path, res.Path) {
log.WithFields(log.Fields{"key": res.Key, "mount": mount.Path}).
Debug("Pruning disabled; the repository tree contains another mount.")
return false
}
}
for key, entry := range state.S.Snapshot() {
if key == res.Key || entry.Kind == kindGeneric {
continue
}
if pathBelow(entry.Path, res.Path) {
log.WithFields(log.Fields{"key": res.Key, "nested": key}).
Debug("Pruning disabled; the repository tree contains another repository.")
return false
}
}
return true
}
// dispatchCrawl runs the format-specific synchronization for a resource
// into the online tree.
func dispatchCrawl(ctx context.Context, res resource) error {
conf := cfg.C.Load()
online := conf.OnlineDomain()
mount, ok := conf.MountFor(res.Path)
if !ok {
return fmt.Errorf("no mount configured for %s", res.Path)
}
// upstreamFor maps a request path onto the mount's upstream URL.
upstreamFor := func(p string) string {
rel := p
if mount.Path != "/" {
rel = strings.TrimPrefix(p, mount.Path)
}
return mount.Upstream + rel
}
// Generic files never reach the crawl path: they refresh on demand at
// serve time and expire through the failure cache and state eviction.
typ := mirror.RepoType(res.Kind)
opts := &mirror.Options{
Type: typ,
Workers: conf.Crawler.Workers,
Prune: prunableTree(conf, res),
PruneGrace: conf.Crawler.PruneGrace,
}
// Deb destinations pin the archive root so pool files land beside
// their suites; every other type syncs into its own directory.
destPath := res.Path
if typ == mirror.RepoDeb {
destPath = res.Root
}
dest, err := fetch.LocalJoin(online.Root, destPath)
if err != nil {
return err
}
repoURL := upstreamFor(res.Path)
return mirror.SyncInto(ctx, typ, fetch.NewSource([]string{repoURL}), repoURL, dest, opts)
}
// CrawlLoop keeps configured and discovered resources fresh until the
// context is canceled, starting with an immediate pass. Its context also
// governs request-dispatched crawls so shutdown cancels them.
func CrawlLoop(ctx context.Context) {
crawlCtx.Store(ctx)
runScheduledCrawls(ctx)
t := time.NewTicker(cfg.C.Load().CrawlCheckInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
fetchFailures.Sweep(time.Now())
runScheduledCrawls(ctx)
// Re-arm from the active config so a reload takes effect.
t.Reset(cfg.C.Load().CrawlCheckInterval)
}
}
}
// runScheduledCrawls runs one pass of configured and request-discovered
// crawls.
func runScheduledCrawls(ctx context.Context) {
crawlConfigured(ctx)
crawlDue(ctx)
}
// configuredResource builds the resource for a mount's configured
// repository.
func configuredResource(repo cfg.MountRepoConfig) resource {
root := repo.Path
if repo.Type == string(mirror.RepoDeb) {
if idx := strings.Index(repo.Path, "/dists/"); idx >= 0 {
root = repo.Path[:idx]
if root == "" {
root = "/"
}
}
}
return resource{
Kind: repo.Type,
Key: repo.Type + ":" + repo.Path,
Path: repo.Path,
Root: root,
ReqPath: repo.Path,
}
}
// crawlConfigured keeps every mount-configured repository synchronized
// regardless of client traffic. Crawls are dispatched asynchronously so a
// slow repository cannot stall the scheduler loop.
func crawlConfigured(ctx context.Context) {
now := time.Now()
for _, mount := range cfg.C.Load().Mounts {
for _, repo := range mount.Repos {
if ctx.Err() != nil {
return
}
res := configuredResource(repo)
state.S.MarkSeenInInventory(res.Kind, res.Key, res.Path, res.Root, now)
entry, _ := state.S.Entry(res.Key)
if !dueForInventory(entry, now) {
continue
}
startCrawl(res)
}
}
}
// dueForInventory gates configured crawls on the standard interval with
// faster retries after failures.
func dueForInventory(entry state.Entry, now time.Time) bool {
if entry.LastCrawled.IsZero() {
return true
}
if !entry.NextCrawl.IsZero() {
return !now.Before(entry.NextCrawl)
}
return now.Sub(entry.LastCrawled) >= cfg.C.Load().RepoCrawlInterval
}
// refreshTier walks the cumulative schedule, returning the refresh
// interval matching how long ago the resource was requested, or stale once
// the whole budget has elapsed.
func refreshTier(sinceRequested time.Duration, schedule []time.Duration) (time.Duration, bool) {
var cumulative time.Duration
for _, step := range schedule {
if step <= 0 {
continue
}
cumulative += step
if sinceRequested < cumulative {
return step, false
}
}
return 0, true
}
// refreshSchedule is the tier list: the active crawl interval followed by
// the configured aging backoff steps.
func refreshSchedule() []time.Duration {
conf := cfg.C.Load()
schedule := make([]time.Duration, 0, 1+len(conf.Crawler.RefreshSchedule))
schedule = append(schedule, conf.RepoCrawlInterval)
return append(schedule, conf.Crawler.RefreshSchedule...)
}
// activeInInventory reports whether a mount still advertised the resource
// recently enough to shield it from eviction.
func activeInInventory(entry state.Entry, now time.Time) bool {
if entry.LastSeenInInventory.IsZero() {
return false
}
return now.Sub(entry.LastSeenInInventory) < inventoryEvictionGrace
}
// nextCrawlAt returns when a resource should next crawl: immediately when
// never crawled, at the failure retry gate after errors, else at the tier
// interval.
func nextCrawlAt(entry state.Entry, tierInterval time.Duration) time.Time {
if entry.LastCrawled.IsZero() {
return time.Time{}
}
if entry.LastError != "" && !entry.NextCrawl.IsZero() {
return entry.NextCrawl
}
tierGate := entry.LastCrawled.Add(tierInterval)
if !entry.NextCrawl.IsZero() && entry.NextCrawl.After(tierGate) {
return entry.NextCrawl
}
return tierGate
}
// crawlDue refreshes request-discovered resources on the tier schedule and
// evicts those whose traffic stopped past the whole budget.
func crawlDue(ctx context.Context) {
now := time.Now()
schedule := refreshSchedule()
for key, entry := range state.S.Snapshot() {
if ctx.Err() != nil {
return
}
// Inventory-only entries are managed by crawlConfigured.
if entry.LastRequested.IsZero() {
continue
}
interval, stale := refreshTier(now.Sub(entry.LastRequested), schedule)
if stale {
if activeInInventory(entry, now) {
continue
}
evictStale(key, entry)
continue
}
// Generic files refresh on demand at serve time.
if entry.Kind == kindGeneric {
continue
}
next := nextCrawlAt(entry, interval)
if !next.IsZero() && now.Before(next) {
continue
}
startCrawl(resource{Kind: entry.Kind, Key: key, Path: entry.Path, Root: entry.Root, ReqPath: entry.Path})
}
}
// evictStale removes a resource whose requests stopped long enough ago,
// re-checking under the lock so a racing request wins.
func evictStale(key string, entry state.Entry) {
// Deb resources share an archive root across sibling suites; hold that
// lock too, matching crawlResource, so eviction never races an
// in-flight sibling-suite crawl touching the shared pool.
if entry.Kind == string(mirror.RepoDeb) && entry.Root != entry.Path {
defer lockResource("deb-root:" + entry.Root)()
}
defer lockResource(key)()
current, ok := state.S.Entry(key)
if ok && current.LastRequested.After(entry.LastRequested) {
return
}
target, err := fetch.LocalJoin(cfg.C.Load().OnlineDomain().Root, entry.Path)
if err == nil && entry.Path != "/" && entry.Path != "" {
if err := os.RemoveAll(target); err != nil {
log.WithError(err).WithField("path", target).Warn("Failed to evict stale resource.")
return
}
log.WithFields(log.Fields{"key": key, "path": target}).Info("Evicted stale resource.")
}
state.S.Delete(key)
}

56
server/failure_cache.go Normal file
View file

@ -0,0 +1,56 @@
package server
import (
"sync"
"time"
)
// failureEntry is one remembered upstream failure with its expiry.
type failureEntry struct {
status int
expires time.Time
}
// failureCache remembers recent upstream failures for generic paths so a
// missing or erroring file does not trigger an upstream request per client.
// It is memory-only; nothing here reaches the state file.
type failureCache struct {
mu sync.Mutex
entries map[string]failureEntry
}
// fetchFailures is the server's shared failure cache.
var fetchFailures = &failureCache{entries: map[string]failureEntry{}}
// Hit returns the remembered status for a key, pruning expired entries.
func (c *failureCache) Hit(key string, now time.Time) (int, bool) {
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.entries[key]
if !ok {
return 0, false
}
if !now.Before(e.expires) {
delete(c.entries, key)
return 0, false
}
return e.status, true
}
// Add remembers a failure until now+ttl.
func (c *failureCache) Add(key string, status int, now time.Time, ttl time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = failureEntry{status: status, expires: now.Add(ttl)}
}
// Sweep drops expired entries that nobody re-requested.
func (c *failureCache) Sweep(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for key, e := range c.entries {
if !now.Before(e.expires) {
delete(c.entries, key)
}
}
}

14
server/main_test.go Normal file
View file

@ -0,0 +1,14 @@
package server
import (
"os"
"testing"
"github.com/grmrgecko/repo-sync/fetch"
)
// TestMain initializes the shared HTTP client all fetch paths depend on.
func TestMain(m *testing.M) {
fetch.Reload()
os.Exit(m.Run())
}

318
server/serve.go Normal file
View file

@ -0,0 +1,318 @@
// Package server runs the caching mirror: it answers requests from the
// local tree, fetches misses through the configured upstream mounts, and
// crawls the repositories it discovers in the background so they stay
// current without holding a request open.
package server
import (
"context"
"errors"
"fmt"
"html"
"io"
"net/http"
"os"
"path"
"sort"
"strings"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// Handler returns the mirror server's HTTP handler.
func Handler() http.Handler {
return http.HandlerFunc(serveMirror)
}
// regularFile stats a path, reporting only regular files as present.
func regularFile(name string) (os.FileInfo, bool) {
info, err := os.Stat(name)
if err != nil || !info.Mode().IsRegular() {
return nil, false
}
return info, true
}
// serveLocal resolves a request path under a domain root, mapping
// directory requests to a cached index.html.
func serveLocal(root, reqPath string, isDir bool) (string, error) {
p := reqPath
if isDir || p == "/" {
p = strings.TrimSuffix(p, "/") + "/index.html"
}
return fetch.LocalJoin(root, p)
}
// internalFile reports paths that belong to the mirror's own bookkeeping
// and must never be served.
func internalFile(reqPath string) bool {
base := path.Base(reqPath)
return base == fetch.LockFileName || base == fetch.PruneStateName ||
strings.HasSuffix(base, fetch.StagedSuffix) || strings.HasSuffix(base, fetch.PartialSuffix)
}
// fetchUpstream retrieves one file through the mount covering the request
// path and stores it under root.
func fetchUpstream(ctx context.Context, root, reqPath string, isDir bool) error {
mount, ok := cfg.C.Load().MountFor(reqPath)
if !ok {
return fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound)
}
rel := reqPath
if mount.Path != "/" {
rel = strings.TrimPrefix(reqPath, mount.Path)
}
if isDir {
rel = strings.TrimSuffix(rel, "/") + "/"
}
dst, err := serveLocal(root, reqPath, isDir)
if err != nil {
return err
}
// Concurrent on-demand fetches and background crawls can target the
// same destination; fetch.File serializes them by path.
src := fetch.NewSource([]string{mount.Upstream})
_, err = fetch.File(ctx, src, rel, dst, nil, false, false)
return err
}
// writeFetchError reports an upstream failure to the client.
func writeFetchError(w http.ResponseWriter, err error) {
if errors.Is(err, fetch.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeCachedFailure reports a remembered upstream failure.
func writeCachedFailure(w http.ResponseWriter, status int) {
if status == http.StatusNotFound {
http.Error(w, "not found", http.StatusNotFound)
return
}
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
}
// writeIndexesDisabled answers a directory request when directory indexes
// are turned off, confirming the mirror is alive without listing content.
func writeIndexesDisabled(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, "<!DOCTYPE html>\n<html>\n<head><title>repo-sync mirror</title></head>\n<body>\n"+
"<h1>repo-sync mirror</h1>\n<p>This mirror is running. Directory indexes are disabled.</p>\n"+
"</body>\n</html>\n")
}
// writeMountIndex answers the root directory request with a generated index
// of every mount and configured repository, giving clients a navigable
// entry point when no catch-all mount covers the root.
func writeMountIndex(w http.ResponseWriter, conf *cfg.Config) {
seen := map[string]bool{}
var paths []string
add := func(p string) {
if p == "/" || seen[p] {
return
}
seen[p] = true
paths = append(paths, p)
}
for _, mount := range conf.Mounts {
add(mount.Path)
for _, repo := range mount.Repos {
add(repo.Path)
}
}
sort.Strings(paths)
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html>\n<head><title>Index of /</title></head>\n<body>\n<h1>Index of /</h1>\n<ul>\n")
for _, p := range paths {
href := html.EscapeString(p + "/")
fmt.Fprintf(&b, "<li><a href=\"%s\">%s</a></li>\n", href, href)
}
b.WriteString("</ul>\n</body>\n</html>\n")
w.Header().Set("Content-Type", "text/html; charset=utf-8")
io.WriteString(w, b.String())
}
// serveMirror handles every mirror request: domain lookup, classification,
// and dispatch to the role-specific handler.
func serveMirror(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
conf := cfg.C.Load()
domain, ok := conf.Domain(r.Host)
if !ok {
http.Error(w, "unknown mirror domain", http.StatusNotFound)
return
}
reqPath := path.Clean("/" + strings.TrimPrefix(r.URL.Path, "/"))
isDir := strings.HasSuffix(r.URL.Path, "/") || reqPath == "/"
if internalFile(reqPath) {
http.Error(w, "not found", http.StatusNotFound)
return
}
// Directory requests are answered before role dispatch: a notice page
// when indexes are disabled, and a generated index of the mounts for a
// root that no catch-all mount covers.
if isDir {
if !conf.HTTP.DirectoryIndexes {
writeIndexesDisabled(w)
return
}
if reqPath == "/" {
if _, ok := conf.MountFor("/"); !ok {
writeMountIndex(w, conf)
return
}
}
}
res := classifyRequest(reqPath)
if domain.Role == cfg.RoleOnline {
handleOnline(w, r, domain, reqPath, isDir, res)
return
}
handleOffline(w, r, domain, reqPath, isDir, res)
}
// handleOnline serves from the writable cache, fetching and crawling on
// demand.
func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
now := time.Now()
// Directory listings are cached pass-through pages with no tracking.
if isDir {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, true); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
if res.Kind == kindGeneric {
// Files under a registered repository keep it alive and are
// fetched on demand until its crawl completes.
if state.S.TouchRepoMembers(reqPath, now) {
if _, exists := regularFile(local); !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
writeFetchError(w, err)
return
}
}
http.ServeFile(w, r, local)
return
}
handleGeneric(w, r, domain, reqPath, res, local, now)
return
}
// A repository entry point registers the repository and dispatches a
// background crawl on first sight so the whole repository fills in
// behind this response without holding it open. The requested file
// itself is fetched on demand so the client is answered immediately.
// Repositories already crawled only re-dispatch here to retry a failed
// crawl; missing optional files must not trigger a full crawl per
// request.
entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now)
_, exists := regularFile(local)
needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "")
if !exists {
if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil {
// A path whose first contact failed was never shown to be a
// repository, so the registration this request created is
// dropped and the scheduler never crawls it. Registrations an
// earlier request established are kept: every entry point of one
// repository shares a key, so a repository serving Release but
// not InRelease would otherwise deregister itself on the miss.
if !tracked {
state.S.Delete(res.Key)
}
writeFetchError(w, err)
return
}
}
if needCrawl {
startCrawl(res)
}
http.ServeFile(w, r, local)
}
// handleGeneric serves a plain cached file, refreshing stale copies and
// remembering upstream failures.
func handleGeneric(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, res resource, local string, now time.Time) {
conf := cfg.C.Load()
if status, hit := fetchFailures.Hit(res.Key, now); hit {
if _, exists := regularFile(local); !exists {
writeCachedFailure(w, status)
return
}
} else {
// GenericMaxAge bounds how often the upstream is revalidated,
// measured from the last upstream check recorded in state. The
// local file's modification time carries the upstream's
// Last-Modified, so it does not record when the mirror last checked.
entry, tracked := state.S.Entry(res.Key)
_, exists := regularFile(local)
if !exists || !tracked || now.Sub(entry.LastCrawled) > conf.GenericMaxAge {
err := fetchUpstream(r.Context(), domain.Root, reqPath, false)
switch {
case errors.Is(err, fetch.ErrNotFound):
fetchFailures.Add(res.Key, http.StatusNotFound, now, conf.NegativeCacheTTL)
state.S.Delete(res.Key)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Warn("Upstream lost a stale generic file; serving the existing copy.")
case err != nil:
fetchFailures.Add(res.Key, http.StatusBadGateway, now, conf.TransientErrorTTL)
if !exists {
writeFetchError(w, err)
return
}
log.WithError(err).WithField("path", reqPath).Error("Unable to refresh a stale generic file; serving the existing copy.")
default:
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
// Record the upstream check so GenericMaxAge gates the next
// request. The scheduler skips generic entries, so the retry
// time this also sets is unused.
state.S.MarkCrawled(res.Key, now, nil)
}
} else {
state.S.MarkRequested(kindGeneric, res.Key, res.Path, "", now)
}
}
http.ServeFile(w, r, local)
}
// handleOffline serves the read-only published tree, reading through the
// online cache for anything it does not hold.
func handleOffline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfig, reqPath string, isDir bool, res resource) {
local, err := serveLocal(domain.Root, reqPath, isDir)
if err != nil {
http.Error(w, "bad request path", http.StatusBadRequest)
return
}
if _, exists := regularFile(local); exists {
http.ServeFile(w, r, local)
return
}
handleOnline(w, r, cfg.C.Load().OnlineDomain(), reqPath, isDir, res)
}

487
server/serve_test.go Normal file
View file

@ -0,0 +1,487 @@
package server
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/internal/testrepos"
"github.com/grmrgecko/repo-sync/mirror"
"github.com/grmrgecko/repo-sync/state"
)
// serverFixture builds an upstream with one repository of each type plus a
// generic file, configures the mirror server against it, and returns the
// mirror's test server and the online root.
func serverFixture(t *testing.T) (*httptest.Server, string, string) {
t.Helper()
// Upstream content behind a single catch-all mount.
www := t.TempDir()
testrepos.BuildRPMRepo(t, filepath.Join(www, "almalinux", "9", "BaseOS", "x86_64", "os"))
testrepos.BuildDebRepo(t, filepath.Join(www, "debian"))
testrepos.BuildArchRepo(t, filepath.Join(www, "archlinux", "core", "os", "x86_64"), "core")
testrepos.BuildApkRepo(t, filepath.Join(www, "alpine", "v3.24", "main", "x86_64"))
testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file"))
upstream := testrepos.ServeDir(t, www)
onlineRoot := t.TempDir()
offlineRoot := t.TempDir()
confDir := t.TempDir()
conf := fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
- domain: offline.test
role: offline
root: %s
mounts:
- path: /
upstream: %s
`, confDir, onlineRoot, offlineRoot, upstream.URL)
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(conf))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
// Crawls dispatched by requests run in the background; drain them so
// they do not race the next test's state reload.
t.Cleanup(WaitCrawls)
return srv, onlineRoot, offlineRoot
}
// waitFor polls a condition until it holds or the deadline passes; crawls
// dispatched by requests run asynchronously.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
// get fetches a mirror URL with an optional Host override.
func get(t *testing.T, srv *httptest.Server, host, path string) (*http.Response, []byte) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil)
if err != nil {
t.Fatal(err)
}
if host != "" {
req.Host = host
}
resp, err := srv.Client().Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Fatal(err)
}
return resp, body
}
// TestServeRPMDiscovery verifies a repomd.xml request registers and crawls
// the repository, filling the local tree behind the response.
func TestServeRPMDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
repo := "/almalinux/9/BaseOS/x86_64/os"
resp, body := get(t, srv, "", repo+"/repodata/repomd.xml")
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
if len(body) == 0 {
t.Fatal("empty repomd.xml response")
}
// The repository was registered and the background crawl mirrors its
// packages behind the response.
if _, ok := state.S.Entry("rpm:" + repo); !ok {
t.Error("repository not registered in state")
}
var pkgs []string
waitFor(t, "rpm crawl", func() bool {
pkgs, _ = filepath.Glob(filepath.Join(onlineRoot, filepath.FromSlash(repo[1:]), "Packages", "*.rpm"))
return len(pkgs) > 0
})
// Package requests are served from the crawled tree.
resp, _ = get(t, srv, "", repo+"/Packages/"+filepath.Base(pkgs[0]))
if resp.StatusCode != http.StatusOK {
t.Errorf("package request status = %d", resp.StatusCode)
}
}
// TestServeDebDiscovery verifies a Release request crawls the suite with
// pool files under the archive root.
func TestServeDebDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, _ := get(t, srv, "", "/debian/dists/test/Release")
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
if _, ok := state.S.Entry("deb:/debian/dists/test"); !ok {
t.Error("suite not registered in state")
}
waitFor(t, "deb crawl", func() bool {
_, err := os.Stat(filepath.Join(onlineRoot, "debian", "pool", "main", "h", "hello", "hello_1.0_amd64.deb"))
return err == nil
})
// A pool request under the archive root is a repository member.
resp, _ = get(t, srv, "", "/debian/pool/main/h/hello/hello_1.0_amd64.deb")
if resp.StatusCode != http.StatusOK {
t.Errorf("pool request status = %d", resp.StatusCode)
}
}
// TestServeArchAndApkDiscovery verifies database and index requests crawl
// their repositories.
func TestServeArchAndApkDiscovery(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, _ := get(t, srv, "", "/archlinux/core/os/x86_64/core.db")
if resp.StatusCode != http.StatusOK {
t.Fatalf("arch status = %d", resp.StatusCode)
}
waitFor(t, "arch crawl", func() bool {
pkgs, _ := filepath.Glob(filepath.Join(onlineRoot, "archlinux", "core", "os", "x86_64", "*.pkg.tar.zst"))
return len(pkgs) > 0
})
resp, _ = get(t, srv, "", "/alpine/v3.24/main/x86_64/APKINDEX.tar.gz")
if resp.StatusCode != http.StatusOK {
t.Fatalf("apk status = %d", resp.StatusCode)
}
waitFor(t, "apk crawl", func() bool {
apks, _ := filepath.Glob(filepath.Join(onlineRoot, "alpine", "v3.24", "main", "x86_64", "*.apk"))
return len(apks) > 0
})
}
// TestServeGeneric verifies plain files proxy through with caching and
// missing files are answered from the negative cache.
func TestServeGeneric(t *testing.T) {
srv, onlineRoot, _ := serverFixture(t)
resp, body := get(t, srv, "", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("generic = %d %q", resp.StatusCode, body)
}
if _, err := os.Stat(filepath.Join(onlineRoot, "notes", "README.txt")); err != nil {
t.Error("generic file not cached:", err)
}
resp, _ = get(t, srv, "", "/notes/missing.txt")
if resp.StatusCode != http.StatusNotFound {
t.Errorf("missing file status = %d", resp.StatusCode)
}
if status, hit := fetchFailures.Hit("generic:/notes/missing.txt", time.Now()); !hit || status != http.StatusNotFound {
t.Error("missing file not negative-cached")
}
}
// TestServeGenericRevalidation verifies GenericMaxAge gates how often a
// cached plain file is revalidated. The upstream file carries an old
// Last-Modified that the cached copy inherits, so the gate is measured from
// the recorded upstream check rather than the local modification time.
func TestServeGenericRevalidation(t *testing.T) {
www := t.TempDir()
name := filepath.Join(www, "notes", "README.txt")
testrepos.WriteFile(t, name, []byte("generic file"))
old := time.Now().Add(-30 * 24 * time.Hour)
if err := os.Chtimes(name, old, old); err != nil {
t.Fatal(err)
}
var upstreamHits atomic.Int64
files := http.FileServer(http.Dir(www))
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamHits.Add(1)
files.ServeHTTP(w, r)
}))
t.Cleanup(upstream.Close)
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(`
state_path: %s/state.yaml
generic_max_age: 6h
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /
upstream: %s
`, confDir, t.TempDir(), upstream.URL)))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
for i := 0; i < 3; i++ {
resp, body := get(t, srv, "", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("request %d = %d %q", i, resp.StatusCode, body)
}
}
if got := upstreamHits.Load(); got != 1 {
t.Errorf("upstream requests = %d, want 1 within generic_max_age", got)
}
}
// TestServeOffline verifies offline domains serve their own tree first and
// read through the online cache otherwise.
func TestServeOffline(t *testing.T) {
srv, _, offlineRoot := serverFixture(t)
// Not present offline: delegated to online, which proxies upstream.
resp, body := get(t, srv, "offline.test", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "generic file" {
t.Fatalf("offline read-through = %d %q", resp.StatusCode, body)
}
// Present offline: served from the offline tree.
testrepos.WriteFile(t, filepath.Join(offlineRoot, "notes", "README.txt"), []byte("published copy"))
resp, body = get(t, srv, "offline.test", "/notes/README.txt")
if resp.StatusCode != http.StatusOK || string(body) != "published copy" {
t.Errorf("offline copy = %d %q", resp.StatusCode, body)
}
}
// TestServeDirectoryIndexes verifies the generated root index without a
// catch-all mount and the notice page when indexes are disabled.
func TestServeDirectoryIndexes(t *testing.T) {
www := t.TempDir()
testrepos.WriteFile(t, filepath.Join(www, "notes", "README.txt"), []byte("generic file"))
upstream := testrepos.ServeDir(t, www)
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
conf := fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /notes
upstream: %s/notes
- path: /almalinux
upstream: %s/almalinux
repos:
- path: /almalinux/9/BaseOS/x86_64/os
type: rpm
`, confDir, t.TempDir(), upstream.URL, upstream.URL)
testrepos.WriteFile(t, confPath, []byte(conf))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
// Without a catch-all mount the root serves a generated index linking
// every mount and configured repository.
resp, body := get(t, srv, "", "/")
if resp.StatusCode != http.StatusOK {
t.Fatalf("index status = %d", resp.StatusCode)
}
for _, want := range []string{`"/notes/"`, `"/almalinux/"`, `"/almalinux/9/BaseOS/x86_64/os/"`} {
if !strings.Contains(string(body), want) {
t.Errorf("index missing %s in %q", want, body)
}
}
// Disabling indexes answers every directory request with the notice.
disabled := strings.Replace(conf, "mounts:", "http:\n directory_indexes: false\nmounts:", 1)
testrepos.WriteFile(t, confPath, []byte(disabled))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
resp, body = get(t, srv, "", "/")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") {
t.Errorf("disabled root = %d %q", resp.StatusCode, body)
}
resp, body = get(t, srv, "", "/notes/")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Directory indexes are disabled") {
t.Errorf("disabled subdirectory = %d %q", resp.StatusCode, body)
}
}
// TestServeRejections verifies unknown domains and internal bookkeeping
// files are refused.
func TestServeRejections(t *testing.T) {
srv, _, _ := serverFixture(t)
resp, _ := get(t, srv, "unknown.test", "/notes/README.txt")
if resp.StatusCode != http.StatusNotFound {
t.Errorf("unknown domain status = %d", resp.StatusCode)
}
resp, _ = get(t, srv, "", path.Join("/almalinux", fetch.LockFileName))
if resp.StatusCode != http.StatusNotFound {
t.Errorf("internal file status = %d", resp.StatusCode)
}
}
// TestPrunableTree verifies a crawl only prunes a tree it wholly owns, so a
// repository at the mirror root cannot delete another mount's cached files.
func TestPrunableTree(t *testing.T) {
serverFixture(t)
conf := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}, {Path: "/other"}}}
if prunableTree(conf, resource{Key: "rpm:/", Path: "/"}) {
t.Error("a root repository may not prune a tree holding another mount")
}
if !prunableTree(conf, resource{Key: "rpm:/other/el9", Path: "/other/el9"}) {
t.Error("a repository owning its own tree must prune")
}
// A repository nested under another one blocks the outer one's prune.
single := &cfg.Config{Mounts: []cfg.MountConfig{{Path: "/"}}}
state.S.MarkRequested("rpm", "rpm:/vendor/el9", "/vendor/el9", "/vendor/el9", time.Now())
if prunableTree(single, resource{Key: "rpm:/vendor", Path: "/vendor"}) {
t.Error("an outer repository may not prune a tree holding a nested one")
}
if !prunableTree(single, resource{Key: "rpm:/vendor/el9", Path: "/vendor/el9"}) {
t.Error("a repository must not be blocked by its own registration")
}
}
// TestServeRootRepositoryKeepsOtherMounts verifies a repository discovered at
// the mirror root does not prune content another mount cached beside it.
func TestServeRootRepositoryKeepsOtherMounts(t *testing.T) {
// Two upstreams: a repository at its own root, and unrelated content.
repoWWW := t.TempDir()
testrepos.BuildRPMRepo(t, repoWWW)
repoUpstream := testrepos.ServeDir(t, repoWWW)
otherWWW := t.TempDir()
testrepos.WriteFile(t, filepath.Join(otherWWW, "notes.txt"), []byte("other mount content"))
otherUpstream := testrepos.ServeDir(t, otherWWW)
onlineRoot := t.TempDir()
confDir := t.TempDir()
confPath := filepath.Join(confDir, "config.yaml")
testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(`
state_path: %s/state.yaml
domains:
- domain: 127.0.0.1
role: online
root: %s
mounts:
- path: /
upstream: %s
- path: /other
upstream: %s
`, confDir, onlineRoot, repoUpstream.URL, otherUpstream.URL)))
if err := cfg.Init(confPath); err != nil {
t.Fatal(err)
}
if err := state.Load(); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(Handler())
t.Cleanup(srv.Close)
t.Cleanup(WaitCrawls)
// Cache a file from the unrelated mount, then crawl the root repository.
if resp, _ := get(t, srv, "", "/other/notes.txt"); resp.StatusCode != http.StatusOK {
t.Fatalf("notes status = %d", resp.StatusCode)
}
cached := filepath.Join(onlineRoot, "other", "notes.txt")
if _, err := os.Stat(cached); err != nil {
t.Fatal("unrelated file not cached:", err)
}
if resp, _ := get(t, srv, "", "/repodata/repomd.xml"); resp.StatusCode != http.StatusOK {
t.Fatalf("repomd status = %d", resp.StatusCode)
}
WaitCrawls()
if _, err := os.Stat(cached); err != nil {
t.Error("root repository crawl pruned another mount's cached file:", err)
}
}
// TestServeSiblingEntryPointMiss verifies a repository keeps its
// registration when a sibling entry-point path the upstream does not serve
// is requested. Every entry point of one repository shares a state key, and
// clients probe all of them: apt asks for InRelease, Release, and
// Release.gpg, and dnf asks for signature material beside repomd.xml.
func TestServeSiblingEntryPointMiss(t *testing.T) {
srv, _, _ := serverFixture(t)
for _, tc := range []struct{ found, missing, key string }{
{"/debian/dists/test/Release", "/debian/dists/test/InRelease", "deb:/debian/dists/test"},
{
"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml",
"/almalinux/9/BaseOS/x86_64/os/repodata/repomd.xml.key",
"rpm:/almalinux/9/BaseOS/x86_64/os",
},
} {
if resp, _ := get(t, srv, "", tc.found); resp.StatusCode != http.StatusOK {
t.Fatalf("%s status = %d", tc.found, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); !ok {
t.Fatalf("%s did not register %s", tc.found, tc.key)
}
if resp, _ := get(t, srv, "", tc.missing); resp.StatusCode != http.StatusNotFound {
t.Fatalf("%s status = %d, want 404", tc.missing, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); !ok {
t.Errorf("%s deregistered %s", tc.missing, tc.key)
}
}
}
// TestServeUnresolvedEntryPoint verifies a path that only looks like a
// repository entry point is not left registered for the scheduler to crawl.
func TestServeUnresolvedEntryPoint(t *testing.T) {
srv, _, _ := serverFixture(t)
for _, tc := range []struct{ path, key string }{
{"/nope/repodata/repomd.xml", "rpm:/nope"},
{"/nope/dists/sid/InRelease", "deb:/nope/dists/sid"},
{"/nope/database.db", "arch:/nope"},
{"/nope/" + mirror.APKIndexName, "apk:/nope"},
} {
resp, _ := get(t, srv, "", tc.path)
if resp.StatusCode != http.StatusNotFound {
t.Errorf("%s status = %d", tc.path, resp.StatusCode)
}
if _, ok := state.S.Entry(tc.key); ok {
t.Errorf("%s left %s registered", tc.path, tc.key)
}
}
}

164
server_cmd.go Normal file
View file

@ -0,0 +1,164 @@
package main
import (
"context"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"github.com/grmrgecko/repo-sync/fetch"
"github.com/grmrgecko/repo-sync/server"
"github.com/grmrgecko/repo-sync/state"
log "github.com/sirupsen/logrus"
)
// shutdownTimeout bounds the graceful HTTP shutdown at exit.
const shutdownTimeout = 30 * time.Second
// ServerCmd runs the caching mirror server.
type ServerCmd struct {
ConfigPath string `help:"The path to the config file." optional:"" type:"existingfile"`
Bind string `name:"http.bind" help:"Override the configured bind address." optional:""`
Port uint `name:"http.port" help:"Override the configured port." optional:""`
}
// updateCFG applies command line overrides onto a freshly loaded
// configuration, before it is installed and visible to serving goroutines.
func (c *ServerCmd) updateCFG(conf *cfg.Config) {
if c.Bind != "" {
conf.HTTP.BindAddr = c.Bind
}
if c.Port != 0 {
conf.HTTP.Port = c.Port
}
}
// newHTTPServer builds the listener configuration from the active config.
func newHTTPServer() *http.Server {
conf := cfg.C.Load()
return &http.Server{
Addr: conf.HTTP.ListenAddr(),
Handler: server.Handler(),
ReadHeaderTimeout: conf.HTTP.ReadHeaderTimeout,
ReadTimeout: conf.HTTP.ReadTimeout,
WriteTimeout: conf.HTTP.WriteTimeout,
IdleTimeout: conf.HTTP.IdleTimeout,
}
}
// startHTTP binds the listener and serves in the background, returning the
// server for a later shutdown.
func startHTTP() (*http.Server, error) {
server := newHTTPServer()
ln, err := net.Listen("tcp", server.Addr)
if err != nil {
return nil, err
}
log.WithField("addr", server.Addr).Info("Mirror server listening.")
go func() {
if err := server.Serve(ln); err != nil && err != http.ErrServerClosed {
log.WithError(err).Error("HTTP server failed.")
}
}()
return server, nil
}
// stopHTTP gracefully shuts a server down within the shutdown timeout.
func stopHTTP(server *http.Server) {
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
_ = server.Shutdown(ctx)
}
// Run starts the mirror server: configuration, state, crawl loops, and the
// HTTP listener, then handles signals until shutdown.
func (c *ServerCmd) Run() error {
// Load configuration and apply command line overrides before install.
conf, err := cfg.Load(c.ConfigPath)
if err != nil {
return err
}
c.updateCFG(conf)
cfg.C.Store(conf)
cfg.SetupLogging(conf.Log)
if conf.Crawler.UserAgent != "" {
fetch.SetUserAgent(conf.Crawler.UserAgent)
}
fetch.SetRequestTimeout(conf.Crawler.RequestTimeout)
// Load persisted crawl state and build the shared HTTP client.
if err := state.Load(); err != nil {
return err
}
fetch.Reload()
// Start the background loops and the listener.
loopCtx, loopCancel := context.WithCancel(context.Background())
defer loopCancel()
go state.S.FlushLoop(loopCtx)
crawlDone := make(chan struct{})
go func() {
defer close(crawlDone)
server.CrawlLoop(loopCtx)
}()
// stopCrawls cancels the loops and waits for in-flight crawls so their
// outcomes are recorded before the final state flush. The crawl loop
// must return first: a scheduler pass in flight can still dispatch
// crawls, and a WaitGroup Add racing Wait is a misuse panic.
stopCrawls := func() {
loopCancel()
<-crawlDone
server.WaitCrawls()
}
httpServer, err := startHTTP()
if err != nil {
return err
}
// Handle signals: reload on SIGHUP, shut down on SIGINT/SIGTERM.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
for sig := range signals {
if sig != syscall.SIGHUP {
break
}
log.Info("Reloading configuration.")
oldAddr := cfg.C.Load().HTTP.ListenAddr()
conf, err := cfg.Load(c.ConfigPath)
if err != nil {
log.WithError(err).Error("Unable to reload configuration; keeping previous values.")
continue
}
c.updateCFG(conf)
cfg.C.Store(conf)
cfg.SetupLogging(conf.Log)
if conf.Crawler.UserAgent != "" {
fetch.SetUserAgent(conf.Crawler.UserAgent)
}
fetch.SetRequestTimeout(conf.Crawler.RequestTimeout)
fetch.Reload()
if conf.HTTP.ListenAddr() != oldAddr {
stopHTTP(httpServer)
if httpServer, err = startHTTP(); err != nil {
// Exiting on a failed rebind must still flush crawl state.
stopCrawls()
if saveErr := state.S.Save(); saveErr != nil {
log.WithError(saveErr).Error("Failed to save state while shutting down.")
}
return err
}
}
}
// Shut down: stop the listener, stop the loops, then flush state.
log.Info("Shutting down.")
stopHTTP(httpServer)
stopCrawls()
return state.S.Save()
}

236
state/state.go Normal file
View file

@ -0,0 +1,236 @@
// Package state persists the mirror server's per-resource crawl tracking
// so discovered repositories survive restarts.
package state
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
"gopkg.in/yaml.v3"
)
// Entry tracks one resource: a discovered or configured repository, or a
// generic cached file.
type Entry struct {
// Kind is the repository format, or "generic" for plain files.
Kind string `yaml:"kind"`
// Path is the request path of the resource (a deb entry uses the
// suite directory).
Path string `yaml:"path"`
// Root is the request path prefix covering every file the resource
// owns; requests below it keep the resource alive.
Root string `yaml:"root,omitempty"`
LastRequested time.Time `yaml:"last_requested,omitempty"`
LastCrawled time.Time `yaml:"last_crawled,omitempty"`
NextCrawl time.Time `yaml:"next_crawl,omitempty"`
LastSeenInInventory time.Time `yaml:"last_seen_in_inventory,omitempty"`
LastError string `yaml:"last_error,omitempty"`
}
// file is the on-disk YAML layout.
type file struct {
Entries map[string]*Entry `yaml:"entries"`
}
// Store is the in-memory state with dirty tracking for periodic flushes.
type Store struct {
mu sync.Mutex
file file
dirty bool
}
// S is the active store, installed by Load.
var S *Store
// Load reads the state file, treating a missing or empty file as a fresh
// store. Other read failures abort startup so a state file the process
// cannot read is never silently replaced.
func Load() error {
s := &Store{file: file{Entries: map[string]*Entry{}}}
name := cfg.C.Load().StatePath
data, err := os.ReadFile(name)
switch {
case err == nil && len(data) > 0:
if err := yaml.Unmarshal(data, &s.file); err != nil {
return err
}
// A null or absent entries mapping unmarshals over the initialized
// map with a nil one, so the map is restored before any write.
if s.file.Entries == nil {
s.file.Entries = map[string]*Entry{}
}
// Drop null entries a hand-edited file may carry, as lookups
// dereference the stored pointer.
for key, e := range s.file.Entries {
if e == nil {
delete(s.file.Entries, key)
}
}
case err != nil && !errors.Is(err, fs.ErrNotExist):
return fmt.Errorf("read state %s: %w", name, err)
}
S = s
return nil
}
// MarkRequested records a client request for a resource, creating it if
// needed. It returns a copy of the entry and whether the resource was
// already tracked, so a caller can tell a registration this request created
// from one an earlier request established.
func (s *Store) MarkRequested(kind, key, path, root string, now time.Time) (Entry, bool) {
s.mu.Lock()
defer s.mu.Unlock()
e, tracked := s.file.Entries[key]
if !tracked {
e = &Entry{}
s.file.Entries[key] = e
}
e.Kind = kind
e.Path = path
e.Root = root
e.LastRequested = now
s.dirty = true
return *e, tracked
}
// MarkSeenInInventory records that a configured mount still advertises the
// resource, shielding it from eviction.
func (s *Store) MarkSeenInInventory(kind, key, path, root string, now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
e := s.file.Entries[key]
if e == nil {
e = &Entry{}
s.file.Entries[key] = e
}
e.Kind = kind
e.Path = path
e.Root = root
e.LastSeenInInventory = now
s.dirty = true
}
// MarkCrawled records a crawl outcome, scheduling the next attempt sooner
// after failures.
func (s *Store) MarkCrawled(key string, now time.Time, crawlErr error) {
s.mu.Lock()
defer s.mu.Unlock()
e := s.file.Entries[key]
if e == nil {
return
}
e.LastCrawled = now
if crawlErr != nil {
e.LastError = crawlErr.Error()
e.NextCrawl = now.Add(cfg.C.Load().FailureRetryInterval)
} else {
e.LastError = ""
e.NextCrawl = now.Add(cfg.C.Load().RepoCrawlInterval)
}
s.dirty = true
}
// Delete removes a resource from tracking.
func (s *Store) Delete(key string) {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.file.Entries[key]; ok {
delete(s.file.Entries, key)
s.dirty = true
}
}
// Entry returns a copy of a tracked resource.
func (s *Store) Entry(key string) (Entry, bool) {
s.mu.Lock()
defer s.mu.Unlock()
e, ok := s.file.Entries[key]
if !ok {
return Entry{}, false
}
return *e, true
}
// Snapshot returns a copy of every tracked resource.
func (s *Store) Snapshot() map[string]Entry {
s.mu.Lock()
defer s.mu.Unlock()
out := make(map[string]Entry, len(s.file.Entries))
for key, e := range s.file.Entries {
out[key] = *e
}
return out
}
// TouchRepoMembers refreshes the last-requested time of every repository
// whose root covers a request path, keeping repositories alive while their
// files are fetched. It reports whether any repository matched.
func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool {
s.mu.Lock()
defer s.mu.Unlock()
matched := false
for _, e := range s.file.Entries {
if e.Kind == "generic" || e.Root == "" {
continue
}
under := reqPath == e.Root || e.Root == "/" ||
(len(reqPath) > len(e.Root) && reqPath[:len(e.Root)] == e.Root && reqPath[len(e.Root)] == '/')
if !under {
continue
}
e.LastRequested = now
matched = true
s.dirty = true
}
return matched
}
// Save writes the state atomically when it has changed.
func (s *Store) Save() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.dirty {
return nil
}
data, err := yaml.Marshal(s.file)
if err != nil {
return err
}
name := cfg.C.Load().StatePath
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
return err
}
tmp := name + ".tmp"
if err := os.WriteFile(tmp, data, 0644); err != nil {
return err
}
if err := os.Rename(tmp, name); err != nil {
return err
}
s.dirty = false
return nil
}
// FlushLoop saves periodically and once more at shutdown.
func (s *Store) FlushLoop(ctx context.Context) {
t := time.NewTicker(cfg.C.Load().StateFlushInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
_ = s.Save()
return
case <-t.C:
_ = s.Save()
// Re-arm from the active config so a reload takes effect.
t.Reset(cfg.C.Load().StateFlushInterval)
}
}
}

93
state/state_test.go Normal file
View file

@ -0,0 +1,93 @@
package state
import (
"os"
"path/filepath"
"testing"
"time"
cfg "github.com/grmrgecko/repo-sync/config"
)
// initConfig installs a minimal configuration with the given state path.
func initConfig(t *testing.T, statePath string) {
t.Helper()
conf := `
state_path: ` + statePath + `
domains:
- {domain: m.test, role: online, root: /tmp/online}
mounts:
- {path: /, upstream: "https://example.com"}
`
name := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(name, []byte(conf), 0644); err != nil {
t.Fatal(err)
}
if err := cfg.Init(name); err != nil {
t.Fatal(err)
}
}
// TestLoadMissing verifies a missing state file yields a fresh store.
func TestLoadMissing(t *testing.T) {
initConfig(t, filepath.Join(t.TempDir(), "state.yaml"))
if err := Load(); err != nil {
t.Fatal(err)
}
if len(S.Snapshot()) != 0 {
t.Error("fresh store is not empty")
}
}
// TestLoadUnreadable verifies read failures other than a missing file are
// reported to the caller.
func TestLoadUnreadable(t *testing.T) {
initConfig(t, t.TempDir())
if err := Load(); err == nil {
t.Fatal("expected an error for an unreadable state path")
}
}
// TestLoadNullEntryMap verifies a state file whose entries mapping is null
// yields a writable store.
func TestLoadNullEntryMap(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "state.yaml")
if err := os.WriteFile(statePath, []byte("entries:\n"), 0644); err != nil {
t.Fatal(err)
}
initConfig(t, statePath)
if err := Load(); err != nil {
t.Fatal(err)
}
S.MarkRequested("rpm", "rpm:/repo", "/repo", "/repo", time.Now())
if _, ok := S.Entry("rpm:/repo"); !ok {
t.Error("entry not recorded after loading a null entries mapping")
}
}
// TestLoadNullEntries verifies hand-edited null entries are dropped at load
// so later lookups only see valid entries.
func TestLoadNullEntries(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "state.yaml")
content := `
entries:
rpm:/repo: null
deb:/suite:
kind: deb
path: /suite
`
if err := os.WriteFile(statePath, []byte(content), 0644); err != nil {
t.Fatal(err)
}
initConfig(t, statePath)
if err := Load(); err != nil {
t.Fatal(err)
}
if _, ok := S.Entry("rpm:/repo"); ok {
t.Error("null entry survived load")
}
if _, ok := S.Entry("deb:/suite"); !ok {
t.Error("valid entry lost on load")
}
S.TouchRepoMembers("/repo/repodata/repomd.xml", time.Now())
}

141
sync_cmd.go Normal file
View file

@ -0,0 +1,141 @@
package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/grmrgecko/repo-sync/mirror"
)
// SyncArgs holds the flags and positional arguments shared by every
// repository type.
type SyncArgs struct {
Trim int `help:"Number of leading URL path components to drop when building the destination path." default:"0"`
Flat bool `help:"Place repositories directly in the destination directory without copying the URL path."`
Discover bool `help:"Treat each URL as a directory index and crawl it for repositories."`
Depth int `help:"Maximum directory depth crawled in discovery mode." default:"5"`
Workers int `help:"Number of concurrent download workers." default:"4"`
Verify bool `help:"Re-verify checksums of files that already exist locally."`
Prune bool `help:"Delete local files that are no longer part of the repository."`
PruneGrace time.Duration `help:"Keep files that left the repository for this long before pruning them (e.g. 72h)."`
DryRun bool `help:"Report what would be downloaded or pruned without changing the repository."`
Args []string `arg:"" name:"url" help:"Repository or mirrorlist URLs followed by the destination directory."`
}
// options validates the shared arguments and builds the mirror options.
func (s *SyncArgs) options(typ mirror.RepoType) (*mirror.Options, error) {
if len(s.Args) < 2 {
return nil, errors.New("expected at least one repository URL and a destination directory")
}
urls := s.Args[:len(s.Args)-1]
for _, raw := range urls {
u, err := url.Parse(raw)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, fmt.Errorf("invalid repository URL %q", raw)
}
}
dest, err := filepath.Abs(s.Args[len(s.Args)-1])
if err != nil {
return nil, err
}
if s.Trim < 0 {
return nil, errors.New("trim must not be negative")
}
if s.Depth < 0 {
return nil, errors.New("depth must not be negative")
}
if s.PruneGrace < 0 {
return nil, errors.New("prune grace must not be negative")
}
return &mirror.Options{
Type: typ,
URLs: urls,
Destination: dest,
Trim: s.Trim,
Flat: s.Flat,
Discover: s.Discover,
DiscoverDepth: s.Depth,
Workers: s.Workers,
Verify: s.Verify,
Prune: s.Prune,
PruneGrace: s.PruneGrace,
DryRun: s.DryRun,
}, nil
}
// run executes a synchronization with signal-aware cancellation.
func (s *SyncArgs) run(opts *mirror.Options) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
return mirror.Sync(ctx, opts)
}
// RPMCmd synchronizes RPM repositories.
type RPMCmd struct {
SyncArgs `embed:""`
}
// Run performs the RPM synchronization.
func (c *RPMCmd) Run() error {
opts, err := c.options(mirror.RepoRPM)
if err != nil {
return err
}
return c.run(opts)
}
// ArchCmd synchronizes Arch Linux repositories.
type ArchCmd struct {
SyncArgs `embed:""`
}
// Run performs the Arch Linux synchronization.
func (c *ArchCmd) Run() error {
opts, err := c.options(mirror.RepoArch)
if err != nil {
return err
}
return c.run(opts)
}
// ApkCmd synchronizes Alpine Linux repositories.
type ApkCmd struct {
SyncArgs `embed:""`
}
// Run performs the Alpine Linux synchronization.
func (c *ApkCmd) Run() error {
opts, err := c.options(mirror.RepoApk)
if err != nil {
return err
}
return c.run(opts)
}
// DebCmd synchronizes DEB repositories.
type DebCmd struct {
Component []string `help:"Limit synchronization to these components."`
Arch []string `help:"Limit synchronization to these architectures; include \"source\" to keep source indexes."`
SyncArgs `embed:""`
}
// Run performs the DEB synchronization.
func (c *DebCmd) Run() error {
opts, err := c.options(mirror.RepoDeb)
if err != nil {
return err
}
opts.Components = c.Component
opts.Architectures = c.Arch
return c.run(opts)
}