From 67c64a084d5f0d043a197adc400ecd5e22ea63c3 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Tue, 1 Sep 2026 18:22:04 -0500 Subject: [PATCH] Verify repository OpenPGP signatures --- VERSION | 2 +- config.example.yaml | 13 ++ config/config.go | 34 +++ config/config_test.go | 15 ++ fetch/fetch.go | 21 +- fetch/source.go | 4 + fetch/source_test.go | 18 ++ go.mod | 5 + go.sum | 4 + internal/testrepos/testrepos.go | 109 ++++++++++ mirror/arch.go | 147 ++++++++++++- mirror/arch_test.go | 40 ++++ mirror/deb.go | 242 +++++++++++++++++---- mirror/deb_test.go | 37 ++++ mirror/options.go | 10 + mirror/rpm.go | 356 ++++++++++++++++++++++++++----- mirror/rpm_test.go | 148 +++++++++++++ mirror/signature.go | 366 ++++++++++++++++++++++++++++++++ mirror/signature_test.go | 80 +++++++ server/crawl_loop.go | 153 ++++++++++++- server/serve.go | 61 +++++- server/serve_test.go | 100 +++++++++ state/state.go | 29 ++- sync_cmd.go | 26 +++ 24 files changed, 1896 insertions(+), 124 deletions(-) create mode 100644 mirror/signature.go create mode 100644 mirror/signature_test.go diff --git a/VERSION b/VERSION index 6e8bf73..17e51c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 +0.1.1 diff --git a/config.example.yaml b/config.example.yaml index 0504dc5..7a9b33c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -117,6 +117,19 @@ crawler: # which repositories the upstream has dropped since. Zero scans on every # run. discover_cache: 72h + # signature_mode controls OpenPGP checks for signed repository metadata. + # if-present accepts unsigned repositories but rejects a signature that + # cannot be verified. required also rejects missing signatures. + signature_mode: off + # When local public keyrings are configured, only their signers are + # accepted. Without them, repository-provided and keyserver-retrieved keys + # can prove that a data/signature pair matches but not who published it. + gpg_keys: [] + # The configured list replaces the defaults. Set it to [] to disable + # keyserver lookup. + keyservers: + - https://keyserver.ubuntu.com + - https://keys.openpgp.org # Backoff steps applied once a repository stops being requested. The # first tier is repo_crawl_interval; each entry extends the refresh # interval as the resource ages, and once the total budget elapses the diff --git a/config/config.go b/config/config.go index 007540c..b427abd 100644 --- a/config/config.go +++ b/config/config.go @@ -92,6 +92,13 @@ type CrawlerConfig struct { // DiscoverCache is how long a discovery crawl's results are reused // before the tree is scanned again. DiscoverCache time.Duration `mapstructure:"discover_cache" yaml:"discover_cache" validate:"gte=0"` + // SignatureMode controls OpenPGP verification of repository metadata. + SignatureMode string `mapstructure:"signature_mode" yaml:"signature_mode" validate:"oneof=off if-present required"` + // GPGKeys lists local public keyring files used for signature checks. + GPGKeys []string `mapstructure:"gpg_keys" yaml:"gpg_keys"` + // Keyservers lists optional OpenPGP keyservers used to retrieve unknown + // signature issuers. + Keyservers []string `mapstructure:"keyservers" yaml:"keyservers" validate:"omitempty,dive,url"` } // TraceConfig describes the mirror this instance publishes, and is written @@ -203,6 +210,15 @@ func loadConfig(configPath string) (*Config, error) { c.StatePath = "/etc/repo-sync/state.yaml" } } + if file != "" { + for i, name := range c.Crawler.GPGKeys { + name = strings.TrimSpace(name) + if name != "" && name != "~" && !strings.HasPrefix(name, "~/") && !filepath.IsAbs(name) { + name = filepath.Join(filepath.Dir(file), name) + } + c.Crawler.GPGKeys[i] = name + } + } if err := c.finalize(); err != nil { return nil, err @@ -292,6 +308,11 @@ func setDefaults(v *viper.Viper) { v.SetDefault("crawler.missing_mode", "retry") v.SetDefault("crawler.missing_retries", 3) v.SetDefault("crawler.discover_cache", 72*time.Hour) + v.SetDefault("crawler.signature_mode", "off") + v.SetDefault("crawler.keyservers", []string{ + "https://keyserver.ubuntu.com", + "https://keys.openpgp.org", + }) v.SetDefault("crawler.refresh_schedule", []time.Duration{ 12 * time.Hour, 24 * time.Hour, 48 * time.Hour, 72 * time.Hour, 96 * time.Hour, 120 * time.Hour, @@ -306,6 +327,12 @@ func (c *Config) finalize() error { if err := validateConfig(c); err != nil { return err } + for _, raw := range c.Crawler.Keyservers { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return fmt.Errorf("invalid crawler keyserver URL %q", raw) + } + } if err := c.indexDomains(); err != nil { return err } @@ -315,6 +342,13 @@ func (c *Config) finalize() error { // canonicalize normalizes user-supplied values before validation. func (c *Config) canonicalize() { c.StatePath = expandHome(strings.TrimSpace(c.StatePath)) + c.Crawler.SignatureMode = strings.ToLower(strings.TrimSpace(c.Crawler.SignatureMode)) + for i := range c.Crawler.GPGKeys { + c.Crawler.GPGKeys[i] = expandHome(strings.TrimSpace(c.Crawler.GPGKeys[i])) + } + for i := range c.Crawler.Keyservers { + c.Crawler.Keyservers[i] = strings.TrimRight(strings.TrimSpace(c.Crawler.Keyservers[i]), "/") + } for i := range c.Domains { c.Domains[i].Domain = strings.ToLower(strings.TrimSpace(c.Domains[i].Domain)) c.Domains[i].Role = strings.ToLower(strings.TrimSpace(c.Domains[i].Role)) diff --git a/config/config_test.go b/config/config_test.go index 48bd831..f441248 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -4,6 +4,9 @@ import ( "os" "path/filepath" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // writeConfig writes a temp config file and returns its path. @@ -140,3 +143,15 @@ func TestInitWithoutServerSections(t *testing.T) { t.Error("InitServer accepted a configuration with no domains") } } + +// TestKeyserverConfiguration verifies the built-in lookup services can be +// replaced or disabled without retaining entries from the default list. +func TestKeyserverConfiguration(t *testing.T) { + path := writeConfig(t, "crawler:\n keyservers:\n - https://keys.example.com\n") + require.NoError(t, Init(path)) + assert.Equal(t, []string{"https://keys.example.com"}, C.Crawler.Keyservers) + + path = writeConfig(t, "crawler:\n keyservers: []\n") + require.NoError(t, Init(path)) + assert.Empty(t, C.Crawler.Keyservers) +} diff --git a/fetch/fetch.go b/fetch/fetch.go index 5b79c43..fabf6c3 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -71,6 +71,10 @@ func Reload() { // treat missing optional files differently from transport failures. var ErrNotFound = errors.New("upstream file not found") +// ErrForbidden marks an upstream file that returned HTTP 403. Callers may +// tolerate this for optional files on object stores that hide missing keys. +var ErrForbidden = errors.New("upstream file forbidden") + // ErrNotModified marks a conditional request the upstream answered with // 304, meaning the local copy is already current. var ErrNotModified = errors.New("upstream file not modified") @@ -229,6 +233,17 @@ func lockFile(dst string) func() { // With stage set the download is written next to dst for a later promote so // readers of a live tree never see a partially updated repository. func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting bool) (FileState, error) { + return file(ctx, src, reqPath, dst, want, stage, verifyExisting, false) +} + +// FileFresh downloads reqPath without conditionally reusing the current +// destination. It is used when related upstream files must be refetched as +// one generation after a consistency check fails. +func FileFresh(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool) (FileState, error) { + return file(ctx, src, reqPath, dst, want, stage, false, true) +} + +func file(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage, verifyExisting, force bool) (FileState, error) { // Serialize writers to this destination so concurrent callers, such // as the mirror server's on-demand fetches and background crawls, // cannot interleave partial downloads of the same file. @@ -259,7 +274,7 @@ func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, s } for { - state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom) + state, err := download(ctx, src, reqPath, dst, want, stage, resumeFrom, force) if err != nil && resumeFrom > 0 && !errors.Is(err, ErrNotFound) && ctx.Err() == nil { // The partial may not be a prefix of the current upstream file; // retry once from scratch. @@ -274,12 +289,12 @@ func File(ctx context.Context, src *Source, reqPath, dst string, want *Expect, s // download performs one transfer attempt for File, appending to the // partial file when resuming from a prior failure. -func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64) (FileState, error) { +func download(ctx context.Context, src *Source, reqPath, dst string, want *Expect, stage bool, resumeFrom int64, force bool) (FileState, error) { // Files without published checksums cannot be verified locally, so an // existing copy is revalidated with a conditional request instead of // being re-downloaded every run. var modifiedSince time.Time - if want == nil { + if want == nil && !force { if info, err := os.Stat(dst); err == nil && info.Mode().IsRegular() { modifiedSince = info.ModTime() } diff --git a/fetch/source.go b/fetch/source.go index 501b14a..297e1ac 100644 --- a/fetch/source.go +++ b/fetch/source.go @@ -99,6 +99,10 @@ func (s *Source) Get(ctx context.Context, reqPath string, o GetOptions) (*http.R resp.Body.Close() return nil, fmt.Errorf("fetch %s: %w", target, ErrNotFound) } + if resp.StatusCode == http.StatusForbidden { + resp.Body.Close() + return nil, fmt.Errorf("fetch %s: %w", target, ErrForbidden) + } if resp.StatusCode == http.StatusNotModified && !o.ModifiedSince.IsZero() { resp.Body.Close() return nil, fmt.Errorf("fetch %s: %w", target, ErrNotModified) diff --git a/fetch/source_test.go b/fetch/source_test.go index 01ea993..9b63464 100644 --- a/fetch/source_test.go +++ b/fetch/source_test.go @@ -3,10 +3,14 @@ package fetch import ( "context" "errors" + "net/http" + "net/http/httptest" "path/filepath" "testing" "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestParseMirrorlist verifies URL extraction, comment skipping, and @@ -47,6 +51,20 @@ func TestSourceFailover(t *testing.T) { } } +// TestSourceForbidden verifies HTTP 403 remains distinct from a missing file +// so only callers fetching optional object-store keys can ignore it. +func TestSourceForbidden(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + _, err := NewSource([]string{srv.URL}).Get(context.Background(), "private", GetOptions{}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrForbidden) + assert.NotErrorIs(t, err, ErrNotFound) +} + // TestParseMetalink verifies base URL extraction from metalink documents. func TestParseMetalink(t *testing.T) { body := `` + diff --git a/go.mod b/go.mod index 5f9996a..bbba296 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/grmrgecko/repo-sync go 1.26.4 require ( + github.com/ProtonMail/go-crypto v1.4.1 github.com/alecthomas/kong v1.16.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/go-playground/validator/v10 v10.30.3 @@ -10,12 +11,15 @@ require ( github.com/klauspost/compress v1.19.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 github.com/ulikunitz/xz v0.5.16 golang.org/x/net v0.57.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/cloudflare/circl v1.6.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -23,6 +27,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect diff --git a/go.sum b/go.sum index 890d18f..6880e52 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kong v1.16.0 h1:g92/kUxBcdcTPOM79yE63viJgtcp5dNyrB3/O2cjYT4= github.com/alecthomas/kong v1.16.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/cloudflare/circl v1.6.2 h1:hL7VBpHHKzrV5WTfHCaBsgx/HGbBYlgrwvNXEVDYYsQ= +github.com/cloudflare/circl v1.6.2/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/testrepos/testrepos.go b/internal/testrepos/testrepos.go index 7a9630f..30224fd 100644 --- a/internal/testrepos/testrepos.go +++ b/internal/testrepos/testrepos.go @@ -16,8 +16,117 @@ import ( "path/filepath" "strings" "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/clearsign" ) +// SigningKey is a generated OpenPGP key used by signed repository fixtures. +type SigningKey struct { + entity *openpgp.Entity +} + +// NewSigningKey generates an OpenPGP signing key for a test repository. +func NewSigningKey(t *testing.T) *SigningKey { + t.Helper() + entity, err := openpgp.NewEntity("Repository test key", "", "repo@example.com", nil) + if err != nil { + t.Fatal(err) + } + return &SigningKey{entity: entity} +} + +// PublicKey returns the armored public keyring for the signing key. +func (k *SigningKey) PublicKey(t *testing.T) []byte { + t.Helper() + var out bytes.Buffer + w, err := armor.Encode(&out, openpgp.PublicKeyType, nil) + if err != nil { + t.Fatal(err) + } + if err := k.entity.Serialize(w); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +// Sign returns an armored detached signature over data. +func (k *SigningKey) Sign(t *testing.T, data []byte) []byte { + t.Helper() + var out bytes.Buffer + if err := openpgp.ArmoredDetachSign(&out, k.entity, bytes.NewReader(data), nil); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +// SignBinary returns a binary detached signature over data. +func (k *SigningKey) SignBinary(t *testing.T, data []byte) []byte { + t.Helper() + var out bytes.Buffer + if err := openpgp.DetachSign(&out, k.entity, bytes.NewReader(data), nil); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +// ClearSign returns an armored cleartext signature over data. +func (k *SigningKey) ClearSign(t *testing.T, data []byte) []byte { + t.Helper() + var out bytes.Buffer + w, err := clearsign.Encode(&out, k.entity.PrivateKey, nil) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +// SignRPMRepo replaces the fixture signature and publishes its public key. +func (k *SigningKey) SignRPMRepo(t *testing.T, dir string) { + t.Helper() + repomd, err := os.ReadFile(filepath.Join(dir, "repodata", "repomd.xml")) + if err != nil { + t.Fatal(err) + } + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.asc"), k.Sign(t, repomd)) + WriteFile(t, filepath.Join(dir, "repodata", "repomd.xml.key"), k.PublicKey(t)) +} + +// SignDebRepo signs the fixture Release file in both apt formats. +func (k *SigningKey) SignDebRepo(t *testing.T, dir string) { + t.Helper() + release, err := os.ReadFile(filepath.Join(dir, "dists", "test", "Release")) + if err != nil { + t.Fatal(err) + } + WriteFile(t, filepath.Join(dir, "dists", "test", "InRelease"), k.ClearSign(t, release)) + WriteFile(t, filepath.Join(dir, "dists", "test", "Release.gpg"), k.SignBinary(t, release)) +} + +// SignArchRepo signs the fixture database and each package with binary +// detached signatures, matching files published by Arch mirrors. +func (k *SigningKey) SignArchRepo(t *testing.T, dir, name string, packages map[string][]byte) { + t.Helper() + database, err := os.ReadFile(filepath.Join(dir, name+".db")) + if err != nil { + t.Fatal(err) + } + WriteFile(t, filepath.Join(dir, name+".db.sig"), k.SignBinary(t, database)) + for filename, data := range packages { + WriteFile(t, filepath.Join(dir, filename+".sig"), k.SignBinary(t, data)) + } +} + // WriteFile creates a file with parent directories under a fixture tree. func WriteFile(t *testing.T, name string, data []byte) { t.Helper() diff --git a/mirror/arch.go b/mirror/arch.go index c7e1148..747bf47 100644 --- a/mirror/arch.go +++ b/mirror/arch.go @@ -42,7 +42,6 @@ var archExtras = []string{ ".db.tar.gz", ".files.tar.gz", ".links.tar.gz", - ".db.sig", ".db.tar.gz.sig", ".files.sig", ".files.tar.gz.sig", @@ -55,6 +54,17 @@ func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, o miss := opts.newMissing(destDir) tr := newTrace(opts) ctx = tr.track(ctx) + mode := opts.SignatureMode + if mode == "" { + mode = SignatureOff + } + signed := mode != SignatureOff + var stagedPaths []string + defer func() { + for _, filename := range stagedPaths { + _ = os.Remove(filename + fetch.StagedSuffix) + } + }() // Determine the database name, which matches the repository rather // than any fixed path. @@ -70,30 +80,134 @@ func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, o if err != nil { return err } + stagedPaths = append(stagedPaths, dbDst) if _, err := fetch.File(ctx, src, name+".db", dbDst, nil, true, false); err != nil { return fmt.Errorf("fetch %s.db: %w", name, err) } + dbSigDst, err := fetch.LocalJoin(destDir, name+".db.sig") + if err != nil { + return err + } + stagedPaths = append(stagedPaths, dbSigDst) + dbSigState, err := fetch.File(ctx, src, name+".db.sig", dbSigDst, nil, true, false) + if err != nil && !errors.Is(err, fetch.ErrNotFound) && !errors.Is(err, fetch.ErrForbidden) { + return fmt.Errorf("fetch %s.db.sig: %w", name, err) + } + if err != nil { + dbSigState = fetch.FileMissing + } + + var verifier *signatureVerifier + if signed { + verifier, err = newSignatureVerifier(opts) + if err != nil { + return err + } + } + if signed && dbSigState != fetch.FileMissing { + for attempt := 0; attempt < 2; attempt++ { + fingerprint, verifyErr := verifier.verifyDetached(ctx, fetch.StagedOrFinal(dbDst), fetch.StagedOrFinal(dbSigDst), "") + if verifyErr == nil { + log.WithField("fingerprint", fingerprint).Debug("Verified pacman database signature.") + break + } + if attempt == 1 { + return fmt.Errorf("verify %s.db signature after refetch: %w", name, verifyErr) + } + log.WithError(verifyErr).Warn("Staged pacman database signature failed verification; refetching the pair.") + if _, err := fetch.FileFresh(ctx, src, name+".db", dbDst, nil, true); err != nil { + return fmt.Errorf("refetch %s.db: %w", name, err) + } + if _, err := fetch.FileFresh(ctx, src, name+".db.sig", dbSigDst, nil, true); err != nil { + return fmt.Errorf("refetch %s.db.sig: %w", name, err) + } + } + } pkgs, err := readArchDB(fetch.StagedOrFinal(dbDst)) if err != nil { return err } - // Download packages with their detached signatures. Signatures have no - // published checksums, so an existing file is trusted as-is. + // Download packages with their detached signatures. Signed modes stage + // both members so no unverified package becomes visible. var jobs []fetch.Job for _, pkg := range pkgs { dst, err := fetch.LocalJoin(destDir, pkg.filename) if err != nil { return err } - jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect()}) - jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: &fetch.Expect{Size: -1}, Optional: true}) + var signatureExpectation *fetch.Expect + if !signed { + signatureExpectation = &fetch.Expect{Size: -1} + } + stagedPaths = append(stagedPaths, dst, dst+".sig") + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename, Dst: dst, Want: pkg.expect(), Stage: signed}) + jobs = append(jobs, fetch.Job{ReqPath: pkg.filename + ".sig", Dst: dst + ".sig", Want: signatureExpectation, Optional: true, Stage: signed}) } log.WithField("packages", len(pkgs)).Info("Synchronizing packages.") + var packageStates []fetch.FileState if opts.DryRun { - fetch.PlanJobs(jobs, opts.Verify, keep) - } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil { - return fmt.Errorf("fetch packages: %w", err) + fetch.PlanJobs(jobs, opts.Verify || signed, keep) + } else { + packageStates, err = fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify || signed, keep, miss) + if err != nil { + return fmt.Errorf("fetch packages: %w", err) + } + } + + // Verify all package pairs before publishing any of them. Required mode + // applies to package signatures; official Arch mirrors commonly omit a + // detached signature for the repository database itself. + if signed && !opts.DryRun { + for i, pkg := range pkgs { + packageJob := jobs[i*2] + signatureJob := jobs[i*2+1] + if packageStates[i*2] == fetch.FileMissing { + continue + } + if packageStates[i*2+1] == fetch.FileMissing { + if mode == SignatureRequired { + return fmt.Errorf("package signature is required but %s is missing", signatureJob.ReqPath) + } + continue + } + for attempt := 0; attempt < 2; attempt++ { + fingerprint, verifyErr := verifier.verifyDetached(ctx, fetch.StagedOrFinal(packageJob.Dst), fetch.StagedOrFinal(signatureJob.Dst), "") + if verifyErr == nil { + log.WithFields(log.Fields{"fingerprint": fingerprint, "package": pkg.filename}).Debug("Verified package signature.") + break + } + if attempt == 1 { + return fmt.Errorf("verify package signature %s after refetch: %w", signatureJob.ReqPath, verifyErr) + } + log.WithError(verifyErr).WithField("package", pkg.filename).Warn("Staged package signature failed verification; refetching the pair.") + if _, err := fetch.FileFresh(ctx, src, packageJob.ReqPath, packageJob.Dst, packageJob.Want, true); err != nil { + return fmt.Errorf("refetch package %s: %w", packageJob.ReqPath, err) + } + if _, err := fetch.FileFresh(ctx, src, signatureJob.ReqPath, signatureJob.Dst, signatureJob.Want, true); err != nil { + return fmt.Errorf("refetch package signature %s: %w", signatureJob.ReqPath, err) + } + } + } + if err := miss.Finish(); err != nil { + return err + } + for pass := 1; pass >= 0; pass-- { + for i, job := range jobs { + if i%2 != pass { + continue + } + if packageStates[i] == fetch.FileMissing { + if i%2 == 1 { + fetch.RemoveStale(job.Dst) + } + continue + } + if err := fetch.PromoteStaged(job.Dst); err != nil { + return err + } + } + } } // Stage the companion metadata files that exist upstream. A dry run @@ -104,6 +218,7 @@ func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, o if err != nil { return err } + stagedPaths = append(stagedPaths, dst) extraJobs = append(extraJobs, fetch.Job{ReqPath: name + suffix, Dst: dst, Optional: true, Stage: true}) } if opts.DryRun { @@ -128,6 +243,15 @@ func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, o // Promote the database last so the published metadata chain is // complete. + if dbSigState == fetch.FileMissing { + if !opts.DryRun { + fetch.RemoveStale(dbSigDst) + } + } else if err := fetch.PromoteOrDiscard(dbSigDst, opts.DryRun); err != nil { + return err + } else { + keep.Add(dbSigDst) + } if err := fetch.PromoteOrDiscard(dbDst, opts.DryRun); err != nil { return err } @@ -142,8 +266,11 @@ func syncArch(ctx context.Context, src *fetch.Source, repoURL, destDir string, o fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) } - // The database is published either way; missing packages only decide - // whether the run reports itself as failed. + if signed { + return nil + } + // Unsigned mode publishes the database before reporting package files + // that remained unavailable after retries. return miss.Finish() } diff --git a/mirror/arch_test.go b/mirror/arch_test.go index e46b456..b8dcb1f 100644 --- a/mirror/arch_test.go +++ b/mirror/arch_test.go @@ -10,9 +10,12 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/grmrgecko/repo-sync/fetch" "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestParseDesc verifies field extraction from a pacman desc entry. @@ -33,6 +36,43 @@ func TestParseDesc(t *testing.T) { } } +// TestSyncArchSignedPackages verifies Arch binary signatures before replacing +// any package or repository database in the live tree. +func TestSyncArchSignedPackages(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "core", "os", "x86_64") + packages := testrepos.BuildArchRepo(t, repoDir, "core") + key := testrepos.NewSigningKey(t) + key.SignArchRepo(t, repoDir, "core", packages) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{ + Type: RepoArch, + Destination: dest, + Workers: 2, + SignatureMode: SignatureRequired, + GPGKeyData: [][]byte{key.PublicKey(t)}, + } + repoURL := srv.URL + "/core/os/x86_64" + require.NoError(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + + var packageName string + for packageName = range packages { + break + } + localSig := filepath.Join(dest, "core", "os", "x86_64", packageName+".sig") + published := requireReadFile(t, localSig) + upstreamSig := filepath.Join(repoDir, packageName+".sig") + require.NoError(t, os.WriteFile(upstreamSig, []byte("invalid signature"), 0644)) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(upstreamSig, future, future)) + + require.Error(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + assert.Equal(t, published, requireReadFile(t, localSig), "a bad package signature must not replace the verified pair") + assert.NoFileExists(t, localSig+fetch.StagedSuffix) +} + // TestReadArchDB verifies database parsing for both compressed and plain // tar archives. func TestReadArchDB(t *testing.T) { diff --git a/mirror/deb.go b/mirror/deb.go index bbe2ceb..9a2e233 100644 --- a/mirror/deb.go +++ b/mirror/deb.go @@ -45,6 +45,178 @@ type debRelease struct { files map[string]*debFile } +// debReleaseSet tracks one staged set of apt release documents. +type debReleaseSet struct { + names []string + dsts map[string]string + missing map[string]bool + data []byte + authenticated bool +} + +// cleanup removes unpublished release files left by a failed synchronization. +func (r *debReleaseSet) cleanup() { + for _, dst := range r.dsts { + _ = os.Remove(dst + fetch.StagedSuffix) + } +} + +// stageDebRelease stages and verifies the apt release documents before any +// checksums or package paths are trusted. +func stageDebRelease(ctx context.Context, src *fetch.Source, destSuite, suitePrefix string, opts *Options) (*debReleaseSet, error) { + r := &debReleaseSet{ + names: []string{"InRelease", "Release", "Release.gpg"}, + dsts: map[string]string{}, + missing: map[string]bool{}, + } + for _, name := range r.names { + dst, err := fetch.LocalJoin(destSuite, name) + if err != nil { + return nil, err + } + r.dsts[name] = dst + if err := r.fetch(ctx, src, suitePrefix, name, false); err != nil { + r.cleanup() + return nil, err + } + } + + mode := opts.SignatureMode + if mode == "" { + mode = SignatureOff + } + if mode == SignatureOff { + return r.selectData(false) + } + verifier, err := newSignatureVerifier(opts) + if err != nil { + r.cleanup() + return nil, err + } + + // InRelease is self-contained, so a mismatch only needs that document + // refetched. Detached Release signatures retry both members together. + if !r.missing["InRelease"] { + for attempt := 0; attempt < 2; attempt++ { + data, err := os.ReadFile(fetch.StagedOrFinal(r.dsts["InRelease"])) + if err != nil { + r.cleanup() + return nil, err + } + plaintext, fingerprint, verifyErr := verifier.verifyClearsigned(ctx, data) + if verifyErr == nil { + r.data = plaintext + r.authenticated = true + log.WithField("fingerprint", fingerprint).Debug("Verified InRelease signature.") + break + } + if attempt == 1 { + r.cleanup() + return nil, fmt.Errorf("verify InRelease signature after refetch: %w", verifyErr) + } + log.WithError(verifyErr).Warn("Staged InRelease signature failed verification; refetching.") + if err := r.fetch(ctx, src, suitePrefix, "InRelease", true); err != nil { + r.cleanup() + return nil, err + } + if r.missing["InRelease"] { + break + } + } + } + + if !r.missing["Release.gpg"] { + if r.missing["Release"] { + r.cleanup() + return nil, errors.New("repository serves Release.gpg without Release") + } + for attempt := 0; attempt < 2; attempt++ { + fingerprint, verifyErr := verifier.verifyDetached(ctx, fetch.StagedOrFinal(r.dsts["Release"]), fetch.StagedOrFinal(r.dsts["Release.gpg"]), "") + if verifyErr == nil { + log.WithField("fingerprint", fingerprint).Debug("Verified Release.gpg signature.") + if r.missing["InRelease"] { + r.authenticated = true + } + break + } + if attempt == 1 { + r.cleanup() + return nil, fmt.Errorf("verify Release.gpg signature after refetch: %w", verifyErr) + } + log.WithError(verifyErr).Warn("Staged Release signature failed verification; refetching the pair.") + for _, name := range []string{"Release", "Release.gpg"} { + if err := r.fetch(ctx, src, suitePrefix, name, true); err != nil { + r.cleanup() + return nil, err + } + } + if r.missing["Release"] { + r.cleanup() + return nil, errors.New("release disappeared while refetching its signature pair") + } + if r.missing["Release.gpg"] { + break + } + } + } + + if r.missing["InRelease"] { + if r.missing["Release"] { + r.cleanup() + return nil, errors.New("repository serves neither InRelease nor Release") + } + if r.missing["Release.gpg"] && mode == SignatureRequired { + r.cleanup() + return nil, errors.New("repository signature is required but Release.gpg is missing") + } + data, err := os.ReadFile(fetch.StagedOrFinal(r.dsts["Release"])) + if err != nil { + r.cleanup() + return nil, err + } + r.data = data + } + return r, nil +} + +func (r *debReleaseSet) fetch(ctx context.Context, src *fetch.Source, suitePrefix, name string, fresh bool) error { + var err error + if fresh { + _, err = fetch.FileFresh(ctx, src, suitePrefix+name, r.dsts[name], nil, true) + } else { + _, err = fetch.File(ctx, src, suitePrefix+name, r.dsts[name], nil, true, false) + } + switch { + case err == nil: + r.missing[name] = false + return nil + case errors.Is(err, fetch.ErrNotFound), errors.Is(err, fetch.ErrForbidden): + r.missing[name] = true + return nil + default: + return fmt.Errorf("fetch %s: %w", name, err) + } +} + +func (r *debReleaseSet) selectData(authenticated bool) (*debReleaseSet, error) { + name := "InRelease" + if r.missing[name] { + name = "Release" + } + if r.missing[name] { + r.cleanup() + return nil, errors.New("repository serves neither InRelease nor Release") + } + data, err := os.ReadFile(fetch.StagedOrFinal(r.dsts[name])) + if err != nil { + r.cleanup() + return nil, err + } + r.data = data + r.authenticated = authenticated + return r, nil +} + // releaseSumFields maps Release checksum block names to algorithm names, // including the MD5sum casing some repositories use. var releaseSumFields = map[string]string{ @@ -139,45 +311,13 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio // the archive's other suites. miss := opts.newMissing(destSuite) - // Stage the release files; they are promoted last so a live tree keeps - // a consistent metadata chain. - releaseNames := []string{"InRelease", "Release", "Release.gpg"} - releaseStates := make([]fetch.FileState, len(releaseNames)) - for i, name := range releaseNames { - dst, err := fetch.LocalJoin(destSuite, name) - if err != nil { - return err - } - state, err := fetch.File(ctx, src, suitePrefix+name, dst, nil, true, false) - if err != nil { - if errors.Is(err, fetch.ErrNotFound) { - // The upstream dropped this release file; drop the local - // copy so outdated metadata is never served. - if !opts.DryRun { - fetch.RemoveStale(dst) - } - continue - } - return fmt.Errorf("fetch %s: %w", name, err) - } - releaseStates[i] = state - } - - // Parse the strongest release document available, which may be the - // promoted copy when the upstream reported it unchanged. - var relData []byte - switch { - case releaseStates[0] != fetch.FileMissing: - relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "InRelease"))) - case releaseStates[1] != fetch.FileMissing: - relData, err = os.ReadFile(fetch.StagedOrFinal(filepath.Join(destSuite, "Release"))) - default: - return errors.New("repository serves neither InRelease nor Release") - } + // Authenticate release checksums before selecting any index or pool file. + releaseSet, err := stageDebRelease(ctx, src, destSuite, suitePrefix, opts) if err != nil { return err } - rel, err := parseRelease(relData) + defer releaseSet.cleanup() + rel, err := parseRelease(releaseSet.data) if err != nil { return err } @@ -190,7 +330,7 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio for _, f := range sortedFiles(rel.files) { // Releases commonly list their own release files; those are // already staged above and must not be fetched twice. - if slices.Contains(releaseNames, f.path) { + if slices.Contains(releaseSet.names, f.path) { continue } if !includeIndexFile(f.path, opts.Components, opts.Architectures) { @@ -216,8 +356,9 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio } indexJobs = append(indexJobs, job) } - fetch.PlanJobs(plannedJobs, opts.Verify, keep) - states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, opts.Verify, keep, nil) + verifyFiles := opts.Verify || releaseSet.authenticated + fetch.PlanJobs(plannedJobs, verifyFiles, keep) + states, err := fetch.Many(ctx, src, indexJobs, opts.Workers, verifyFiles, keep, nil) if err != nil { return fmt.Errorf("fetch suite indexes: %w", err) } @@ -271,10 +412,15 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio } log.WithField("files", len(poolJobs)).Info("Synchronizing pool files.") if opts.DryRun { - fetch.PlanJobs(poolJobs, opts.Verify, keep) - } else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, opts.Verify, keep, miss); err != nil { + fetch.PlanJobs(poolJobs, verifyFiles, keep) + } else if _, err := fetch.Many(ctx, src, poolJobs, opts.Workers, verifyFiles, keep, miss); err != nil { return fmt.Errorf("fetch pool files: %w", err) } + if releaseSet.authenticated { + if err := miss.Finish(); err != nil { + return err + } + } // Promote the staged indexes now the pool files they reference exist. for i, job := range indexJobs { @@ -291,9 +437,12 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio // Promote the release files last to complete the metadata chain. for _, name := range []string{"Release.gpg", "Release", "InRelease"} { - dst, err := fetch.LocalJoin(destSuite, name) - if err != nil { - return err + dst := releaseSet.dsts[name] + if releaseSet.missing[name] { + if !opts.DryRun { + fetch.RemoveStale(dst) + } + continue } if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { return err @@ -320,8 +469,9 @@ func syncDeb(ctx context.Context, src *fetch.Source, repoURL string, opts *Optio } } - // The suite is published either way; missing pool files only decide - // whether the run reports itself as failed. + if releaseSet.authenticated { + return nil + } return miss.Finish() } diff --git a/mirror/deb_test.go b/mirror/deb_test.go index b0f9b6e..abe2950 100644 --- a/mirror/deb_test.go +++ b/mirror/deb_test.go @@ -6,9 +6,12 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/grmrgecko/repo-sync/fetch" "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestParseRelease verifies field extraction and checksum block merging. @@ -38,6 +41,40 @@ func TestParseRelease(t *testing.T) { } } +// TestSyncDebSignedRelease verifies apt cleartext and detached signatures +// before replacing the live release generation. +func TestSyncDebSignedRelease(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "debian") + testrepos.BuildDebRepo(t, repoDir) + key := testrepos.NewSigningKey(t) + key.SignDebRepo(t, repoDir) + srv := testrepos.ServeDir(t, www) + + dest := t.TempDir() + opts := &Options{ + Type: RepoDeb, + Destination: dest, + Workers: 2, + SignatureMode: SignatureRequired, + GPGKeyData: [][]byte{key.PublicKey(t)}, + } + repoURL := srv.URL + "/debian/dists/test" + require.NoError(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + + local := filepath.Join(dest, "debian", "dists", "test", "InRelease") + published := requireReadFile(t, local) + upstream := filepath.Join(repoDir, "dists", "test", "InRelease") + tampered := bytes.Replace(requireReadFile(t, upstream), []byte("Origin: Test"), []byte("Origin: Pest"), 1) + require.NoError(t, os.WriteFile(upstream, tampered, 0644)) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(upstream, future, future)) + + require.Error(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + assert.Equal(t, published, requireReadFile(t, local), "a bad InRelease must not replace the verified release") + assert.NoFileExists(t, local+fetch.StagedSuffix) +} + // TestIndexArch verifies architecture extraction from release file paths. func TestIndexArch(t *testing.T) { cases := map[string]string{ diff --git a/mirror/options.go b/mirror/options.go index 9a105ea..a623b37 100644 --- a/mirror/options.go +++ b/mirror/options.go @@ -83,6 +83,16 @@ type Options struct { Workers int // Verify re-verifies checksums of existing local files. Verify bool + // SignatureMode controls OpenPGP verification of repository metadata. + SignatureMode SignatureMode + // GPGKeys lists local public keyring files used for signature checks. + GPGKeys []string + // GPGKeyData carries public keyrings preloaded from one configuration + // snapshot. The mirror server uses it to keep reloads out of a crawl. + GPGKeyData [][]byte + // Keyservers lists optional OpenPGP keyservers used to retrieve an unknown + // signature issuer. Retrieved keys prove pairing but are not trust anchors. + Keyservers []string // Prune deletes local files no longer part of the repository. Prune bool // PruneGrace defers pruning a file until it has been unreferenced for diff --git a/mirror/rpm.go b/mirror/rpm.go index 6372293..5672ace 100644 --- a/mirror/rpm.go +++ b/mirror/rpm.go @@ -12,6 +12,278 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + rpmRepomdPath = "repodata/repomd.xml" + rpmSigPath = "repodata/repomd.xml.asc" + rpmKeyPath = "repodata/repomd.xml.key" +) + +// rpmRoot is one staged repomd generation and its optional signature +// material. +type rpmRoot struct { + repomdDst string + sigDst string + keyDst string + sigMissing bool + keyMissing bool + authenticated bool +} + +// rpmPublishedFile is a rollback copy of one live root metadata file. +type rpmPublishedFile struct { + path string + data []byte + mode os.FileMode + exists bool +} + +// rpmWithdrawnPackage is a stale package hidden until root publication +// succeeds or restored if publication fails. +type rpmWithdrawnPackage struct { + path string + backup string +} + +// cleanup removes unpublished files left from staging a repomd generation. +func (r *rpmRoot) cleanup() { + for _, name := range []string{r.repomdDst, r.sigDst, r.keyDst} { + _ = os.Remove(name + fetch.StagedSuffix) + } +} + +// stageRPMRoot stages and optionally verifies repomd.xml with its detached +// signature. A failed check refetches the complete set once because an +// upstream rotation can occur between the requests. +func stageRPMRoot(ctx context.Context, src *fetch.Source, destDir string, opts *Options) (*rpmRoot, error) { + r := &rpmRoot{} + var err error + if r.repomdDst, err = fetch.LocalJoin(destDir, rpmRepomdPath); err != nil { + return nil, err + } + if r.sigDst, err = fetch.LocalJoin(destDir, rpmSigPath); err != nil { + return nil, err + } + if r.keyDst, err = fetch.LocalJoin(destDir, rpmKeyPath); err != nil { + return nil, err + } + + mode := opts.SignatureMode + if mode == "" { + mode = SignatureOff + } + var verifier *signatureVerifier + if mode != SignatureOff { + verifier, err = newSignatureVerifier(opts) + if err != nil { + return nil, err + } + } + + for attempt := 0; attempt < 2; attempt++ { + fresh := attempt > 0 + file := fetch.File + if fresh { + file = func(ctx context.Context, src *fetch.Source, reqPath, dst string, want *fetch.Expect, stage, _ bool) (fetch.FileState, error) { + return fetch.FileFresh(ctx, src, reqPath, dst, want, stage) + } + } + if _, err := file(ctx, src, rpmRepomdPath, r.repomdDst, nil, true, false); err != nil { + r.cleanup() + return nil, fmt.Errorf("fetch repomd.xml: %w", err) + } + + r.sigMissing = false + r.keyMissing = false + for _, extra := range []struct { + req string + dst string + missing *bool + }{ + {rpmSigPath, r.sigDst, &r.sigMissing}, + {rpmKeyPath, r.keyDst, &r.keyMissing}, + } { + _, err := file(ctx, src, extra.req, extra.dst, nil, true, false) + switch { + case err == nil: + case errors.Is(err, fetch.ErrNotFound), errors.Is(err, fetch.ErrForbidden): + *extra.missing = true + case err != nil: + r.cleanup() + return nil, fmt.Errorf("fetch %s: %w", extra.req, err) + } + } + + if mode == SignatureOff { + return r, nil + } + if r.sigMissing { + if mode == SignatureRequired { + r.cleanup() + return nil, errors.New("repository signature is required but repomd.xml.asc is missing") + } + return r, nil + } + + keyPath := "" + if !r.keyMissing { + keyPath = fetch.StagedOrFinal(r.keyDst) + } + fingerprint, verifyErr := verifier.verifyDetached( + ctx, + fetch.StagedOrFinal(r.repomdDst), + fetch.StagedOrFinal(r.sigDst), + keyPath, + ) + if verifyErr == nil { + r.authenticated = true + log.WithField("fingerprint", fingerprint).Debug("Verified repomd.xml signature.") + return r, nil + } + if !fresh { + log.WithError(verifyErr).Warn("Staged repomd.xml signature failed verification; refetching the pair.") + continue + } + r.cleanup() + return nil, fmt.Errorf("verify repomd.xml signature after refetch: %w", verifyErr) + } + return nil, errors.New("unable to stage repomd.xml") +} + +// snapshotRPMRoot reads the current live root set before publication. +func snapshotRPMRoot(paths ...string) ([]rpmPublishedFile, error) { + files := make([]rpmPublishedFile, 0, len(paths)) + for _, name := range paths { + file := rpmPublishedFile{path: name} + info, err := os.Stat(name) + if os.IsNotExist(err) { + files = append(files, file) + continue + } + if err != nil { + return nil, err + } + file.data, err = os.ReadFile(name) + if err != nil { + return nil, err + } + file.mode = info.Mode() + file.exists = true + files = append(files, file) + } + return files, nil +} + +// restoreRPMRoot replaces a partially published root with its prior files. +func restoreRPMRoot(files []rpmPublishedFile) error { + var errs []error + for _, file := range files { + if !file.exists { + if err := os.Remove(file.path); err != nil && !os.IsNotExist(err) { + errs = append(errs, err) + } + continue + } + tmp := file.path + ".rollback" + fetch.StagedSuffix + if err := os.WriteFile(tmp, file.data, file.mode); err != nil { + errs = append(errs, err) + continue + } + if err := os.Rename(tmp, file.path); err != nil { + _ = os.Remove(tmp) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// withdrawMissingPackages hides checksum-invalid packages the authenticated +// root no longer provides upstream. +func withdrawMissingPackages(jobs []fetch.Job, states []fetch.FileState) ([]rpmWithdrawnPackage, error) { + var withdrawn []rpmWithdrawnPackage + for i, state := range states { + if state != fetch.FileMissing { + continue + } + if _, err := os.Stat(jobs[i].Dst); os.IsNotExist(err) { + continue + } else if err != nil { + return withdrawn, err + } + backup := jobs[i].Dst + ".withdrawn" + fetch.StagedSuffix + if err := os.Remove(backup); err != nil && !os.IsNotExist(err) { + return withdrawn, err + } + if err := os.Rename(jobs[i].Dst, backup); err != nil { + return withdrawn, err + } + withdrawn = append(withdrawn, rpmWithdrawnPackage{path: jobs[i].Dst, backup: backup}) + } + return withdrawn, nil +} + +// finishWithdrawnPackages removes hidden packages after publication or puts +// them back when publication fails. +func finishWithdrawnPackages(files []rpmWithdrawnPackage, published bool) error { + var errs []error + for _, file := range files { + if published { + if err := os.Remove(file.backup); err != nil && !os.IsNotExist(err) { + errs = append(errs, err) + } + continue + } + if err := os.Rename(file.backup, file.path); err != nil && !os.IsNotExist(err) { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// publishRPMRoot publishes signature material immediately before repomd.xml. +// Missing optional files are removed only after the replacement is ready. +func publishRPMRoot(r *rpmRoot, keep *fetch.KeepSet, dryRun bool) (retErr error) { + var published []rpmPublishedFile + if !dryRun { + var err error + published, err = snapshotRPMRoot(r.keyDst, r.sigDst, r.repomdDst) + if err != nil { + return err + } + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, restoreRPMRoot(published)) + } + }() + } + for _, extra := range []struct { + dst string + missing bool + }{ + {r.keyDst, r.keyMissing}, + {r.sigDst, r.sigMissing}, + } { + if extra.missing { + if !dryRun { + if err := os.Remove(extra.dst); err != nil && !os.IsNotExist(err) { + return err + } + } + continue + } + if err := fetch.PromoteOrDiscard(extra.dst, dryRun); err != nil { + return err + } + if _, err := os.Stat(extra.dst); err == nil { + keep.Add(extra.dst) + } + } + if err := fetch.PromoteOrDiscard(r.repomdDst, dryRun); err != nil { + return err + } + keep.Add(r.repomdDst) + return nil +} + // repoMD is the partial XML schema for a yum repomd.xml file. Only the // fields the sync needs are decoded; unknown elements are ignored so // upstream additions do not break parsing. @@ -58,16 +330,13 @@ func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Optio tr := newTrace(opts) ctx = tr.track(ctx) - // Stage repomd.xml so clients of a live tree keep a consistent view of - // the old repository until everything else is in place. - repomdDst, err := fetch.LocalJoin(destDir, "repodata/repomd.xml") + // Authenticate the root metadata before using it to select any files. + root, err := stageRPMRoot(ctx, src, destDir, opts) if err != nil { return err } - if _, err := fetch.File(ctx, src, "repodata/repomd.xml", repomdDst, nil, true, false); err != nil { - return fmt.Errorf("fetch repomd.xml: %w", err) - } - md, err := readRepomd(fetch.StagedOrFinal(repomdDst)) + defer root.cleanup() + md, err := readRepomd(fetch.StagedOrFinal(root.repomdDst)) if err != nil { return err } @@ -111,8 +380,9 @@ func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Optio } jobs = append(jobs, job) } - fetch.PlanJobs(planned, opts.Verify, keep) - if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, nil); err != nil { + verifyMetadata := opts.Verify || root.authenticated + fetch.PlanJobs(planned, verifyMetadata, keep) + if _, err := fetch.Many(ctx, src, jobs, opts.Workers, verifyMetadata, keep, nil); err != nil { return fmt.Errorf("fetch repository metadata: %w", err) } if primary == nil { @@ -169,10 +439,15 @@ func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Optio } } log.WithField("packages", len(jobs)).Info("Synchronizing packages.") + verifyPackages := opts.Verify || root.authenticated + var packageStates []fetch.FileState if opts.DryRun { - fetch.PlanJobs(jobs, opts.Verify, keep) - } else if _, err := fetch.Many(ctx, src, jobs, opts.Workers, opts.Verify, keep, miss); err != nil { - return fmt.Errorf("fetch packages: %w", err) + fetch.PlanJobs(jobs, verifyPackages, keep) + } else { + packageStates, err = fetch.Many(ctx, src, jobs, opts.Workers, verifyPackages, keep, miss) + if err != nil { + return fmt.Errorf("fetch packages: %w", err) + } } // A dry run discards the indexes it staged for parsing. @@ -180,43 +455,26 @@ func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Optio _ = os.Remove(dst + fetch.StagedSuffix) } - // Stage the optional signature material so it is published together - // with the repomd.xml it signs, never beside the previous index. - var sigDsts []string - for _, extra := range []string{"repodata/repomd.xml.asc", "repodata/repomd.xml.key"} { - dst, err := fetch.LocalJoin(destDir, extra) - if err != nil { - return err - } - _, err = fetch.File(ctx, src, extra, dst, nil, true, false) - switch { - case err == nil: - sigDsts = append(sigDsts, dst) - case errors.Is(err, fetch.ErrNotFound): - // The upstream dropped the signature; drop the local copy so - // a stale signature is never served beside a new repomd.xml. - if !opts.DryRun { - fetch.RemoveStale(dst) - } - default: - return fmt.Errorf("fetch %s: %w", extra, err) - } - } - - // Promote the signatures and then repomd.xml so the published metadata - // chain is complete. - for _, dst := range sigDsts { - if err := fetch.PromoteOrDiscard(dst, opts.DryRun); err != nil { - return err - } - if _, err := os.Stat(dst); err == nil { - keep.Add(dst) - } - } - if err := fetch.PromoteOrDiscard(repomdDst, opts.DryRun); err != nil { + // Keep the previous root live until every referenced package is present + // under the configured missing-file policy. + if err := miss.Finish(); err != nil { return err } - keep.Add(repomdDst) + var withdrawn []rpmWithdrawnPackage + if root.authenticated { + withdrawn, err = withdrawMissingPackages(jobs, packageStates) + if err != nil { + _ = finishWithdrawnPackages(withdrawn, false) + return fmt.Errorf("withdraw stale packages: %w", err) + } + } + // Publish the verified root only after every referenced file is ready. + if err := publishRPMRoot(root, keep, opts.DryRun); err != nil { + return errors.Join(err, finishWithdrawnPackages(withdrawn, false)) + } + if err := finishWithdrawnPackages(withdrawn, true); err != nil { + log.WithError(err).Warn("Unable to remove withdrawn package files.") + } // Publish the traces, upstream's included, before pruning so the keep // set covers them. @@ -227,9 +485,7 @@ func syncRPM(ctx context.Context, src *fetch.Source, destDir string, opts *Optio fetch.PruneTree(destDir, keep, opts.PruneGrace, opts.DryRun) } - // The metadata is published either way; missing packages only decide - // whether the run reports itself as failed. - return miss.Finish() + return nil } // rpmDelta is one delta package referenced from prestodelta metadata. diff --git a/mirror/rpm_test.go b/mirror/rpm_test.go index 99c53d7..874dc6b 100644 --- a/mirror/rpm_test.go +++ b/mirror/rpm_test.go @@ -3,13 +3,18 @@ package mirror import ( "bytes" "context" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" "github.com/grmrgecko/repo-sync/fetch" "github.com/grmrgecko/repo-sync/internal/testrepos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // syncRPMFixture builds a served RPM repository, synchronizes it, and @@ -148,6 +153,149 @@ func TestSyncRPMDroppedSignature(t *testing.T) { } } +// TestSyncRPMMissingKeyForbidden verifies S3's 403 response for a missing +// optional key does not abort synchronization. +func TestSyncRPMMissingKeyForbidden(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + testrepos.BuildRPMRepo(t, repoDir) + + files := http.FileServer(http.Dir(www)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/repos/el9/repodata/repomd.xml.key" { + http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return + } + files.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + dest := t.TempDir() + opts := &Options{Type: RepoRPM, Destination: dest, Workers: 2} + err := syncOne(context.Background(), srv.URL+"/repos/el9", opts.Type, opts) + require.NoError(t, err) + + local := filepath.Join(dest, "repos", "el9", "repodata") + assert.FileExists(t, filepath.Join(local, "repomd.xml")) + assert.FileExists(t, filepath.Join(local, "repomd.xml.asc")) + assert.NoFileExists(t, filepath.Join(local, "repomd.xml.key")) +} + +// TestSyncRPMSignedPair verifies repository keys and keyserver retrieval, +// retries a split upstream rotation, and preserves the live pair when the +// upstream remains inconsistent. +func TestSyncRPMSignedPair(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "repos", "el9") + testrepos.BuildRPMRepo(t, repoDir) + key := testrepos.NewSigningKey(t) + key.SignRPMRepo(t, repoDir) + + repomdPath := filepath.Join(repoDir, "repodata", "repomd.xml") + firstRepomd, err := os.ReadFile(repomdPath) + require.NoError(t, err) + firstSig := key.Sign(t, firstRepomd) + + var phase atomic.Int32 + var signatureRequests atomic.Int32 + secondRepomd := append(append([]byte(nil), firstRepomd...), '\n') + secondSig := key.Sign(t, secondRepomd) + thirdRepomd := append(append([]byte(nil), secondRepomd...), '\n') + files := http.FileServer(http.Dir(www)) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/el9/repodata/repomd.xml": + switch phase.Load() { + case 0: + _, _ = w.Write(firstRepomd) + case 1: + _, _ = w.Write(secondRepomd) + default: + _, _ = w.Write(thirdRepomd) + } + case "/repos/el9/repodata/repomd.xml.asc": + request := signatureRequests.Add(1) + if phase.Load() == 1 && request > 1 { + _, _ = w.Write(secondSig) + return + } + if phase.Load() == 2 { + _, _ = w.Write(secondSig) + return + } + _, _ = w.Write(firstSig) + default: + files.ServeHTTP(w, r) + } + })) + t.Cleanup(upstream.Close) + + dest := t.TempDir() + opts := &Options{ + Type: RepoRPM, + Destination: dest, + Workers: 2, + SignatureMode: SignatureIfPresent, + } + repoURL := upstream.URL + "/repos/el9" + require.NoError(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + + local := filepath.Join(dest, "repos", "el9", "repodata") + assert.Equal(t, firstRepomd, requireReadFile(t, filepath.Join(local, "repomd.xml"))) + + // Remove the adjacent key so the rotation resolves its signer through + // the configured keyserver. + require.NoError(t, os.Remove(filepath.Join(repoDir, "repodata", "repomd.xml.key"))) + publicKey := key.PublicKey(t) + otherKey := testrepos.NewSigningKey(t) + otherPublicKey := otherKey.PublicKey(t) + unrelatedKeyserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(otherPublicKey) + })) + t.Cleanup(unrelatedKeyserver.Close) + keyserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "get", r.URL.Query().Get("op")) + assert.NotEmpty(t, r.URL.Query().Get("search")) + _, _ = w.Write(publicKey) + })) + t.Cleanup(keyserver.Close) + opts.Keyservers = []string{unrelatedKeyserver.URL, keyserver.URL} + phase.Store(1) + signatureRequests.Store(0) + require.NoError(t, syncOne(context.Background(), repoURL, opts.Type, opts)) + assert.GreaterOrEqual(t, signatureRequests.Load(), int32(2)) + assert.Equal(t, secondRepomd, requireReadFile(t, filepath.Join(local, "repomd.xml"))) + assert.Equal(t, secondSig, requireReadFile(t, filepath.Join(local, "repomd.xml.asc"))) + + phase.Store(2) + signatureRequests.Store(0) + err = syncOne(context.Background(), repoURL, opts.Type, opts) + require.Error(t, err) + assert.Equal(t, secondRepomd, requireReadFile(t, filepath.Join(local, "repomd.xml"))) + assert.Equal(t, secondSig, requireReadFile(t, filepath.Join(local, "repomd.xml.asc"))) + assert.NoFileExists(t, filepath.Join(local, "repomd.xml"+fetch.StagedSuffix)) + + // Configured keyrings pin the accepted signer, so repository and + // keyserver keys cannot override an operator-managed key. + keyPath := filepath.Join(t.TempDir(), "trusted.asc") + testrepos.WriteFile(t, keyPath, otherPublicKey) + opts.GPGKeys = []string{keyPath} + phase.Store(1) + signatureRequests.Store(1) + err = syncOne(context.Background(), repoURL, opts.Type, opts) + require.Error(t, err) + assert.Contains(t, err.Error(), "not present in the configured GPG keys") + assert.Equal(t, secondRepomd, requireReadFile(t, filepath.Join(local, "repomd.xml"))) +} + +// requireReadFile reads an asserted fixture result. +func requireReadFile(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(name) + require.NoError(t, err) + return data +} + // TestSyncRPMChecksumMismatch verifies a package whose served content does // not match the published checksum fails the synchronization. func TestSyncRPMChecksumMismatch(t *testing.T) { diff --git a/mirror/signature.go b/mirror/signature.go new file mode 100644 index 0000000..5aef211 --- /dev/null +++ b/mirror/signature.go @@ -0,0 +1,366 @@ +package mirror + +import ( + "bytes" + "context" + "crypto" + "crypto/rsa" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "os" + "strings" + + "github.com/ProtonMail/go-crypto/openpgp" + pgparmor "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/clearsign" + pgperrors "github.com/ProtonMail/go-crypto/openpgp/errors" + "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/grmrgecko/repo-sync/fetch" +) + +// SignatureMode controls OpenPGP verification of repository metadata. +type SignatureMode string + +const ( + // SignatureOff mirrors signatures without checking them. + SignatureOff SignatureMode = "off" + // SignatureIfPresent accepts unsigned repositories but verifies every + // signature the upstream publishes. + SignatureIfPresent SignatureMode = "if-present" + // SignatureRequired refuses repository metadata without a valid signature. + SignatureRequired SignatureMode = "required" +) + +// ParseSignatureMode validates a signature policy name. +func ParseSignatureMode(value string) (SignatureMode, error) { + mode := SignatureMode(strings.ToLower(strings.TrimSpace(value))) + switch mode { + case SignatureOff, SignatureIfPresent, SignatureRequired: + return mode, nil + default: + return "", fmt.Errorf("unsupported signature mode %q", value) + } +} + +// signatureVerifier verifies detached signatures with local, repository, and +// optionally keyserver-provided public keys. +type signatureVerifier struct { + keys openpgp.EntityList + keyservers []string + pinned bool +} + +// legacySignature is the parsed subset of an OpenPGP version 3 detached RSA +// signature needed by CentOS 7-era repositories. +type legacySignature struct { + issuer uint64 + hash crypto.Hash + hashTag [2]byte + hashed []byte + signature []byte +} + +// newSignatureVerifier loads the operator-configured public keys. +func newSignatureVerifier(opts *Options) (*signatureVerifier, error) { + v := &signatureVerifier{keyservers: opts.Keyservers, pinned: len(opts.GPGKeys) > 0 || len(opts.GPGKeyData) > 0} + for _, data := range opts.GPGKeyData { + keys, err := readPublicKeys(data) + if err != nil { + return nil, fmt.Errorf("parse configured GPG key: %w", err) + } + v.keys = append(v.keys, keys...) + } + for _, name := range opts.GPGKeys { + data, err := os.ReadFile(name) + if err != nil { + return nil, fmt.Errorf("read GPG key %s: %w", name, err) + } + keys, err := readPublicKeys(data) + if err != nil { + return nil, fmt.Errorf("parse GPG key %s: %w", name, err) + } + v.keys = append(v.keys, keys...) + } + return v, nil +} + +// readPublicKeys accepts armored and binary OpenPGP public keyrings. +func readPublicKeys(data []byte) (openpgp.EntityList, error) { + if bytes.HasPrefix(bytes.TrimSpace(data), []byte("-----BEGIN PGP")) { + return openpgp.ReadArmoredKeyRing(bytes.NewReader(data)) + } + return openpgp.ReadKeyRing(bytes.NewReader(data)) +} + +// decodedSignature returns the packet stream from an armored or binary +// detached signature. +func decodedSignature(data []byte) ([]byte, error) { + if !bytes.HasPrefix(bytes.TrimSpace(data), []byte("-----BEGIN PGP")) { + return data, nil + } + block, err := pgparmor.Decode(bytes.NewReader(data)) + if err != nil { + return nil, err + } + if block.Type != openpgp.SignatureType { + return nil, fmt.Errorf("armor contains %q instead of a signature", block.Type) + } + return io.ReadAll(block.Body) +} + +// parseLegacySignature parses the version 3 signature packet format emitted +// by GnuPG v1. Modern signature packets return ok=false for normal handling. +func parseLegacySignature(data []byte) (sig legacySignature, ok bool, err error) { + opaque, err := packet.NewOpaqueReader(bytes.NewReader(data)).Next() + if err != nil { + return sig, false, err + } + if opaque.Tag != 2 || len(opaque.Contents) == 0 || opaque.Contents[0] != 3 { + return sig, false, nil + } + body := opaque.Contents + if len(body) < 21 || body[1] != 5 { + return sig, true, errors.New("invalid OpenPGP version 3 signature packet") + } + if body[2] != byte(packet.SigTypeBinary) { + return sig, true, fmt.Errorf("unsupported OpenPGP version 3 signature type %d", body[2]) + } + if body[15] != byte(packet.PubKeyAlgoRSA) && body[15] != byte(packet.PubKeyAlgoRSASignOnly) { + return sig, true, fmt.Errorf("unsupported OpenPGP version 3 public key algorithm %d", body[15]) + } + hash, supported := openpgp.HashIdToHash(body[16]) + if !supported || !hash.Available() { + return sig, true, fmt.Errorf("unsupported OpenPGP version 3 hash algorithm %d", body[16]) + } + bits := int(binary.BigEndian.Uint16(body[19:21])) + bytesLen := (bits + 7) / 8 + if bytesLen == 0 || len(body) != 21+bytesLen { + return sig, true, errors.New("invalid OpenPGP version 3 RSA signature") + } + sig.issuer = binary.BigEndian.Uint64(body[7:15]) + sig.hash = hash + copy(sig.hashTag[:], body[17:19]) + sig.hashed = append([]byte(nil), body[2:7]...) + sig.signature = append([]byte(nil), body[21:]...) + return sig, true, nil +} + +// verifyLegacySignature verifies a version 3 binary-document RSA signature. +func verifyLegacySignature(keys openpgp.EntityList, data []byte, sig legacySignature) (*openpgp.Entity, error) { + h := sig.hash.New() + if _, err := h.Write(data); err != nil { + return nil, err + } + if _, err := h.Write(sig.hashed); err != nil { + return nil, err + } + digest := h.Sum(nil) + if !bytes.Equal(digest[:2], sig.hashTag[:]) { + return nil, errors.New("OpenPGP signature hash tag does not match") + } + for _, entity := range keys { + publicKeys := []*packet.PublicKey{entity.PrimaryKey} + for _, subkey := range entity.Subkeys { + publicKeys = append(publicKeys, subkey.PublicKey) + } + for _, key := range publicKeys { + if key == nil || key.KeyId != sig.issuer { + continue + } + pub, ok := key.PublicKey.(*rsa.PublicKey) + if !ok { + continue + } + signature := sig.signature + if size := pub.Size(); len(signature) < size { + padded := make([]byte, size) + copy(padded[size-len(signature):], signature) + signature = padded + } + if err := rsa.VerifyPKCS1v15(pub, sig.hash, digest, signature); err == nil { + return entity, nil + } + } + } + return nil, pgperrors.ErrUnknownIssuer +} + +// signatureLookup identifies the signer requested by the first signature +// packet. Full fingerprints are preferred over 64-bit key IDs. +func signatureLookup(data []byte) (string, error) { + decoded, err := decodedSignature(data) + if err != nil { + return "", err + } + legacy, ok, err := parseLegacySignature(decoded) + if err != nil { + return "", err + } + if ok { + return fmt.Sprintf("%016X", legacy.issuer), nil + } + p, err := packet.NewReader(bytes.NewReader(decoded)).Next() + if err != nil { + return "", err + } + sig, ok := p.(*packet.Signature) + if !ok { + return "", errors.New("detached signature contains no signature packet") + } + if len(sig.IssuerFingerprint) > 0 { + return strings.ToUpper(hex.EncodeToString(sig.IssuerFingerprint)), nil + } + if sig.IssuerKeyId != nil { + return fmt.Sprintf("%016X", *sig.IssuerKeyId), nil + } + return "", errors.New("detached signature has no issuer") +} + +// keyserverLookupURL builds a Hockeypuck-compatible exact key lookup. +func keyserverLookupURL(base, issuer string) (string, error) { + u, err := url.Parse(base) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", fmt.Errorf("invalid keyserver URL %q", base) + } + u.Path = strings.TrimRight(u.Path, "/") + "/pks/lookup" + q := u.Query() + q.Set("op", "get") + q.Set("options", "mr") + q.Set("search", "0x"+issuer) + u.RawQuery = q.Encode() + return u.String(), nil +} + +// retrieveSigner asks configured keyservers for the signature issuer and +// ignores responses whose keys do not verify the pair. +func (v *signatureVerifier) retrieveSigner( + ctx context.Context, + signature []byte, + base openpgp.EntityList, + verify func(openpgp.EntityList) (*openpgp.Entity, error), +) (*openpgp.Entity, error) { + issuer, err := signatureLookup(signature) + if err != nil { + return nil, err + } + var errs []error + for _, server := range v.keyservers { + lookup, err := keyserverLookupURL(server, issuer) + if err != nil { + errs = append(errs, err) + continue + } + data, err := fetch.ReadURL(ctx, lookup) + if err != nil { + errs = append(errs, fmt.Errorf("fetch key %s from %s: %w", issuer, server, err)) + continue + } + keys, err := readPublicKeys(data) + if err != nil { + errs = append(errs, fmt.Errorf("parse key %s from %s: %w", issuer, server, err)) + continue + } + candidate := append(append(openpgp.EntityList(nil), base...), keys...) + signer, err := verify(candidate) + if err != nil { + errs = append(errs, fmt.Errorf("verify key %s from %s: %w", issuer, server, err)) + continue + } + v.keys = append(v.keys, keys...) + return signer, nil + } + if len(errs) == 0 { + return nil, fmt.Errorf("signature issuer %s is unknown; configure a GPG key or keyserver", issuer) + } + return nil, errors.Join(errs...) +} + +// verifyDetachedBytes verifies a detached signature over data and returns the +// matching primary-key fingerprint. +func (v *signatureVerifier) verifyDetachedBytes(ctx context.Context, data, signature, repoKey []byte) (string, error) { + keys := append(openpgp.EntityList(nil), v.keys...) + if len(repoKey) > 0 && !v.pinned { + repoKeys, err := readPublicKeys(repoKey) + if err != nil { + return "", fmt.Errorf("parse repository GPG key: %w", err) + } + keys = append(keys, repoKeys...) + } + + decoded, err := decodedSignature(signature) + if err != nil { + return "", err + } + legacy, legacyOK, err := parseLegacySignature(decoded) + if err != nil { + return "", err + } + verify := func(keyring openpgp.EntityList) (*openpgp.Entity, error) { + if legacyOK { + return verifyLegacySignature(keyring, data, legacy) + } + _, signer, err := openpgp.VerifyDetachedSignature(keyring, bytes.NewReader(data), bytes.NewReader(decoded), nil) + return signer, err + } + + signer, err := verify(keys) + if errors.Is(err, pgperrors.ErrUnknownIssuer) && len(v.keyservers) > 0 && !v.pinned { + signer, err = v.retrieveSigner(ctx, signature, keys, verify) + } + if err != nil { + if errors.Is(err, pgperrors.ErrUnknownIssuer) { + issuer, lookupErr := signatureLookup(signature) + if lookupErr == nil { + if v.pinned { + return "", fmt.Errorf("signature issuer %s is not present in the configured GPG keys", issuer) + } + return "", fmt.Errorf("signature issuer %s is unknown; configure a GPG key or keyserver", issuer) + } + } + return "", err + } + return strings.ToUpper(hex.EncodeToString(signer.PrimaryKey.Fingerprint)), nil +} + +// verifyDetached verifies sigPath against the exact bytes at dataPath and +// returns the matching primary-key fingerprint. +func (v *signatureVerifier) verifyDetached(ctx context.Context, dataPath, sigPath, repoKeyPath string) (string, error) { + data, err := os.ReadFile(dataPath) + if err != nil { + return "", err + } + signature, err := os.ReadFile(sigPath) + if err != nil { + return "", err + } + var repoKey []byte + if repoKeyPath != "" { + repoKey, err = os.ReadFile(repoKeyPath) + if err != nil { + return "", err + } + } + return v.verifyDetachedBytes(ctx, data, signature, repoKey) +} + +// verifyClearsigned verifies an OpenPGP cleartext message and returns its +// authenticated plaintext and signer fingerprint. +func (v *signatureVerifier) verifyClearsigned(ctx context.Context, data []byte) ([]byte, string, error) { + block, rest := clearsign.Decode(data) + if block == nil || len(bytes.TrimSpace(rest)) != 0 { + return nil, "", errors.New("invalid OpenPGP clearsigned message") + } + signature, err := io.ReadAll(block.ArmoredSignature.Body) + if err != nil { + return nil, "", err + } + fingerprint, err := v.verifyDetachedBytes(ctx, block.Bytes, signature, nil) + if err != nil { + return nil, "", err + } + return block.Plaintext, fingerprint, nil +} diff --git a/mirror/signature_test.go b/mirror/signature_test.go new file mode 100644 index 0000000..9abebf3 --- /dev/null +++ b/mirror/signature_test.go @@ -0,0 +1,80 @@ +package mirror + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "math/big" + "os" + "path/filepath" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestVerifyLegacyDetachedSignature locks the OpenPGP version 3 RSA packet +// shape used by GnuPG v1 to sign archived CentOS 7 repomd.xml files. +func TestVerifyLegacyDetachedSignature(t *testing.T) { + entity, err := openpgp.NewEntity("CentOS fixture", "", "security@example.com", nil) + require.NoError(t, err) + privateKey, ok := entity.PrivateKey.PrivateKey.(*rsa.PrivateKey) + require.True(t, ok) + + data := []byte("signed repository metadata\n") + hashed := make([]byte, 5) + hashed[0] = byte(packet.SigTypeBinary) + binary.BigEndian.PutUint32(hashed[1:], 1678292750) + h := crypto.SHA256.New() + _, _ = h.Write(data) + _, _ = h.Write(hashed) + digest := h.Sum(nil) + rsaSignature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, digest) + require.NoError(t, err) + mpi := new(big.Int).SetBytes(rsaSignature) + mpiBytes := mpi.Bytes() + + body := []byte{3, 5} + body = append(body, hashed...) + body = binary.BigEndian.AppendUint64(body, entity.PrimaryKey.KeyId) + body = append(body, byte(packet.PubKeyAlgoRSA), 8, digest[0], digest[1]) + body = binary.BigEndian.AppendUint16(body, uint16(mpi.BitLen())) + body = append(body, mpiBytes...) + var packetData bytes.Buffer + require.NoError(t, (&packet.OpaquePacket{Tag: 2, Contents: body}).Serialize(&packetData)) + var signature bytes.Buffer + armoredSignature, err := armor.Encode(&signature, openpgp.SignatureType, nil) + require.NoError(t, err) + _, err = armoredSignature.Write(packetData.Bytes()) + require.NoError(t, err) + require.NoError(t, armoredSignature.Close()) + + var publicKey bytes.Buffer + armoredKey, err := armor.Encode(&publicKey, openpgp.PublicKeyType, nil) + require.NoError(t, err) + require.NoError(t, entity.Serialize(armoredKey)) + require.NoError(t, armoredKey.Close()) + + dir := t.TempDir() + dataPath := filepath.Join(dir, "repomd.xml") + sigPath := dataPath + ".asc" + keyPath := dataPath + ".key" + require.NoError(t, os.WriteFile(dataPath, data, 0644)) + require.NoError(t, os.WriteFile(sigPath, signature.Bytes(), 0644)) + require.NoError(t, os.WriteFile(keyPath, publicKey.Bytes(), 0644)) + + verifier := &signatureVerifier{} + fingerprint, err := verifier.verifyDetached(context.Background(), dataPath, sigPath, keyPath) + require.NoError(t, err) + assert.NotEmpty(t, fingerprint) + + require.NoError(t, os.WriteFile(dataPath, []byte("modified repository metadata\n"), 0644)) + _, err = verifier.verifyDetached(context.Background(), dataPath, sigPath, keyPath) + assert.Error(t, err) +} diff --git a/server/crawl_loop.go b/server/crawl_loop.go index ae06d40..3e9123e 100644 --- a/server/crawl_loop.go +++ b/server/crawl_loop.go @@ -2,7 +2,10 @@ package server import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" + "io" "os" "strings" "sync" @@ -156,11 +159,153 @@ func crawlResource(ctx context.Context, res resource) error { } defer lockResource(res.Key)() - err := dispatchCrawl(ctx, res) + return crawlResourceLocked(ctx, res) +} + +// crawlResourceLocked synchronizes a resource whose repository locks are +// already held. +func crawlResourceLocked(ctx context.Context, res resource) error { + settings := signatureSettings{mode: mirror.SignatureOff} + protected := protectedRepositoryKind(res.Kind) && cfg.C.Crawler.SignatureMode != string(mirror.SignatureOff) + var err error + if protected { + settings, err = loadSignatureSettings(cfg.C) + } + if err == nil { + err = dispatchCrawl(ctx, res, settings) + } state.S.MarkCrawled(res.Key, time.Now(), err) + if err == nil && protected { + state.S.MarkSignaturePolicy(res.Key, settings.policy) + } return err } +// crawlProtectedRepository verifies a repository synchronously before its +// entry point is served under an active signature policy. +func crawlProtectedRepository(ctx context.Context, res resource, artifactsPresent bool, settings signatureSettings, settingsErr error) error { + acquireCrawlSlot() + defer releaseCrawlSlot() + if res.Kind == string(mirror.RepoDeb) && res.Root != res.Path { + defer lockResource("deb-root:" + res.Root)() + } + defer lockResource(res.Key)() + if settingsErr != nil { + state.S.MarkCrawled(res.Key, time.Now(), settingsErr) + return settingsErr + } + if entry, ok := state.S.Entry(res.Key); ok && entry.SignaturePolicy == settings.policy && entry.LastError == "" { + if artifactsPresent { + return nil + } + } + err := dispatchCrawl(ctx, res, settings) + state.S.MarkCrawled(res.Key, time.Now(), err) + if err == nil { + state.S.MarkSignaturePolicy(res.Key, settings.policy) + } + return err +} + +// signaturePolicyArtifactsPresent reports whether a repository still has the +// entry-point files required by its active signature policy. +func signaturePolicyArtifactsPresent(conf *cfg.Config, res resource, mode mirror.SignatureMode) bool { + repo, err := fetch.LocalJoin(conf.OnlineDomain().Root, res.Path) + if err != nil { + return false + } + switch mirror.RepoType(res.Kind) { + case mirror.RepoRPM: + repomd, err := fetch.LocalJoin(repo, "repodata/repomd.xml") + if err != nil { + return false + } + if _, exists := regularFile(repomd); !exists { + return false + } + if mode != mirror.SignatureRequired { + return true + } + signature, err := fetch.LocalJoin(repo, "repodata/repomd.xml.asc") + if err != nil { + return false + } + _, exists := regularFile(signature) + return exists + case mirror.RepoDeb: + inRelease, _ := fetch.LocalJoin(repo, "InRelease") + if _, exists := regularFile(inRelease); exists { + return true + } + release, _ := fetch.LocalJoin(repo, "Release") + if _, exists := regularFile(release); !exists { + return false + } + if mode != mirror.SignatureRequired { + return true + } + signature, _ := fetch.LocalJoin(repo, "Release.gpg") + _, exists := regularFile(signature) + return exists + case mirror.RepoArch: + if !strings.HasSuffix(res.ReqPath, ".db") { + return false + } + database, err := fetch.LocalJoin(conf.OnlineDomain().Root, res.ReqPath) + if err != nil { + return false + } + _, exists := regularFile(database) + return exists + default: + return false + } +} + +// signatureSettings is one immutable verification configuration used for +// both a crawl and its persisted policy identity. +type signatureSettings struct { + mode mirror.SignatureMode + keyData [][]byte + keyservers []string + policy string +} + +// loadSignatureSettings snapshots key contents before a crawl so a reload +// cannot change verification inputs halfway through it. +func loadSignatureSettings(conf *cfg.Config) (signatureSettings, error) { + mode, err := mirror.ParseSignatureMode(conf.Crawler.SignatureMode) + if err != nil { + return signatureSettings{}, err + } + settings := signatureSettings{ + mode: mode, + keyservers: append([]string(nil), conf.Crawler.Keyservers...), + } + h := sha256.New() + _, _ = io.WriteString(h, "repository-openpgp-v1\x00"+string(mode)+"\x00") + for _, name := range conf.Crawler.GPGKeys { + _, _ = io.WriteString(h, name+"\x00") + data, err := os.ReadFile(name) + if err != nil { + return signatureSettings{}, fmt.Errorf("read GPG key %s: %w", name, err) + } + settings.keyData = append(settings.keyData, data) + _, _ = h.Write(data) + _, _ = io.WriteString(h, "\x00") + } + for _, server := range settings.keyservers { + _, _ = io.WriteString(h, server+"\x00") + } + settings.policy = hex.EncodeToString(h.Sum(nil)) + return settings, nil +} + +// protectedRepositoryKind reports formats covered by the OpenPGP policy. +func protectedRepositoryKind(kind string) bool { + return kind == string(mirror.RepoRPM) || kind == string(mirror.RepoDeb) || kind == string(mirror.RepoArch) +} + // pathBelow reports whether a request path sits strictly below a base path. func pathBelow(p, base string) bool { if p == base { @@ -217,7 +362,7 @@ func traceInfo(t cfg.TraceConfig) mirror.TraceInfo { // dispatchCrawl runs the format-specific synchronization for a resource // into the online tree. -func dispatchCrawl(ctx context.Context, res resource) error { +func dispatchCrawl(ctx context.Context, res resource, signatures signatureSettings) error { online := cfg.C.OnlineDomain() mount, ok := cfg.C.MountFor(res.Path) if !ok { @@ -237,7 +382,6 @@ func dispatchCrawl(ctx context.Context, res resource) error { if err != nil { return err } - // Generic files never reach the crawl path: they refresh on demand at // serve time and expire through the failure cache and state eviction. typ := mirror.RepoType(res.Kind) @@ -248,6 +392,9 @@ func dispatchCrawl(ctx context.Context, res resource) error { PruneGrace: cfg.C.Crawler.PruneGrace, Missing: missing, MissingRetries: cfg.C.Crawler.MissingRetries, + SignatureMode: signatures.mode, + GPGKeyData: signatures.keyData, + Keyservers: signatures.keyservers, Trace: cfg.C.Trace.Enabled, TraceInfo: traceInfo(cfg.C.Trace), } diff --git a/server/serve.go b/server/serve.go index 68e7cd7..d5840c2 100644 --- a/server/serve.go +++ b/server/serve.go @@ -19,6 +19,7 @@ import ( cfg "github.com/grmrgecko/repo-sync/config" "github.com/grmrgecko/repo-sync/fetch" + "github.com/grmrgecko/repo-sync/mirror" "github.com/grmrgecko/repo-sync/state" log "github.com/sirupsen/logrus" ) @@ -209,9 +210,39 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi if res.Kind == kindGeneric { // Files under a registered repository keep it alive and are // fetched on demand until its crawl completes. - if state.S.TouchRepoMembers(reqPath, now) { + members := state.S.TouchRepoMembers(reqPath, now) + if len(members) > 0 { if _, exists := regularFile(local); !exists { - if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + var protectedKey string + var protectedEntry state.Entry + for key, entry := range members { + if protectedRepositoryKind(entry.Kind) && cfg.C.Crawler.SignatureMode != string(mirror.SignatureOff) { + protectedKey = key + protectedEntry = entry + break + } + } + var err error + if protectedKey != "" { + settings, settingsErr := loadSignatureSettings(cfg.C) + protectedRes := resource{ + Kind: protectedEntry.Kind, + Key: protectedKey, + Path: protectedEntry.Path, + Root: protectedEntry.Root, + ReqPath: protectedEntry.Path, + } + err = crawlProtectedRepository(r.Context(), protectedRes, false, settings, settingsErr) + if err == nil { + _, exists = regularFile(local) + if !exists { + err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound) + } + } + } else { + err = fetchUpstream(r.Context(), domain.Root, reqPath, false) + } + if err != nil { writeFetchError(w, err) return } @@ -233,8 +264,30 @@ func handleOnline(w http.ResponseWriter, r *http.Request, domain cfg.DomainConfi entry, tracked := state.S.MarkRequested(res.Kind, res.Key, res.Path, res.Root, now) _, exists := regularFile(local) needCrawl := entry.LastCrawled.IsZero() || (!exists && entry.LastError != "") - if !exists { - if err := fetchUpstream(r.Context(), domain.Root, reqPath, false); err != nil { + conf := cfg.C + protected := protectedRepositoryKind(res.Kind) && conf.Crawler.SignatureMode != string(mirror.SignatureOff) + var signatureSettings signatureSettings + var signatureSettingsErr error + if protected { + signatureSettings, signatureSettingsErr = loadSignatureSettings(conf) + } + artifactsPresent := protected && signaturePolicyArtifactsPresent(conf, res, signatureSettings.mode) + needsSignatureCheck := protected && (signatureSettingsErr != nil || entry.SignaturePolicy != signatureSettings.policy || !artifactsPresent) + if !exists || needsSignatureCheck { + var err error + if protected { + // A protected entry point cannot be served until the crawl has + // verified and published its metadata generation. + err = crawlProtectedRepository(r.Context(), res, artifactsPresent, signatureSettings, signatureSettingsErr) + needCrawl = false + _, exists = regularFile(local) + if err == nil && !exists { + err = fmt.Errorf("fetch %s: %w", reqPath, fetch.ErrNotFound) + } + } else { + err = fetchUpstream(r.Context(), domain.Root, reqPath, false) + } + if err != nil { // A path whose first contact failed was never shown to be a // repository, so the registration this request created is // dropped and the scheduler never crawls it. Registrations an diff --git a/server/serve_test.go b/server/serve_test.go index 30c56aa..7c4ce4d 100644 --- a/server/serve_test.go +++ b/server/serve_test.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "fmt" "io" "net/http" @@ -18,6 +19,8 @@ import ( "github.com/grmrgecko/repo-sync/internal/testrepos" "github.com/grmrgecko/repo-sync/mirror" "github.com/grmrgecko/repo-sync/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // serverFixture builds an upstream with one repository of each type plus a @@ -465,6 +468,103 @@ func TestServeSiblingEntryPointMiss(t *testing.T) { } } +// TestServeRejectsUnverifiedRPMEntryPoint verifies the first request cannot +// serve cached repomd.xml before its required signature has been checked. +func TestServeRejectsUnverifiedRPMEntryPoint(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "repo") + testrepos.BuildRPMRepo(t, repoDir) + key := testrepos.NewSigningKey(t) + key.SignRPMRepo(t, repoDir) + repomd := filepath.Join(repoDir, "repodata", "repomd.xml") + data, err := os.ReadFile(repomd) + if err != nil { + t.Fatal(err) + } + testrepos.WriteFile(t, repomd, append(data, '\n')) + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + local := filepath.Join(onlineRoot, "repo", "repodata", "repomd.xml") + testrepos.WriteFile(t, local, []byte("unverified cached metadata")) + confDir := t.TempDir() + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - {domain: 127.0.0.1, role: online, root: %s} +mounts: + - {path: /, upstream: %s} +crawler: + signature_mode: required +`, confDir, onlineRoot, upstream.URL))) + if err := cfg.Init(confPath); err != nil { + t.Fatal(err) + } + if err := state.Load(); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + resp, _ := get(t, srv, "", "/repo/repodata/repomd.xml") + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("repomd status = %d, want 502", resp.StatusCode) + } + got, err := os.ReadFile(local) + if err != nil { + t.Fatal(err) + } + if string(got) != "unverified cached metadata" { + t.Error("failed verification replaced the cached repomd.xml") + } +} + +// TestServeRejectsUnverifiedDebEntryPoint verifies a cached InRelease cannot +// bypass the synchronous signature gate on its first request. +func TestServeRejectsUnverifiedDebEntryPoint(t *testing.T) { + www := t.TempDir() + repoDir := filepath.Join(www, "debian") + testrepos.BuildDebRepo(t, repoDir) + key := testrepos.NewSigningKey(t) + key.SignDebRepo(t, repoDir) + inRelease := filepath.Join(repoDir, "dists", "test", "InRelease") + signedRelease, err := os.ReadFile(inRelease) + require.NoError(t, err) + tampered := bytes.Replace(signedRelease, []byte("Origin: Test"), []byte("Origin: Pest"), 1) + require.NoError(t, os.WriteFile(inRelease, tampered, 0644)) + upstream := testrepos.ServeDir(t, www) + + onlineRoot := t.TempDir() + local := filepath.Join(onlineRoot, "debian", "dists", "test", "InRelease") + testrepos.WriteFile(t, local, []byte("unverified cached metadata")) + confDir := t.TempDir() + keyPath := filepath.Join(confDir, "repository.asc") + testrepos.WriteFile(t, keyPath, key.PublicKey(t)) + confPath := filepath.Join(confDir, "config.yaml") + testrepos.WriteFile(t, confPath, []byte(fmt.Sprintf(` +state_path: %s/state.yaml +domains: + - {domain: 127.0.0.1, role: online, root: %s} +mounts: + - {path: /, upstream: %s} +crawler: + signature_mode: required + gpg_keys: [%s] + keyservers: [] +`, confDir, onlineRoot, upstream.URL, keyPath))) + require.NoError(t, cfg.Init(confPath)) + require.NoError(t, state.Load()) + srv := httptest.NewServer(Handler()) + t.Cleanup(srv.Close) + + resp, _ := get(t, srv, "", "/debian/dists/test/InRelease") + assert.Equal(t, http.StatusBadGateway, resp.StatusCode) + got, err := os.ReadFile(local) + require.NoError(t, err) + assert.Equal(t, []byte("unverified cached metadata"), got, "failed verification must retain the cached InRelease") +} + // TestServeUnresolvedEntryPoint verifies a path that only looks like a // repository entry point is not left registered for the scheduler to crawl. func TestServeUnresolvedEntryPoint(t *testing.T) { diff --git a/state/state.go b/state/state.go index 010c614..0865766 100644 --- a/state/state.go +++ b/state/state.go @@ -32,6 +32,9 @@ type Entry struct { NextCrawl time.Time `yaml:"next_crawl,omitempty"` LastSeenInInventory time.Time `yaml:"last_seen_in_inventory,omitempty"` LastError string `yaml:"last_error,omitempty"` + // SignaturePolicy records the verification settings applied by the last + // successful crawl. + SignaturePolicy string `yaml:"signature_policy,omitempty"` } // file is the on-disk YAML layout. @@ -137,6 +140,19 @@ func (s *Store) MarkCrawled(key string, now time.Time, crawlErr error) { s.dirty = true } +// MarkSignaturePolicy records the signature policy completed by a successful +// repository crawl. +func (s *Store) MarkSignaturePolicy(key, policy string) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.file.Entries[key] + if e == nil || e.SignaturePolicy == policy { + return + } + e.SignaturePolicy = policy + s.dirty = true +} + // Delete removes a resource from tracking. func (s *Store) Delete(key string) { s.mu.Lock() @@ -169,14 +185,13 @@ func (s *Store) Snapshot() map[string]Entry { return out } -// TouchRepoMembers refreshes the last-requested time of every repository -// whose root covers a request path, keeping repositories alive while their -// files are fetched. It reports whether any repository matched. -func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool { +// TouchRepoMembers refreshes every repository whose root covers a request +// path and returns copies keyed by their state keys. +func (s *Store) TouchRepoMembers(reqPath string, now time.Time) map[string]Entry { s.mu.Lock() defer s.mu.Unlock() - matched := false - for _, e := range s.file.Entries { + matched := map[string]Entry{} + for key, e := range s.file.Entries { if e.Kind == "generic" || e.Root == "" { continue } @@ -186,7 +201,7 @@ func (s *Store) TouchRepoMembers(reqPath string, now time.Time) bool { continue } e.LastRequested = now - matched = true + matched[key] = *e s.dirty = true } return matched diff --git a/sync_cmd.go b/sync_cmd.go index 883b7df..a28835f 100644 --- a/sync_cmd.go +++ b/sync_cmd.go @@ -33,6 +33,10 @@ type SyncArgs struct { Verify bool `help:"Re-verify checksums of files that already exist locally."` Prune bool `help:"Delete local files that are no longer part of the repository."` + SignatureMode string `help:"OpenPGP metadata signature policy: off, if-present, or required; defaults to the configured crawler policy." enum:",off,if-present,required" default:""` + GPGKey []string `help:"Public keyring file used for metadata signature checks; repeatable." type:"existingfile"` + Keyserver []string `help:"OpenPGP keyserver used to retrieve unknown signature issuers; repeatable and defaults to the configured crawler keyservers."` + Exclude []string `help:"Glob of directory or file names discovery never crawls; repeatable, and matched against the path below the crawled URL when it contains a slash, which a leading slash anchors to that URL (e.g. sles, 'yum/docker*', /docker)."` IncludeFile []string `help:"Glob of loose files found outside repositories that discovery mirrors as well; repeatable, and matched by path on the same rules as --exclude (e.g. '*.rpm', 'RPM-GPG-KEY-*', '/*.rpm')."` @@ -103,10 +107,29 @@ func (s *SyncArgs) options(types []mirror.RepoType) (*mirror.Options, error) { if s.DiscoverCache < 0 { s.DiscoverCache = cfg.C.Crawler.DiscoverCache } + if s.SignatureMode == "" { + s.SignatureMode = cfg.C.Crawler.SignatureMode + } + if len(s.GPGKey) == 0 { + s.GPGKey = append([]string(nil), cfg.C.Crawler.GPGKeys...) + } + if len(s.Keyserver) == 0 { + s.Keyserver = append([]string(nil), cfg.C.Crawler.Keyservers...) + } missing, err := fetch.ParseMissingMode(s.Missing) if err != nil { return nil, err } + signatures, err := mirror.ParseSignatureMode(s.SignatureMode) + if err != nil { + return nil, err + } + for _, raw := range s.Keyserver { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("invalid keyserver URL %q", raw) + } + } opts := &mirror.Options{ Types: types, URLs: urls, @@ -120,6 +143,9 @@ func (s *SyncArgs) options(types []mirror.RepoType) (*mirror.Options, error) { IncludeFiles: s.IncludeFile, Workers: s.Workers, Verify: s.Verify, + SignatureMode: signatures, + GPGKeys: s.GPGKey, + Keyservers: s.Keyserver, Prune: s.Prune, PruneGrace: s.PruneGrace, DryRun: s.DryRun,