// Package mirror synchronizes remote package repositories into a local // directory tree, preserving the upstream layout so the result can be // served directly to package managers. package mirror import ( "net/url" "path" "strings" "time" "github.com/grmrgecko/repo-sync/fetch" ) // RepoType identifies the repository format being synchronized. type RepoType string const ( // RepoRPM synchronizes yum/dnf style repositories. RepoRPM RepoType = "rpm" // RepoDeb synchronizes apt style repositories. RepoDeb RepoType = "deb" // RepoArch synchronizes pacman style repositories. RepoArch RepoType = "arch" // RepoApk synchronizes Alpine apk style repositories. RepoApk RepoType = "apk" ) // AllTypes lists every supported repository format, in the order a // directory's metadata is checked against them. RPM and deb come first as // their markers are the most specific. var AllTypes = []RepoType{RepoRPM, RepoDeb, RepoArch, RepoApk} // TypeNames lists the supported format names, for user facing messages. func TypeNames() []string { names := make([]string, 0, len(AllTypes)) for _, typ := range AllTypes { names = append(names, string(typ)) } return names } // Options configures a synchronization run. type Options struct { // Type selects the repository format. It is empty for a run whose // repositories are identified by their metadata rather than named on the // command line, and is pinned to the repository's own format before that // repository is synchronized. Type RepoType // Types limits the formats a run identifies repositories as. An empty // list accepts every supported format. Types []RepoType // URLs lists the repository or mirrorlist URLs to synchronize. URLs []string // Destination is the local directory repositories are placed under. Destination string // Trim drops this many leading URL path components when building the // destination path. Trim int // Flat places repositories directly in Destination without copying the // URL path. Flat bool // Discover crawls each URL's directory listings for repositories. Discover bool // DiscoverDepth limits how deep discovery crawls below each URL. DiscoverDepth int // DiscoverCache is how long a crawl's results are reused before the // tree is scanned again. Zero scans on every run. DiscoverCache time.Duration // Exclude lists globs matched against the names of entries found while // crawling; a match is neither crawled nor mirrored. A pattern // containing a slash is matched against the entry's path below the // crawled URL instead of its name, and a leading slash anchors it to // that URL. Exclude []string // IncludeFiles lists globs matched against the names of files found // outside of any repository while crawling; a match is mirrored as a // plain file. Patterns match by path the same way Exclude does, so a // leading slash limits one to the crawled directory itself. Files // inside a repository come from its metadata and are not matched here. IncludeFiles []string // Workers is the number of concurrent download workers. Workers int // Verify re-verifies checksums of existing local files. Verify bool // Prune deletes local files no longer part of the repository. Prune bool // PruneGrace defers pruning a file until it has been unreferenced for // this long across runs. PruneGrace time.Duration // DryRun reports what would change without downloading packages or // altering the published repository. DryRun bool // Missing selects what happens when the upstream does not serve a // package the repository metadata lists. The zero value is the strict // mode, failing the repository at the first such file. Missing fetch.MissingMode // MissingRetries is how many consecutive runs must miss a file before // the retry mode accepts it as gone and stops failing the run over it. MissingRetries int // Trace writes a trace file describing this mirror into every // repository synchronized, and mirrors the traces upstream publishes // beside it. Trace bool // TraceInfo describes the mirror recorded in that trace file. TraceInfo TraceInfo // destBase, when set by the mirror server, pins the repository root // directory directly instead of deriving it from the URL path. destBase string // Components limits deb synchronization to these components. Components []string // Architectures limits deb synchronization to these architectures. Architectures []string } // discoverTypes returns the formats a run identifies repositories as: the // formats it was limited to, the single format it was given, or every // supported format when it named none. func (o *Options) discoverTypes() []RepoType { if len(o.Types) > 0 { return o.Types } if o.Type != "" { return []RepoType{o.Type} } return AllTypes } // newMissing starts tracking the packages an upstream does not serve for // one repository, recording them in dir. That is the repository's own // directory, and for apt the suite directory rather than the archive root, // so suites sharing a pool keep their own records. func (o *Options) newMissing(dir string) *fetch.Missing { return fetch.NewMissing(dir, o.Missing, o.MissingRetries, o.DryRun) } // repoDest maps a repository URL onto the destination directory according // to the configured trimming rules. func (o *Options) repoDest(repoURL string) (string, error) { if o.destBase != "" { return o.destBase, nil } if o.Flat { return o.Destination, nil } u, err := url.Parse(repoURL) if err != nil { return "", err } var segs []string for _, seg := range strings.Split(u.Path, "/") { if seg != "" { segs = append(segs, seg) } } if o.Trim >= len(segs) { return o.Destination, nil } return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...)) }