// Package mirror synchronizes remote package repositories into a local // directory tree, preserving the upstream layout so the result can be // served directly to package managers. package mirror import ( "net/url" "path" "strings" "time" "github.com/grmrgecko/repo-sync/fetch" ) // RepoType identifies the repository format being synchronized. type RepoType string const ( // RepoRPM synchronizes yum/dnf style repositories. RepoRPM RepoType = "rpm" // RepoDeb synchronizes apt style repositories. RepoDeb RepoType = "deb" // RepoArch synchronizes pacman style repositories. RepoArch RepoType = "arch" // RepoApk synchronizes Alpine apk style repositories. RepoApk RepoType = "apk" ) // AllTypes lists every supported repository format, in the order a // directory's metadata is checked against them. RPM and deb come first as // their markers are the most specific. var AllTypes = []RepoType{RepoRPM, RepoDeb, RepoArch, RepoApk} // TypeNames lists the supported format names, for user facing messages. func TypeNames() []string { names := make([]string, 0, len(AllTypes)) for _, typ := range AllTypes { names = append(names, string(typ)) } return names } // Options configures a synchronization run. type Options struct { // Type selects the repository format. It is empty for a run whose // repositories are identified by their metadata rather than named on the // command line, and is pinned to the repository's own format before that // repository is synchronized. Type RepoType // Types limits the formats a run identifies repositories as. An empty // list accepts every supported format. Types []RepoType // URLs lists the repository or mirrorlist URLs to synchronize. URLs []string // Destination is the local directory repositories are placed under. Destination string // Trim drops this many leading URL path components when building the // destination path. Trim int // Flat places repositories directly in Destination without copying the // URL path. Flat bool // Discover crawls each URL's directory listings for repositories. Discover bool // DiscoverDepth limits how deep discovery crawls below each URL. DiscoverDepth int // Exclude lists globs matched against the names of entries found while // crawling; a match is neither crawled nor mirrored. A pattern // containing a slash is matched against the entry's path below the // crawled URL instead of its name. Exclude []string // IncludeFiles lists globs matched against the names of files found // outside of any repository while crawling; a match is mirrored as a // plain file. Files inside a repository come from its metadata and are // not matched here. IncludeFiles []string // Workers is the number of concurrent download workers. Workers int // Verify re-verifies checksums of existing local files. Verify bool // Prune deletes local files no longer part of the repository. Prune bool // PruneGrace defers pruning a file until it has been unreferenced for // this long across runs. PruneGrace time.Duration // DryRun reports what would change without downloading packages or // altering the published repository. DryRun bool // Trace writes a trace file describing this mirror into every // repository synchronized, and mirrors the traces upstream publishes // beside it. Trace bool // TraceInfo describes the mirror recorded in that trace file. TraceInfo TraceInfo // destBase, when set by the mirror server, pins the repository root // directory directly instead of deriving it from the URL path. destBase string // Components limits deb synchronization to these components. Components []string // Architectures limits deb synchronization to these architectures. Architectures []string } // discoverTypes returns the formats a run identifies repositories as: the // formats it was limited to, the single format it was given, or every // supported format when it named none. func (o *Options) discoverTypes() []RepoType { if len(o.Types) > 0 { return o.Types } if o.Type != "" { return []RepoType{o.Type} } return AllTypes } // repoDest maps a repository URL onto the destination directory according // to the configured trimming rules. func (o *Options) repoDest(repoURL string) (string, error) { if o.destBase != "" { return o.destBase, nil } if o.Flat { return o.Destination, nil } u, err := url.Parse(repoURL) if err != nil { return "", err } var segs []string for _, seg := range strings.Split(u.Path, "/") { if seg != "" { segs = append(segs, seg) } } if o.Trim >= len(segs) { return o.Destination, nil } return fetch.LocalJoin(o.Destination, path.Join(segs[o.Trim:]...)) }