repo-sync/sync_cmd.go

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	cfg "github.com/grmrgecko/repo-sync/config"
	"github.com/grmrgecko/repo-sync/mirror"
)

// SyncArgs holds the flags and positional arguments shared by every
// repository type.
type SyncArgs struct {
	Trim     int  `help:"Number of leading URL path components to drop when building the destination path." default:"0"`
	Flat     bool `help:"Place repositories directly in the destination directory without copying the URL path."`
	Discover bool `help:"Treat each URL as a directory index and crawl it for repositories."`
	Depth    int  `help:"Maximum directory depth crawled in discovery mode." default:"5"`
	Workers  int  `help:"Number of concurrent download workers; defaults to the configured crawler workers."`
	Verify   bool `help:"Re-verify checksums of files that already exist locally."`
	Prune    bool `help:"Delete local files that are no longer part of the repository."`

	PruneGrace time.Duration `help:"Keep files that left the repository for this long before pruning them (e.g. 72h); defaults to the configured crawler prune grace."`
	DryRun     bool          `help:"Report what would be downloaded or pruned without changing the repository."`

	Args []string `arg:"" name:"url" help:"Repository or mirrorlist URLs followed by the destination directory."`
}

// options validates the shared arguments and builds the mirror options.
func (s *SyncArgs) options(typ mirror.RepoType) (*mirror.Options, error) {
	if len(s.Args) < 2 {
		return nil, errors.New("expected at least one repository URL and a destination directory")
	}
	urls := s.Args[:len(s.Args)-1]
	for _, raw := range urls {
		u, err := url.Parse(raw)
		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
			return nil, fmt.Errorf("invalid repository URL %q", raw)
		}
	}
	dest, err := filepath.Abs(s.Args[len(s.Args)-1])
	if err != nil {
		return nil, err
	}
	if s.Trim < 0 {
		return nil, errors.New("trim must not be negative")
	}
	if s.Depth < 0 {
		return nil, errors.New("depth must not be negative")
	}
	if s.PruneGrace < 0 {
		return nil, errors.New("prune grace must not be negative")
	}

	// Unset worker and grace flags fall back to the crawler configuration,
	// so a config file tunes the sync commands the same way it tunes the
	// server's background crawls.
	if s.Workers == 0 {
		s.Workers = cfg.C.Crawler.Workers
	}
	if s.PruneGrace == 0 {
		s.PruneGrace = cfg.C.Crawler.PruneGrace
	}
	return &mirror.Options{
		Type:          typ,
		URLs:          urls,
		Destination:   dest,
		Trim:          s.Trim,
		Flat:          s.Flat,
		Discover:      s.Discover,
		DiscoverDepth: s.Depth,
		Workers:       s.Workers,
		Verify:        s.Verify,
		Prune:         s.Prune,
		PruneGrace:    s.PruneGrace,
		DryRun:        s.DryRun,
		Trace:         cfg.C.Trace.Enabled,
		TraceInfo: mirror.TraceInfo{
			Host:       cfg.C.Trace.HostName(),
			Maintainer: cfg.C.Trace.Maintainer,
			Sponsor:    cfg.C.Trace.Sponsor,
			Country:    cfg.C.Trace.Country,
			Location:   cfg.C.Trace.Location,
			Throughput: cfg.C.Trace.Throughput,
		},
	}, nil
}

// run executes a synchronization with signal-aware cancellation, reporting
// what the run transferred once it ends.
func (s *SyncArgs) run(opts *mirror.Options) error {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	summary, err := mirror.Sync(ctx, opts)
	printSummary(os.Stdout, summary)
	return err
}

// printSummary reports the run totals as "Field: value" lines, in the style
// of rsync's own statistics, so a calling script can act on what changed by
// reading the output. It is written to standard output rather than through
// the logger, which a configuration is free to send elsewhere or format as
// JSON, and it is reported even for a failed run, as a run that failed part
// way through still changed the destination.
func printSummary(w io.Writer, summary *mirror.Summary) {
	if summary == nil {
		return
	}
	fmt.Fprintln(w)
	fmt.Fprintf(w, "Number of repositories: %d\n", summary.Repositories)
	fmt.Fprintf(w, "Number of failed repositories: %d\n", summary.Failed)

	// A dry run transfers metadata to plan against, so reporting that as
	// files transferred would read as a change. It reports what a real run
	// would have moved instead.
	if summary.DryRun {
		fmt.Fprintf(w, "Dry run: true\n")
		fmt.Fprintf(w, "Number of files to transfer: %d\n", summary.Planned)
		fmt.Fprintf(w, "Total file size to transfer: %d bytes\n", summary.PlannedBytes)
	} else {
		fmt.Fprintf(w, "Number of files transferred: %d\n", summary.Fetched)
		fmt.Fprintf(w, "Number of files pruned: %d\n", summary.Pruned)
		fmt.Fprintf(w, "Total transferred file size: %d bytes\n", summary.FetchedBytes)
	}
	fmt.Fprintf(w, "Number of files unchanged: %d\n", summary.Unchanged)
	fmt.Fprintf(w, "Repository changed: %t\n", summary.Changed())
}

// RPMCmd synchronizes RPM repositories.
type RPMCmd struct {
	SyncArgs `embed:""`
}

// Run performs the RPM synchronization.
func (c *RPMCmd) Run() error {
	opts, err := c.options(mirror.RepoRPM)
	if err != nil {
		return err
	}
	return c.run(opts)
}

// ArchCmd synchronizes Arch Linux repositories.
type ArchCmd struct {
	SyncArgs `embed:""`
}

// Run performs the Arch Linux synchronization.
func (c *ArchCmd) Run() error {
	opts, err := c.options(mirror.RepoArch)
	if err != nil {
		return err
	}
	return c.run(opts)
}

// ApkCmd synchronizes Alpine Linux repositories.
type ApkCmd struct {
	SyncArgs `embed:""`
}

// Run performs the Alpine Linux synchronization.
func (c *ApkCmd) Run() error {
	opts, err := c.options(mirror.RepoApk)
	if err != nil {
		return err
	}
	return c.run(opts)
}

// DebCmd synchronizes DEB repositories.
type DebCmd struct {
	Component []string `help:"Limit synchronization to these components."`
	Arch      []string `help:"Limit synchronization to these architectures; include \"source\" to keep source indexes."`

	SyncArgs `embed:""`
}

// Run performs the DEB synchronization.
func (c *DebCmd) Run() error {
	opts, err := c.options(mirror.RepoDeb)
	if err != nil {
		return err
	}
	opts.Components = c.Component
	opts.Architectures = c.Arch
	return c.run(opts)
}