Add per-repo file purge times

config.go

@@ -20,6 +20,7 @@ type Repo struct {
 	URL                  string    `yaml:"url"`
 	URLs                 []string  `yaml:"urls"`
 	Mirrorlist           string    `yaml:"mirrorlist"`
+	PurgeFilesAfter      *int      `yaml:"purge_files_after"`
 	LastMirrorlistCheck  time.Time `yaml:"-"`
 	LastModificationTime time.Time `yaml:"-"`
 	urlsChan             chan chan []string
@@ -88,6 +89,15 @@ func parseConfig(raw []byte) *Config {
 			}
 			log.Fatalf("mirrorlist file %v for repo %v does not exist or isn't readable for user %v", repo.Mirrorlist, name, u.Username)
 		}
+
+		purge := result.PurgeFilesAfter
+		if repo.PurgeFilesAfter != nil {
+			purge = *repo.PurgeFilesAfter
+		}
+		if 0 < purge && purge < 10*60 {
+			log.Fatalf("'purge_files_after' period is too low (%v) please specify at least 10 minutes", purge)
+		}
+
 		initURLsChannel(name, repo)
 	}
 

config_test.go

@@ -9,6 +9,19 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
+func (r *Repo) Equal(x interface{}) bool {
+	s, ok := x.(*Repo)
+	return ok && r.URL == s.URL &&
+		cmp.Equal(r.URLs, s.URLs) &&
+		r.Mirrorlist == s.Mirrorlist &&
+		r.LastMirrorlistCheck == s.LastMirrorlistCheck &&
+		r.LastModificationTime == s.LastModificationTime &&
+		(r.PurgeFilesAfter == s.PurgeFilesAfter ||
+			(r.PurgeFilesAfter != nil &&
+				s.PurgeFilesAfter != nil &&
+				*r.PurgeFilesAfter == *s.PurgeFilesAfter))
+}
+
 // test that `parseConfig()` can successfully load YAML config
 func TestLoadConfig(t *testing.T) {
 	var temp = t.TempDir()
@@ -56,7 +69,7 @@ repos:
 		DownloadTimeout: 200,
 		Prefetch:        &RefreshPeriod{Cron: "0 0 3 * * * *", TTLUnaccessed: 5, TTLUnupdated: 200},
 	}
-	if !cmp.Equal(*got, *want, cmpopts.IgnoreFields(Config{}, "Prefetch"), cmpopts.IgnoreUnexported(Repo{})) {
+	if !cmp.Equal(*got, *want, cmpopts.IgnoreFields(Config{}, "Prefetch")) {
 		t.Errorf("got %v, want %v", *got, *want)
 	}
 	gotR := *(*got).Prefetch
@@ -88,7 +101,7 @@ repos:
 		Prefetch:        nil,
 	}
 
-	if !cmp.Equal(*got, *want, cmpopts.IgnoreUnexported(Repo{})) {
+	if !cmp.Equal(*got, *want) {
 		t.Errorf("got %v, want %v", *got, *want)
 	}
 }
@@ -114,7 +127,50 @@ repos:
 		Prefetch:        nil,
 	}
 
-	if !cmp.Equal(*got, *want, cmpopts.IgnoreUnexported(Repo{})) {
+	if !cmp.Equal(*got, *want) {
+		t.Errorf("got %v, want %v", *got, *want)
+	}
+}
+
+func TestPerRepoPurgeFilesAfter(t *testing.T) {
+	zero := 0
+	oneHundredThousand := 100000
+	got := parseConfig([]byte(`
+cache_dir: /tmp
+purge_files_after: 30000
+repos:
+  archlinux:
+    url: http://mirrors.kernel.org/archlinux
+  anotherlinux:
+    url: http://dev.null
+    purge_files_after: 0
+  yetanotherlinux:
+    url: http://dev.zero
+    purge_files_after: 100000
+
+`))
+	want := &Config{
+		CacheDir: `/tmp`,
+		Port:     9129,
+		Repos: map[string]*Repo{
+			"archlinux": &Repo{
+				URL: "http://mirrors.kernel.org/archlinux",
+			},
+			"anotherlinux": &Repo{
+				URL:             "http://dev.null",
+				PurgeFilesAfter: &zero,
+			},
+			"yetanotherlinux": &Repo{
+				URL:             "http://dev.zero",
+				PurgeFilesAfter: &oneHundredThousand,
+			},
+		},
+		PurgeFilesAfter: 30000,
+		DownloadTimeout: 0,
+		Prefetch:        nil,
+	}
+
+	if !cmp.Equal(*got, *want) {
 		t.Errorf("got %v, want %v", *got, *want)
 	}
 }
@@ -153,7 +209,7 @@ repos:
 		DownloadTimeout: 200,
 		Prefetch:        &RefreshPeriod{Cron: "0 0 3 * * * *", TTLUnaccessed: 5, TTLUnupdated: 200},
 	}
-	if !cmp.Equal(*got, *want, cmpopts.IgnoreFields(Config{}, "Prefetch"), cmpopts.IgnoreUnexported(Repo{})) {
+	if !cmp.Equal(*got, *want, cmpopts.IgnoreFields(Config{}, "Prefetch")) {
 		t.Errorf("got %v, want %v", *got, *want)
 	}
 	gotR := *(*got).Prefetch
@@ -182,7 +238,7 @@ repos:
 			},
 		},
 	}
-	if !cmp.Equal(*got, *want, cmpopts.IgnoreUnexported(Repo{})) {
+	if !cmp.Equal(*got, *want) {
 		t.Errorf("got %v, want %v", *got, *want)
 	}
 }

pacoloco.go

@@ -111,10 +111,8 @@ func main() {
 		setupPrefetch() // enable refresh
 	}
 
-	if config.PurgeFilesAfter != 0 {
-		cleanupTicker := setupPurgeStaleFilesRoutine()
-		defer cleanupTicker.Stop()
-	}
+	cleanupTicker := setupPurgeStaleFilesRoutine()
+	defer cleanupTicker.Stop()
 
 	if config.HttpProxy != "" {
 		proxyUrl, err := url.Parse(config.HttpProxy)

purge.go

@@ -8,32 +8,36 @@ import (
 	"time"
 )
 
+func (r *Repo) purgeSeconds() int {
+	if r.PurgeFilesAfter != nil {
+		return *r.PurgeFilesAfter
+	}
+	return config.PurgeFilesAfter
+}
+
 func setupPurgeStaleFilesRoutine() *time.Ticker {
 	ticker := time.NewTicker(time.Duration(24) * time.Hour) // purge files once a day
 	go func() {
-		purgeStaleFiles(config.CacheDir, config.PurgeFilesAfter)
-		for {
-			select {
-			case <-ticker.C:
-				purgeStaleFiles(config.CacheDir, config.PurgeFilesAfter)
+		for _ = range ticker.C {
+			for repoName, repo := range config.Repos {
+				dir := filepath.Join(config.CacheDir, "pkgs", repoName)
+				purgeStaleFiles(dir, repo.purgeSeconds())
 			}
 		}
 	}()
-
 	return ticker
 }
 
 // purgeStaleFiles purges files in the pacoloco cache
-// it recursively scans `cacheDir`/pkgs and if the file access time is older than
+// it recursively scans `cacheDir`and if the file access time is older than
 // `now` - purgeFilesAfter(seconds) then the file gets removed
 func purgeStaleFiles(cacheDir string, purgeFilesAfter int) {
-	// safety check, so we don't unintentionally wipe the whole cache
+	// 0 means never purge.
 	if purgeFilesAfter == 0 {
-		log.Fatalf("Stopping because purgeFilesAfter=%v and that would purge the whole cache", purgeFilesAfter)
+		return
 	}
 
 	removeIfOlder := time.Now().Add(time.Duration(-purgeFilesAfter) * time.Second)
-	pkgDir := filepath.Join(cacheDir, "pkgs")
 
 	// Go through all files in the repos, and check if access time is older than `removeIfOlder`
 	walkfn := func(path string, info os.FileInfo, err error) error {
@@ -55,7 +59,7 @@ func purgeStaleFiles(cacheDir string, purgeFilesAfter int) {
 		}
 		return nil
 	}
-	if err := filepath.Walk(pkgDir, walkfn); err != nil {
+	if err := filepath.Walk(cacheDir, walkfn); err != nil {
 		log.Println(err)
 	}
 }

purge_test.go

@@ -52,3 +52,15 @@ func TestPurge(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestPurgeSeconds(t *testing.T) {
+	r := Repo{}
+	if r.purgeSeconds() != config.PurgeFilesAfter {
+		t.Fail()
+	}
+
+	n := 99
+	if r.PurgeFilesAfter = &n; r.purgeSeconds() != n {
+		t.Fail()
+	}
+}