Merge branch 'master' into feature/legacy-random_release-wrapper

cmd/alice/flags.go

@@ -265,8 +265,29 @@ func flagsInitialization() []cli.Flag {
 			// "https://download.maxmind.com/geoip/databases/GeoLite2-Country/download?suffix=tar.gz.sha256"
 		},
 		&cli.BoolFlag{
-			Name:               "geoip-download-sha256-skip",
+			Name:     "geoip-download-sha256-skip",
+			Category: "GeoIP",
+			Usage: `sha256 helps to check database contents of the mmdb database
+			and avoid unnecessary requests to MaxMind CDN`,
+			DisableDefaultText: true,
+		},
+		&cli.DurationFlag{
+			Name:     "geoip-update-frequency",
+			Category: "GeoIP",
+			Usage: `when geoip-maxmind-permalink is selected and geoip-db-path is empty,
+			once within a certain 'PERIOD' of time app will update the geoip database;
+			do not forget about maxmind donwload limits; set to 0s if want to disable`,
+			Value: 24 * time.Hour,
+		},
+		&cli.DurationFlag{
+			Name:     "geoip-update-retry-frequency",
+			Category: "GeoIP",
+			Value:    1 * time.Hour,
+		},
+		&cli.BoolFlag{
+			Name:               "geoip-skip-database-verify",
 			Category:           "GeoIP",
+			Usage:              "skip mmdb contents validation by vendor function db.Verify()",
 			DisableDefaultText: true,
 		},
 

cmd/alice/main.go

@@ -114,8 +114,8 @@ func main() {
 		exitcode = 1
 	}
 
-	// TODO avoid this shit
-	// fucking diode was no `wait` method, so we need to use this `250` shit
+	// TODO avoid this
+	// diode hasn't Wait() method, so we need to use this `250` shit
 	log.Trace().Msg("waiting for diode buf")
 	time.Sleep(250 * time.Millisecond)
 }

internal/cache/api.go

@@ -97,7 +97,7 @@ func (m *Cache) ApiStats() io.Reader {
 
 	tb.SetOutputMirror(buf)
 	tb.AppendHeader(table.Row{
-		"zone", "number of entries", "capacity (mb)", "hits", "misses", "delhits", "delmisses", "collisions",
+		"zone", "number of entries", "capacity (mb)", "hits", "misses", "delhits", "delmisses", "collisions", "misses %",
 	})
 
 	for zone, cache := range m.pools {
@@ -110,11 +110,16 @@ func (m *Cache) ApiStats() io.Reader {
 			cache.Stats().DelHits,
 			cache.Stats().DelMisses,
 			cache.Stats().Collisions,
+			round(float64(cache.Stats().Misses*100/cache.Stats().Hits), 2),
 		})
 	}
 
 	tb.Style().Options.SeparateRows = true
 
+	tb.SortBy([]table.SortBy{
+		{Number: 0, Mode: table.Asc},
+	})
+
 	return buf
 }
 

internal/cache/cache.go

@@ -21,8 +21,8 @@ const (
 )
 
 var zoneHumanize = map[cacheZone]string{
-	defaultCache:    "default cache",
-	quarantineCache: "quarantine cache",
+	defaultCache:    "default",
+	quarantineCache: "quarantine",
 }
 
 type Cache struct {
@@ -107,7 +107,8 @@ func (m *Cache) Bootstrap() {
 	m.log.Info().Msg("internal abort() has been caught; initiate application closing...")
 
 	for zone, cache := range m.pools {
-		m.log.Info().Msgf("Serving SUMMARY: DelHits %d, DelMiss %d, Coll %d, Hit %d, Miss %d",
+		m.log.Info().Msgf("Cache %s serving SUMMARY: DelHits %d, DelMiss %d, Coll %d, Hit %d, Miss %d",
+			zoneHumanize[zone],
 			cache.Stats().DelHits, cache.Stats().DelMisses, cache.Stats().Collisions,
 			cache.Stats().Hits, cache.Stats().Misses)
 

internal/geoip/extract.go

@@ -0,0 +1,50 @@
+package geoip
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"strings"
+
+	"github.com/rs/zerolog"
+)
+
+func extractTarGzArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) {
+	var rd *gzip.Reader
+	if rd, e = gzip.NewReader(src); e != nil {
+		return
+	}
+
+	return extractTarArchive(log, dst, rd)
+}
+
+func extractTarArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) {
+	tr := tar.NewReader(src)
+	for {
+		var hdr *tar.Header
+		hdr, e = tr.Next()
+
+		if e == io.EOF {
+			break // End of archive
+		} else if e != nil {
+			return
+		}
+
+		log.Trace().Msg("found file in maxmind tar archive - " + hdr.Name)
+		if !strings.HasSuffix(hdr.Name, "mmdb") {
+			continue
+		}
+
+		log.Trace().Msg("found mmdb file, copy to temporary file")
+
+		var written int64
+		if written, e = io.Copy(dst, tr); e != nil { // skipcq: GO-S2110 decompression bomb isn't possible here
+			return
+		}
+
+		log.Debug().Msgf("parsed response has written in temporary file with %d bytes", written)
+		break
+	}
+
+	return
+}

internal/geoip/geoip_file.go → internal/geoip/file.go

@@ -12,12 +12,14 @@ import (
 )
 
 type GeoIPFileClient struct {
+	mu sync.RWMutex
 	*maxminddb.Reader
 
 	appname, tempdir string
+	skipVerify       bool
 
-	mu      sync.RWMutex
-	isReady bool
+	muReady sync.RWMutex
+	ready   bool
 
 	log *zerolog.Logger
 
@@ -34,42 +36,43 @@ func NewGeoIPFileClient(c context.Context, path string) (_ GeoIPClient, e error)
 		done:  c.Done,
 		abort: c.Value(utils.CKAbortFunc).(context.CancelFunc),
 
-		appname: cli.App.Name,
-		tempdir: fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version),
+		appname:    cli.App.Name,
+		tempdir:    fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version),
+		skipVerify: cli.Bool("geoip-skip-database-verify"),
 	}
 
 	gipc.Reader, e = maxminddb.Open(path)
 	return gipc, e
 }
 
 func (m *GeoIPFileClient) Bootstrap() {
-	m.log.Info().Msg("geoip has been initied")
-
-	var e error
-	if e = m.Reader.Verify(); e != nil {
-		m.log.Error().Msg("could not verify maxmind DB - " + e.Error())
-		m.abort()
-		return
+	if !m.skipVerify {
+		if e := m.Reader.Verify(); e != nil {
+			m.log.Error().Msg("could not verify maxmind DB - " + e.Error())
+			m.abort()
+			return
+		}
 	}
 
-	m.mu.Lock()
-	m.isReady = true
-	m.mu.Unlock()
+	m.log.Debug().Msg("geoip has been initied")
+	m.setReady(true)
 
 	<-m.done()
 	m.log.Info().Msg("internal abort() has been caught; initiate application closing...")
 
+	m.setReady(false)
 	m.destroy()
 }
 
 func (m *GeoIPFileClient) LookupCountryISO(ip string) (string, error) {
-	return lookupISOByIP(m.Reader, ip)
+	return lookupISOByIP(&m.mu, m.Reader, ip)
 }
 
 func (m *GeoIPFileClient) IsReady() bool {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.isReady
+	m.muReady.RLock()
+	defer m.muReady.RUnlock()
+
+	return m.ready
 }
 
 //
@@ -79,3 +82,10 @@ func (m *GeoIPFileClient) destroy() {
 		m.log.Warn().Msg("could not close maxmind reader - " + e.Error())
 	}
 }
+
+func (m *GeoIPFileClient) setReady(ready bool) {
+	m.muReady.Lock()
+	defer m.muReady.Unlock()
+
+	m.ready = ready
+}

internal/geoip/geoip.go

@@ -37,7 +37,13 @@ func (m *geoIPRecord) reset() {
 	*m = geoIPRecord{}
 }
 
-func lookupISOByIP(mxrd *maxminddb.Reader, rawip string) (iso string, e error) {
+func lookupISOByIP(mu *sync.RWMutex, mxrd *maxminddb.Reader, rawip string) (iso string, e error) {
+	if !mu.TryRLock() {
+		e = errors.New("could not get lock for geoip lookup()")
+		return
+	}
+	defer mu.RUnlock()
+
 	var ip netip.Addr
 	if ip, e = netip.ParseAddr(rawip); e != nil {
 		e = errors.New(fmt.Sprintf("could not parse ip addr with netip library, ip - %+v - ", rawip) + e.Error())