diff --git a/cmd/alice/flags.go b/cmd/alice/flags.go index 2b57406..1ce29c5 100644 --- a/cmd/alice/flags.go +++ b/cmd/alice/flags.go @@ -265,8 +265,29 @@ func flagsInitialization() []cli.Flag { // "https://download.maxmind.com/geoip/databases/GeoLite2-Country/download?suffix=tar.gz.sha256" }, &cli.BoolFlag{ - Name: "geoip-download-sha256-skip", + Name: "geoip-download-sha256-skip", + Category: "GeoIP", + Usage: `sha256 helps to check database contents of the mmdb database + and avoid unnecessary requests to MaxMind CDN`, + DisableDefaultText: true, + }, + &cli.DurationFlag{ + Name: "geoip-update-frequency", + Category: "GeoIP", + Usage: `when geoip-maxmind-permalink is selected and geoip-db-path is empty, + once within a certain 'PERIOD' of time app will update the geoip database; + do not forget about maxmind donwload limits; set to 0s if want to disable`, + Value: 24 * time.Hour, + }, + &cli.DurationFlag{ + Name: "geoip-update-retry-frequency", + Category: "GeoIP", + Value: 1 * time.Hour, + }, + &cli.BoolFlag{ + Name: "geoip-skip-database-verify", Category: "GeoIP", + Usage: "skip mmdb contents validation by vendor function db.Verify()", DisableDefaultText: true, }, diff --git a/cmd/alice/main.go b/cmd/alice/main.go index ed22fbc..09003b7 100644 --- a/cmd/alice/main.go +++ b/cmd/alice/main.go @@ -114,8 +114,8 @@ func main() { exitcode = 1 } - // TODO avoid this shit - // fucking diode was no `wait` method, so we need to use this `250` shit + // TODO avoid this + // diode hasn't Wait() method, so we need to use this `250` shit log.Trace().Msg("waiting for diode buf") time.Sleep(250 * time.Millisecond) } diff --git a/internal/cache/api.go b/internal/cache/api.go index 486dbd6..0ed778f 100644 --- a/internal/cache/api.go +++ b/internal/cache/api.go @@ -97,7 +97,7 @@ func (m *Cache) ApiStats() io.Reader { tb.SetOutputMirror(buf) tb.AppendHeader(table.Row{ - "zone", "number of entries", "capacity (mb)", "hits", "misses", "delhits", "delmisses", "collisions", + "zone", "number of entries", "capacity (mb)", "hits", "misses", "delhits", "delmisses", "collisions", "misses %", }) for zone, cache := range m.pools { @@ -110,11 +110,16 @@ func (m *Cache) ApiStats() io.Reader { cache.Stats().DelHits, cache.Stats().DelMisses, cache.Stats().Collisions, + round(float64(cache.Stats().Misses*100/cache.Stats().Hits), 2), }) } tb.Style().Options.SeparateRows = true + tb.SortBy([]table.SortBy{ + {Number: 0, Mode: table.Asc}, + }) + return buf } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index f0ebe51..7c7a744 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -21,8 +21,8 @@ const ( ) var zoneHumanize = map[cacheZone]string{ - defaultCache: "default cache", - quarantineCache: "quarantine cache", + defaultCache: "default", + quarantineCache: "quarantine", } type Cache struct { @@ -107,7 +107,8 @@ func (m *Cache) Bootstrap() { m.log.Info().Msg("internal abort() has been caught; initiate application closing...") for zone, cache := range m.pools { - m.log.Info().Msgf("Serving SUMMARY: DelHits %d, DelMiss %d, Coll %d, Hit %d, Miss %d", + m.log.Info().Msgf("Cache %s serving SUMMARY: DelHits %d, DelMiss %d, Coll %d, Hit %d, Miss %d", + zoneHumanize[zone], cache.Stats().DelHits, cache.Stats().DelMisses, cache.Stats().Collisions, cache.Stats().Hits, cache.Stats().Misses) diff --git a/internal/geoip/extract.go b/internal/geoip/extract.go new file mode 100644 index 0000000..ada3c66 --- /dev/null +++ b/internal/geoip/extract.go @@ -0,0 +1,50 @@ +package geoip + +import ( + "archive/tar" + "compress/gzip" + "io" + "strings" + + "github.com/rs/zerolog" +) + +func extractTarGzArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) { + var rd *gzip.Reader + if rd, e = gzip.NewReader(src); e != nil { + return + } + + return extractTarArchive(log, dst, rd) +} + +func extractTarArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) { + tr := tar.NewReader(src) + for { + var hdr *tar.Header + hdr, e = tr.Next() + + if e == io.EOF { + break // End of archive + } else if e != nil { + return + } + + log.Trace().Msg("found file in maxmind tar archive - " + hdr.Name) + if !strings.HasSuffix(hdr.Name, "mmdb") { + continue + } + + log.Trace().Msg("found mmdb file, copy to temporary file") + + var written int64 + if written, e = io.Copy(dst, tr); e != nil { // skipcq: GO-S2110 decompression bomb isn't possible here + return + } + + log.Debug().Msgf("parsed response has written in temporary file with %d bytes", written) + break + } + + return +} diff --git a/internal/geoip/geoip_file.go b/internal/geoip/file.go similarity index 62% rename from internal/geoip/geoip_file.go rename to internal/geoip/file.go index ec08831..7e2188f 100644 --- a/internal/geoip/geoip_file.go +++ b/internal/geoip/file.go @@ -12,12 +12,14 @@ import ( ) type GeoIPFileClient struct { + mu sync.RWMutex *maxminddb.Reader appname, tempdir string + skipVerify bool - mu sync.RWMutex - isReady bool + muReady sync.RWMutex + ready bool log *zerolog.Logger @@ -34,8 +36,9 @@ func NewGeoIPFileClient(c context.Context, path string) (_ GeoIPClient, e error) done: c.Done, abort: c.Value(utils.CKAbortFunc).(context.CancelFunc), - appname: cli.App.Name, - tempdir: fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version), + appname: cli.App.Name, + tempdir: fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version), + skipVerify: cli.Bool("geoip-skip-database-verify"), } gipc.Reader, e = maxminddb.Open(path) @@ -43,33 +46,33 @@ func NewGeoIPFileClient(c context.Context, path string) (_ GeoIPClient, e error) } func (m *GeoIPFileClient) Bootstrap() { - m.log.Info().Msg("geoip has been initied") - - var e error - if e = m.Reader.Verify(); e != nil { - m.log.Error().Msg("could not verify maxmind DB - " + e.Error()) - m.abort() - return + if !m.skipVerify { + if e := m.Reader.Verify(); e != nil { + m.log.Error().Msg("could not verify maxmind DB - " + e.Error()) + m.abort() + return + } } - m.mu.Lock() - m.isReady = true - m.mu.Unlock() + m.log.Debug().Msg("geoip has been initied") + m.setReady(true) <-m.done() m.log.Info().Msg("internal abort() has been caught; initiate application closing...") + m.setReady(false) m.destroy() } func (m *GeoIPFileClient) LookupCountryISO(ip string) (string, error) { - return lookupISOByIP(m.Reader, ip) + return lookupISOByIP(&m.mu, m.Reader, ip) } func (m *GeoIPFileClient) IsReady() bool { - m.mu.RLock() - defer m.mu.RUnlock() - return m.isReady + m.muReady.RLock() + defer m.muReady.RUnlock() + + return m.ready } // @@ -79,3 +82,10 @@ func (m *GeoIPFileClient) destroy() { m.log.Warn().Msg("could not close maxmind reader - " + e.Error()) } } + +func (m *GeoIPFileClient) setReady(ready bool) { + m.muReady.Lock() + defer m.muReady.Unlock() + + m.ready = ready +} diff --git a/internal/geoip/geoip.go b/internal/geoip/geoip.go index 970b037..9fe4740 100644 --- a/internal/geoip/geoip.go +++ b/internal/geoip/geoip.go @@ -37,7 +37,13 @@ func (m *geoIPRecord) reset() { *m = geoIPRecord{} } -func lookupISOByIP(mxrd *maxminddb.Reader, rawip string) (iso string, e error) { +func lookupISOByIP(mu *sync.RWMutex, mxrd *maxminddb.Reader, rawip string) (iso string, e error) { + if !mu.TryRLock() { + e = errors.New("could not get lock for geoip lookup()") + return + } + defer mu.RUnlock() + var ip netip.Addr if ip, e = netip.ParseAddr(rawip); e != nil { e = errors.New(fmt.Sprintf("could not parse ip addr with netip library, ip - %+v - ", rawip) + e.Error()) diff --git a/internal/geoip/geoip_http.go b/internal/geoip/geoip_http.go deleted file mode 100644 index 046d736..0000000 --- a/internal/geoip/geoip_http.go +++ /dev/null @@ -1,272 +0,0 @@ -package geoip - -import ( - "archive/tar" - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "strings" - "sync" - - "github.com/anilibria/alice/internal/utils" - "github.com/klauspost/compress/gzip" - "github.com/oschwald/maxminddb-golang" - "github.com/rs/zerolog" - "github.com/urfave/cli/v2" - "github.com/valyala/fasthttp" - - futils "github.com/gofiber/fiber/v2/utils" -) - -type GeoIPHTTPClient struct { - *maxminddb.Reader - - hclient *fasthttp.Client - - // maxmind - mmfd *os.File - mmurl string - mmusername string - mmpassword string - - appname, tempdir string - - mu sync.RWMutex - isReady bool - - log *zerolog.Logger - - done func() <-chan struct{} - abort context.CancelFunc -} - -func NewGeoIPHTTPClient(c context.Context) (_ GeoIPClient, e error) { - cli := c.Value(utils.CKCliCtx).(*cli.Context) - - gipc := &GeoIPHTTPClient{ - log: c.Value(utils.CKLogger).(*zerolog.Logger), - - done: c.Done, - abort: c.Value(utils.CKAbortFunc).(context.CancelFunc), - - appname: cli.App.Name, - tempdir: fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version), - } - - return gipc.configureHTTPClient(cli) -} - -func (m *GeoIPHTTPClient) Bootstrap() { - var e error - - if m.Reader, e = m.databaseDownload(); e != nil { - m.log.Error().Msg("could not bootstrap GeoIPHTTPClient - " + e.Error()) - m.abort() - return - } - m.log.Info().Msg("geoip has been initied") - - if e = m.Reader.Verify(); e != nil { - m.log.Error().Msg("could not verify maxmind DB - " + e.Error()) - m.abort() - return - } - - m.mu.Lock() - m.isReady = true - m.mu.Unlock() - - <-m.done() - m.log.Info().Msg("internal abort() has been caught; initiate application closing...") - - m.destroy() -} - -func (m *GeoIPHTTPClient) LookupCountryISO(ip string) (string, error) { - return lookupISOByIP(m.Reader, ip) -} - -func (m *GeoIPHTTPClient) IsReady() bool { - m.mu.RLock() - defer m.mu.RUnlock() - return m.isReady -} - -// - -func (m *GeoIPHTTPClient) destroy() { - if e := m.Close(); e != nil { - m.log.Warn().Msg("could not close maxmind reader - " + e.Error()) - } - - if e := m.mmfd.Close(); e != nil { - m.log.Warn().Msg("could not close temporary geoip file - " + e.Error()) - } - - if e := os.Remove(m.mmfd.Name()); e != nil { - m.log.Warn().Msg("could not remove temporary file - " + e.Error()) - } -} - -func (m *GeoIPHTTPClient) configureHTTPClient(c *cli.Context) (_ GeoIPClient, e error) { - rrl := fasthttp.AcquireURI() - defer fasthttp.ReleaseURI(rrl) - - if e = rrl.Parse(nil, futils.UnsafeBytes(c.String("geoip-maxmind-permalink"))); e != nil { - return - } - m.mmurl = rrl.String() - - var creds []string - if creds = strings.Split(c.String("geoip-maxmind-license"), ":"); len(creds) != 2 { - e = errors.New("license format is not valid; it must be formated as `client_id:key`") - return - } else if len(creds[0]) == 0 || len(creds[1]) == 0 { - e = errors.New("license id or key is empty; record must be formated as `client_id:key`") - return - } - m.mmusername, m.mmpassword = creds[0], creds[1] - - m.hclient = &fasthttp.Client{ - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent#crawler_and_bot_ua_strings - Name: fmt.Sprintf("Mozilla/5.0 (compatible; %s/%s; +https://anilibria.top/support)", - c.App.Name, c.App.Version), - - DisableHeaderNamesNormalizing: true, - DisablePathNormalizing: true, - NoDefaultUserAgentHeader: false, - - // TODO - // ? timeouts - // ? dns cache - // ? keep alive - } - - return m, e -} - -func (m *GeoIPHTTPClient) makeTempFile() (_ *os.File, e error) { - temppath := fmt.Sprintf("%s/%s", os.TempDir(), m.tempdir) - var fstat fs.FileInfo - if fstat, e = os.Stat(temppath); e != nil { - if !os.IsNotExist(e) { - return - } - - os.MkdirAll(temppath, 0700) - } else if !fstat.IsDir() { - e = errors.New("temporary path is exists and it's not a directory - " + temppath) - return - } - - var fd *os.File - fd, e = os.CreateTemp(temppath, m.appname+"_*.mmdb") - - return fd, e -} - -func (m *GeoIPHTTPClient) databaseDownload() (_ *maxminddb.Reader, e error) { - if m.mmfd, e = m.makeTempFile(); e != nil { - return - } - m.log.Debug().Msgf("file %s has been successfully allocated", m.mmfd.Name()) - - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - req.Header.SetUserAgent(m.hclient.Name) - req.SetRequestURI(m.mmurl) - req.URI().SetUsername(m.mmusername) - req.URI().SetPassword(m.mmpassword) - - rsp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(rsp) - - for maxRedirects := 5; ; maxRedirects-- { - if maxRedirects == 0 { - e = errors.New("maxmind responded with too many redirects, redirects count exceeded") - return - } - - m.log.Trace().Msg(req.String()) - if e = m.hclient.Do(req, rsp); e != nil { - return - } - - status := rsp.StatusCode() - if fasthttp.StatusCodeIsRedirect(status) { - m.log.Trace().Msg(rsp.String()) - m.log.Debug().Msgf("maxmind responded with redirect %d, go to %s", status, - futils.UnsafeString(rsp.Header.Peek(fasthttp.HeaderLocation))) - - req.Header.Del(fasthttp.HeaderAuthorization) - - req.SetRequestURIBytes(rsp.Header.Peek(fasthttp.HeaderLocation)) - req.URI().Parse(nil, rsp.Header.Peek(fasthttp.HeaderLocation)) - continue - } - - if status != fasthttp.StatusOK { - m.log.Trace().Msg(rsp.String()) - m.log.Error().Msgf("maxmind responded with %d", status) - - e = errors.New("maxmind api returned non 200 response") - return - } - - if len(rsp.Body()) == 0 { - m.log.Trace().Msg(rsp.String()) - m.log.Error().Msg("maxmind responded with empty body") - - e = errors.New("maxmind responded with empty body") - return - } - - break - } - - // GZIP reader - var rd *gzip.Reader - if rd, e = gzip.NewReader(bytes.NewBuffer(rsp.Body())); e != nil { - return - } - - // TAR reader - tr := tar.NewReader(rd) - for { - var hdr *tar.Header - hdr, e = tr.Next() - - if e == io.EOF { - break // End of archive - } else if e != nil { - return - } - - m.log.Trace().Msg("found file in maxmind tar archive - " + hdr.Name) - if !strings.HasSuffix(hdr.Name, "mmdb") { - continue - } - - m.log.Trace().Msg("found mmdb file, copy to temporary file") - - var written int64 - if written, e = io.Copy(m.mmfd, tr); e != nil { // skipcq: GO-S2110 decompression bomb isn't important here - return - } - - m.log.Debug().Msgf("parsed response has written in temporary file with %d bytes", written) - break - } - - // !!! --geoip-download-sha256-skip - // !!! --geoip-download-sha256-skip - // !!! --geoip-download-sha256-skip - // !!! --geoip-download-sha256-skip - - return maxminddb.Open(m.mmfd.Name()) -} diff --git a/internal/geoip/http.go b/internal/geoip/http.go new file mode 100644 index 0000000..20e1452 --- /dev/null +++ b/internal/geoip/http.go @@ -0,0 +1,374 @@ +package geoip + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "strings" + "sync" + "time" + + "github.com/anilibria/alice/internal/utils" + "github.com/oschwald/maxminddb-golang" + "github.com/rs/zerolog" + "github.com/urfave/cli/v2" + "github.com/valyala/fasthttp" + + futils "github.com/gofiber/fiber/v2/utils" +) + +type GeoIPHTTPClient struct { + mu sync.RWMutex + *maxminddb.Reader + fd *os.File + + hclient *fasthttp.Client + + // maxmind + mmurl string + mmusername string + mmpassword string + + mmSkipHashVerify bool + mmLastHash []byte + + muUpdate sync.Mutex + mmUpdateFreq time.Duration + mmRetryFreq time.Duration + + appname, tempdir string + skipVerify bool + + muReady sync.RWMutex + ready bool + + log *zerolog.Logger + + done func() <-chan struct{} + abort context.CancelFunc +} + +func NewGeoIPHTTPClient(c context.Context) (_ GeoIPClient, e error) { + cli := c.Value(utils.CKCliCtx).(*cli.Context) + + gipc := &GeoIPHTTPClient{ + log: c.Value(utils.CKLogger).(*zerolog.Logger), + + done: c.Done, + abort: c.Value(utils.CKAbortFunc).(context.CancelFunc), + + appname: cli.App.Name, + tempdir: fmt.Sprintf("%s_%s", cli.App.Name, cli.App.Version), + + skipVerify: cli.Bool("geoip-skip-database-verify"), + + mmSkipHashVerify: cli.Bool("geoip-download-sha256-skip"), + mmUpdateFreq: cli.Duration("geoip-update-frequency"), + mmRetryFreq: cli.Duration("geoip-update-retry-frequency"), + } + + return gipc.configureHTTPClient(cli) +} + +func (m *GeoIPHTTPClient) Bootstrap() { + var e error + + if m.fd, m.Reader, e = m.databaseDownload(); e != nil { + m.log.Error().Msg("could not bootstrap GeoIPHTTPClient - " + e.Error()) + m.abort() + return + } + defer m.destroyDB(m.fd, m.Reader) + + if !m.skipVerify { + if e = m.Reader.Verify(); e != nil { + m.log.Error().Msg("could not verify maxmind DB - " + e.Error()) + m.abort() + return + } + } + + m.setReady(true) + defer m.setReady(false) + + m.loop() +} + +func (m *GeoIPHTTPClient) LookupCountryISO(ip string) (string, error) { + return lookupISOByIP(&m.mu, m.Reader, ip) +} + +func (m *GeoIPHTTPClient) IsReady() bool { + m.muReady.RLock() + defer m.muReady.RUnlock() + return m.ready +} + +// + +func (m *GeoIPHTTPClient) loop() { + m.log.Debug().Msg("initiate geoip db update loop...") + defer m.log.Debug().Msg("geoip db update loop has been closed") + + var update *time.Timer + if m.mmUpdateFreq != 0 { + update = time.NewTimer(m.mmUpdateFreq) + m.log.Debug().Msgf("geoip database updater enabled; update period - %s", m.mmUpdateFreq.String()) + } + +LOOP: + for { + select { + case <-update.C: + if !m.muUpdate.TryLock() { + m.log.Error().Msg("could not start the mmdb update, last proccess is not marked as complete") + update.Reset(m.mmRetryFreq) + continue + } + if !m.IsReady() { + m.log.Error().Msg("could not start the mmdb update, ready flag is false at this moment") + update.Reset(m.mmRetryFreq) + m.muUpdate.Unlock() + continue + } + m.log.Info().Msg("starting geoip database update") + m.log.Debug().Msg("geoip database update, downloading...") + defer m.log.Debug().Msg("geoip database update, finished") + + newfd, newrd, e := m.databaseDownload() + if e != nil && newfd != nil && newrd != nil { // update is not required + m.log.Info().Msg(e.Error()) + m.muUpdate.Unlock() + continue + } else if e != nil { + m.log.Error().Msg("could update the mmdb - " + e.Error()) + update.Reset(m.mmRetryFreq) + m.muUpdate.Unlock() + continue + } + + m.log.Trace().Msg("geoip database update, old mmdb - " + m.fd.Name()) + m.log.Trace().Msg("geoip database update, new mmdb - " + newfd.Name()) + + m.log.Debug().Msg("geoip database update, rotating...") + m.rotateActiveDB(newfd, newrd) + + if !m.skipVerify { + m.log.Debug().Msg("geoip database update, verifying...") + m.Verify() + } + + m.muUpdate.Unlock() + case <-m.done(): + m.log.Info().Msg("internal abort() has been caught; initiate application closing...") + break LOOP + } + } +} + +func (m *GeoIPHTTPClient) destroyDB(mmfile *os.File, mmreader *maxminddb.Reader) { + m.log.Trace().Msg("geoip database destroy, maxmind closing...") + if e := mmreader.Close(); e != nil { + m.log.Warn().Msg("could not close maxmind reader - " + e.Error()) + } + + m.log.Trace().Msg("geoip database destroy, mmdb closing...") + if e := mmfile.Close(); e != nil { + m.log.Warn().Msg("could not close temporary geoip file - " + e.Error()) + } + + m.log.Trace().Msg("geoip database destroy, mmdb removing...") + if e := os.Remove(mmfile.Name()); e != nil { + m.log.Warn().Msg("could not remove temporary file - " + e.Error()) + } +} + +func (m *GeoIPHTTPClient) rotateActiveDB(mmfile *os.File, mmreader *maxminddb.Reader) { + m.setReady(false) + defer m.setReady(true) + + m.mu.Lock() + defer m.mu.Unlock() + + m.destroyDB(m.fd, m.Reader) + m.fd, m.Reader = mmfile, mmreader +} + +func (m *GeoIPHTTPClient) setReady(ready bool) { + m.muReady.Lock() + m.ready = ready + m.muReady.Unlock() +} + +func (m *GeoIPHTTPClient) configureHTTPClient(c *cli.Context) (_ GeoIPClient, e error) { + rrl := fasthttp.AcquireURI() + defer fasthttp.ReleaseURI(rrl) + + if e = rrl.Parse(nil, futils.UnsafeBytes(c.String("geoip-maxmind-permalink"))); e != nil { + return + } + m.mmurl = rrl.String() + + var creds []string + if creds = strings.Split(c.String("geoip-maxmind-license"), ":"); len(creds) != 2 { + e = errors.New("license format is not valid; it must be formated as `client_id:key`") + return + } else if len(creds[0]) == 0 || len(creds[1]) == 0 { + e = errors.New("license id or key is empty; record must be formated as `client_id:key`") + return + } + m.mmusername, m.mmpassword = creds[0], creds[1] + + m.hclient = &fasthttp.Client{ + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent#crawler_and_bot_ua_strings + Name: fmt.Sprintf("Mozilla/5.0 (compatible; %s/%s; +https://anilibria.top/support)", + c.App.Name, c.App.Version), + + DisableHeaderNamesNormalizing: true, + DisablePathNormalizing: true, + NoDefaultUserAgentHeader: false, + + // TODO + // ? timeouts + // ? dns cache + // ? keep alive + } + + return m, e +} + +func (m *GeoIPHTTPClient) makeTempFile() (_ *os.File, e error) { + temppath := fmt.Sprintf("%s/%s", os.TempDir(), m.tempdir) + var fstat fs.FileInfo + if fstat, e = os.Stat(temppath); e != nil { + if !os.IsNotExist(e) { + return + } + + os.MkdirAll(temppath, 0700) + } else if !fstat.IsDir() { + e = errors.New("temporary path is exists and it's not a directory - " + temppath) + return + } + + var fd *os.File + fd, e = os.CreateTemp(temppath, m.appname+"_*.mmdb") + + return fd, e +} + +func (m *GeoIPHTTPClient) acquireGeoIPRequest(parent *fasthttp.Request) (req *fasthttp.Request) { + req = fasthttp.AcquireRequest() + + if parent != nil { + parent.CopyTo(req) + } + + req.SetRequestURI(m.mmurl) + req.URI().SetUsername(m.mmusername) + req.URI().SetPassword(m.mmpassword) + + req.Header.SetUserAgent(m.hclient.Name) + + return +} + +func (m *GeoIPHTTPClient) requestWithRedirects(req *fasthttp.Request, rsp *fasthttp.Response) (e error) { + for maxRedirects := 5; ; maxRedirects-- { + if maxRedirects == 0 { + e = errors.New("maxmind responded with too many redirects, redirects count exceeded") + return + } + + m.log.Trace().Msg(req.String()) + if e = m.hclient.Do(req, rsp); e != nil { + return + } + + status := rsp.StatusCode() + if fasthttp.StatusCodeIsRedirect(status) { + m.log.Trace().Msg(rsp.String()) + m.log.Debug().Msgf("maxmind responded with redirect %d, go to %s", status, + futils.UnsafeString(rsp.Header.Peek(fasthttp.HeaderLocation))) + + req.Header.Del(fasthttp.HeaderAuthorization) + + req.SetRequestURIBytes(rsp.Header.Peek(fasthttp.HeaderLocation)) + req.URI().Parse(nil, rsp.Header.Peek(fasthttp.HeaderLocation)) + continue + } + + if status != fasthttp.StatusOK { + e = fmt.Errorf("maxmind api returned %d response", status) + m.log.Trace().Msg(rsp.String()) + return + } + + if len(rsp.Body()) == 0 { + e = errors.New("maxmind responded with empty body") + m.log.Trace().Msg(rsp.String()) + return + } + + break + } + + return +} + +func (m *GeoIPHTTPClient) databaseDownload() (fd *os.File, _ *maxminddb.Reader, e error) { + if fd, e = m.makeTempFile(); e != nil { + return + } + m.log.Debug().Msgf("file %s has been successfully allocated", fd.Name()) + + req := m.acquireGeoIPRequest(nil) + defer fasthttp.ReleaseRequest(req) + + rsp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(rsp) + + var expectedHash []byte + if !m.mmSkipHashVerify { + if expectedHash, e = m.requestSHA256(req); e != nil { + return + } + + if len(m.mmLastHash) != 0 && bytes.Equal(expectedHash, m.mmLastHash) { + e = errors.New("maxmind responded sha256 is not changed; mmdb download will be skipped") + return m.fd, m.Reader, e + } + } + + if e = m.requestWithRedirects(req, rsp); e != nil { + return + } + + if !m.mmSkipHashVerify { + var responseHash []byte + if responseHash = m.databaseSHA256Verify(rsp.Body()); len(responseHash) == 0 { + e = errors.New("databases SHA256 verification returns an empty hash") + return + } + + if !bytes.Equal(responseHash, expectedHash) { + e = errors.New("maxmind databases verification not passed, database could not be updated") + return + } + + m.log.Debug().Msg("maxmind database sha256 verification passed") + m.mmLastHash = expectedHash + } + + if e = extractTarGzArchive(m.log, fd, bytes.NewBuffer(rsp.Body())); e != nil { + return + } + + var reader *maxminddb.Reader + reader, e = maxminddb.Open(fd.Name()) + + return fd, reader, e +} diff --git a/internal/geoip/verify.go b/internal/geoip/verify.go new file mode 100644 index 0000000..3aa1765 --- /dev/null +++ b/internal/geoip/verify.go @@ -0,0 +1,49 @@ +package geoip + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + + futils "github.com/gofiber/fiber/v2/utils" + "github.com/rs/zerolog" + "github.com/valyala/fasthttp" +) + +func (*GeoIPHTTPClient) databaseSHA256Verify(payload []byte) (hash []byte) { + sha := sha256.New() + sha.Write(payload) + + hash = make([]byte, sha.Size()*2) + hex.Encode(hash, sha.Sum(nil)) + + return +} + +func (m *GeoIPHTTPClient) requestSHA256(req *fasthttp.Request) (_ []byte, e error) { + shareq := m.acquireGeoIPRequest(req) + defer fasthttp.ReleaseRequest(shareq) + + if !shareq.URI().QueryArgs().Has("suffix") { + e = errors.New("unknown maxmind url format; suffix arg is missing, sha256 verification is not possible") + return + } + shareq.URI().QueryArgs().Set("suffix", "tar.gz.sha256") + + rsp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(rsp) + + if e = m.requestWithRedirects(shareq, rsp); e != nil { + return + } + + if zerolog.GlobalLevel() <= zerolog.DebugLevel { + m.log.Trace().Msg(rsp.String()) + m.log.Debug().Msgf("maxmind respond with hash - '%s' (string)", futils.UnsafeString(rsp.Body()[:64])) + } + + hash := make([]byte, 64) + copy(hash, rsp.Body()[:64]) + + return hash, e +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 8042371..67e91b3 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -134,6 +134,7 @@ func (m *Proxy) doRequest(c *fiber.Ctx, req *fasthttp.Request, rsp *fasthttp.Res func (*Proxy) unmarshalApiResponse(c *fiber.Ctx, rsp *fasthttp.Response) (ok bool, e error) { var apirsp *utils.ApiResponseWOData if apirsp, e = utils.UnmarshalApiResponse(rsp.Body()); e != nil || apirsp == nil { + rlog(c).Warn().Msg("could not parse legacy api response - " + futils.UnsafeString(rsp.Body())) return } defer utils.ReleaseApiResponseWOData(apirsp)