goaccess 1.4 change in config file + read all access.log* files (including gzipped ones)

.deepsource.toml

@@ -0,0 +1,13 @@
+version = 1
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "shell"
+enabled = true
+
+[[analyzers]]
+name = "secrets"
+enabled = true

Dockerfile

@@ -28,6 +28,6 @@ RUN chmod +x /usr/local/bin/goaccess.sh && \
     chmod -R 777 /var/tmp/nginx
 
 EXPOSE 7889
-VOLUME [ "/config", "/opt/log" ]
+VOLUME [ "/config", "/opt/log", "/opt/data" ]
 
-CMD [ "sh", "/usr/local/bin/goaccess.sh" ]
+CMD [ "sh", "/usr/local/bin/goaccess.sh" ]

root/opt/goaccess.conf

@@ -47,6 +47,7 @@
 #date-format %f
 
 # Squid native log format
+# Caddy
 #
 #date-format %s
 
@@ -96,6 +97,9 @@
 # Kubernetes Nginx Ingress Log Format
 #log-format %^ %^ [%h] %^ %^ [%d:%t %^] "%r" %s %b "%R" "%u" %^ %^ [%v] %^:%^ %^ %T %^ %^
 
+# CADDY JSON Structured
+#log-format {ts:"%x.%^",request:{remote_addr:"%h:%^",proto:"%H",method:"%m",host:"%v",uri:"%U",headers:{"User-Agent":["%u","%^"]},tls:{cipher_suite:"%k",proto:"%K"}},duration:"%T",size:"%b",status:"%s",resp_headers:{"Content-Type":["%M;%^"]}}
+
 # In addition to specifying the raw log/date/time formats, for
 # simplicity, any of the following predefined log format names can be
 # supplied to the log/date/time-format variables. GoAccess  can  also
@@ -112,6 +116,7 @@ log-format COMBINED
 #log-format CLOUDSTORAGE
 #log-format AWSELB
 #log-format AWSS3
+#log-format CADDY
 
 ######################################
 # UI Options
@@ -318,7 +323,7 @@ real-time-html true
 
 # Path to write named pipe (FIFO).
 #
-#fifo-in /tmp/wspipeout.fifo
+#fifo-out /tmp/wspipeout.fifo
 
 ######################################
 # File Options
@@ -327,7 +332,7 @@ real-time-html true
 # Specify the path to the input log file. If set, it will take
 # priority over -f from the command line.
 #
-log-file /opt/log/access.log
+#log-file /opt/log/access.log /opt/log/access.log.1
 
 # Send all debug messages to the specified file.
 #
@@ -411,11 +416,6 @@ no-term-resolver false
 #
 4xx-to-unique-count false
 
-# Store accumulated processing time from parsing day-by-day logs.
-# Only if configured with --enable-tcb=btree
-#
-#accumulated-time false
-
 # IP address anonymization
 # The IP anonymization option sets the last octet of IPv4 user IP addresses and
 # the last 80 bits of IPv6 addresses to zeros.
@@ -460,12 +460,15 @@ double-decode false
 #enable-panel KEYPHRASES
 #enable-panel STATUS_CODES
 #enable-panel REMOTE_USER
+#enable-panel CACHE_STATUS
 #enable-panel GEO_LOCATION
+#enable-panel MIME_TYPE
+#enable-panel TLS_TYPE
 
-# Hide a referer but still count it. Wild cards are allowed. i.e., *.bing.com
+# Hide a referrer but still count it. Wild cards are allowed. i.e., *.bing.com
 #
-#hide-referer *.google.com
-#hide-referer bing.com
+#hide-referrer *.google.com
+#hide-referrer bing.com
 
 # Hour specificity. Possible values: `hr` (default), or `min` (tenth
 # of a minute).
@@ -508,21 +511,29 @@ ignore-panel REFERRERS
 ignore-panel KEYPHRASES
 #ignore-panel STATUS_CODES
 #ignore-panel REMOTE_USER
+#ignore-panel CACHE_STATUS
 #ignore-panel GEO_LOCATION
+#ignore-panel MIME_TYPE
+#ignore-panel TLS_TYPE
 
-# Ignore referers from being counted.
+# Ignore referrers from being counted.
 # This supports wild cards. For instance,
 # '*' matches 0 or more characters (including spaces)
 # '?' matches exactly one character
 #
-#ignore-referer *.domain.com
-#ignore-referer ww?.domain.*
+#ignore-referrer *.domain.com
+#ignore-referrer ww?.domain.*
 
 # Ignore parsing and displaying one or multiple status code(s)
 #
 #ignore-status 400
 #ignore-status 502
 
+# Keep the last specified number of days in storage. This will recycle the
+# storage tables. e.g., keep & show only the last 7 days.
+#
+keep-last 365
+
 # Disable client IP validation. Useful if IP addresses have been
 # obfuscated before being logged.
 #
@@ -575,7 +586,10 @@ real-os true
 #sort-panel KEYPHRASES,BY_HITS,ASC
 #sort-panel STATUS_CODES,BY_HITS,ASC
 #sort-panel REMOTE_USER,BY_HITS,ASC
+#sort-panel CACHE_STATUS,BY_HITS,ASC
 #sort-panel GEO_LOCATION,BY_HITS,ASC
+#sort-panel MIME_TYPE,BY_HITS,ASC
+#sort-panel TLS_TYPE,BY_HITS,ASC
 
 # Consider the following extensions as static files
 # The actual '.' is required and extensions are case sensitive
@@ -618,6 +632,9 @@ static-file .tiff
 static-file .tif
 static-file .ttf
 static-file .flv
+static-file .dmg
+static-file .xz
+static-file .zst
 #static-file .less
 #static-file .ac3
 #static-file .avi
@@ -656,119 +673,59 @@ static-file .flv
 # Only if configured with --enable-geoip
 ######################################
 
-# Standard GeoIP database for less memory usage.
-#
-# std-geoip true
-
-# Specify path to GeoIP database file. i.e., GeoLiteCity.dat
-# .dat file needs to be downloaded from maxmind.com.
+# To feed a database either through GeoIP Legacy or GeoIP2, you need to use the
+# geoip-database flag below.
 #
-# For IPv4 City database:
-# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
-# gunzip GeoLiteCity.dat.gz
+# === GeoIP Legacy
+# Legacy GeoIP has been discontinued. If your GNU+Linux distribution does not ship
+# with the legacy databases, you may still be able to find them through
+# different sources. Make sure to download the .dat files.
 #
-# For IPv6 City database:
-# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz
-# gunzip GeoLiteCityv6.dat.gz
+# Distributed with Creative Commons Attribution-ShareAlike 4.0 International License.
+# https://mailfud.org/geoip-legacy/
+
+# IPv4 Country database:
+# Download the GeoIP.dat.gz
+# gunzip GeoIP.dat.gz
 #
-# For IPv6 Country database:
-# wget -N http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
-# gunzip GeoIPv6.dat.gz
+# IPv4 City database:
+# Download the GeoIPCity.dat.gz
+# gunzip GeoIPCity.dat.gz
+
+# Standard GeoIP database for less memory usage (GeoIP Legacy).
 #
+#std-geoip false
+
+# === GeoIP2
+# For GeoIP2 databases, you can use DB-IP Lite databases.
+# DB-IP is licensed under a Creative Commons Attribution 4.0 International License.
+# https://db-ip.com/db/lite.php
+
+# Or you can download them from MaxMind
+# https://dev.maxmind.com/geoip/geoip2/geolite2/
+
 # For GeoIP2 City database:
-# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
+# Download the GeoLite2-City.mmdb.gz
 # gunzip GeoLite2-City.mmdb.gz
 #
 # For GeoIP2 Country database:
-# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz
+# Download the GeoLite2-Country.mmdb.gz
 # gunzip GeoLite2-Country.mmdb.gz
 #
-# Note: `geoip-city-data` is an alias of `geoip-database`
-#
+#geoip-database /usr/local/share/GeoIP/GeoLiteCity.dat
 geoip-database /usr/local/share/GeoIP/GeoLite2-City.mmdb
-
 ######################################
-# Tokyo Cabinet Options
-# Only if configured with --enable-tcb=btree
+# Persistence Options
 ######################################
 
-# GoAccess has the ability to process logs incrementally through the on-disk
-# B+Tree database.
-#
-# It works in the following way:
-# - A data set must be persisted first with --keep-db-files, then the same data
-#   set can be loaded with --load-from-disk.
-# - If new data is passed (piped or through a log file), it will append it to
-#   the original data set.
-# - To preserve the data at all times, --keep-db-files must be used.
-# - If --load-from-disk is used without --keep-db-files, database files will be
-#   deleted upon closing the program.
-
-# On-disk B+ Tree
-# Persist parsed data into disk. This should be set to
-# the first dataset prior to use `load-from-disk`.
-# Setting it to false will delete all database files
-# when exiting the program.
-#keep-db-files true
-
-# On-disk B+ Tree
-# Load previously stored data from disk.
-# Database files need to exist. See `keep-db-files`.
-#load-from-disk false
+# Path where the persisted database files are stored on disk.
+# The default value is the /tmp directory.
+db-path /tmp
 
-# On-disk B+ Tree
-# Path where the on-disk database files are stored.
-# The default value is the /tmp/ directory
-# Note the trailing forward-slash.
-#
-#db-path /tmp/
-
-# On-disk B+ Tree
-# Set the size in bytes of the extra mapped memory.
-# The default value is 0.
-#
-#xmmap 0
-
-# On-disk B+ Tree
-# Max number of leaf nodes to be cached.
-# Specifies the maximum number of leaf nodes to be cached.
-# If it is not more than 0, the default value is specified.
-# The default value is 1024.
-#
-#cache-lcnum 1024
-
-# On-disk B+ Tree
-# Specifies the maximum number of non-leaf nodes to be cached.
-# If it is not more than 0, the default value is specified.
-# The default value is 512.
-#
-#cache-ncnum 512
+# Persist parsed data into disk.
+persist true
 
-# On-disk B+ Tree
-# Specifies the number of members in each leaf page.
-# If it is not more than 0, the default value is specified.
-# The default value is 128.
-#
-#tune-lmemb 128
-
-# On-disk B+ Tree
-# Specifies the number of members in each non-leaf page.
-# If it is not more than 0, the default value is specified.
-# The default value is 256.
-#
-#tune-nmemb 256
-
-# On-disk B+ Tree
-# Specifies the number of elements of the bucket array.
-# If it is not more than 0, the default value is specified.
-# The default value is 32749.
-# Suggested size of the bucket array is about from 1 to 4
-# times of the number of all pages to be stored.
-#
-#tune-bnum 32749
+# Load previously stored data from disk.
+# Database files need to exist. See `persist`.
+restore true
 
-# On-disk B+ Tree
-# Specifies that each page is compressed with ZLIB|BZ2 encoding.
-# Disabled by default.
-#
-#compression zlib

root/usr/local/bin/goaccess.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -x
 
 echo -e "Variables set:\\n\
 PUID=${PUID}\\n\
@@ -20,4 +20,4 @@ chmod -R 777 /config
 
 # ready to go
 /sbin/tini -s -- nginx -c /opt/nginx.conf
-/sbin/tini -s -- goaccess --no-global-config --config-file=/config/goaccess.conf
+/sbin/tini -s -- zcat /opt/log/access.log.*.gz | goaccess - /opt/log/access.log /opt/log/access.log.1 --no-global-config --config-file=/config/goaccess.conf