v.1.6 update

some new gui features and bugfixes

CHANGES.md

@@ -6,9 +6,30 @@ Unreleased VER.
 - remove python privex helpers code (?)
 - save settings into cookies (for firefox private window mainly) + accept cookies modal (?)
 - fix dropdown update if use fomantic ui div based dropdown
-- add not (!) to text filters
 - fix sort icon to the same shevrons as main menu gui show/hide buttons  
 - upgrade procedure from GUI (?) if decided to do so use [this manual](https://stackoverflow.com/questions/32163955/how-to-run-shell-script-on-host-from-docker-container/63719458#63719458)
+- optimize and fix mail_to and log_lines queries with not(!)
+- BUG: fix process of '...and more' in stats
+- [fix of non-dockerizeed deploy issue](https://github.com/drlight17/mta-log-parser/issues/10)
+
+
+VER. 1.6
+- ~~use more then one CPU core when parsing (moment timestamp convert of related logs is variable)~~
+- ~~add ability to exclude some addresses from the TOP charts (frontend gui cookies + backend permanent using settings)~~
+- ~~add stats cached text background for better readability~~
+- ~~show all currently excluded in gui (backend in tips, frontend in gui)~~
+- ~~BUG: currently excluded unknown list fix~~
+- ~~BUG: fix currently excluded context menu (right click) doesn't appear in mobile view~~
+- ~~clear all currently excluded from cookies button~~
+- ~~BUG: exclude unknown~~
+- ~~add not (!) to text filters~~
+- ~~BUG: fix not(!) appearance in mobile mode~~
+- ~~BUGs with all stats after not(!) were added~~
+- ~~process sharelink, save and restore not (!) parameter value and from link~~
+- ~~BUG: stats_app is not defined~~
+- ~~fix "invalid data" in cached stats after cookie clear (replace with error text or copy last date to browser localstorage to use it in this case)~~
+- ~~[security fix](https://github.com/drlight17/mta-log-parser/security/dependabot/1)~~
+- ~~closed graphs and settings/tips by default~~
 
 VER. 1.5.5
 - ~~BUG: auto turn on "save filters" on every link come even without paramaters~~

Dockerfile

@@ -2,10 +2,12 @@
 FROM python:3.8-alpine
 RUN apk update
 RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
 RUN pip3 install -U pipenv
 WORKDIR /app
 COPY Pipfile /app/
 # uncomment to append ypur ca cert for TLS connections i.e. LDAPS
 #COPY ca.crt /usr/local/share/ca-certificates/ca.crt
 #RUN cat /usr/local/share/ca-certificates/ca.crt >> /etc/ssl/certs/ca-certificates.crt
-RUN pipenv install
+RUN pipenv install

Pipfile

@@ -12,8 +12,8 @@ python-dateutil = "*"
 pytz = "*"
 rethinkdb = ">=2.4.7"
 python-dotenv = "*"
-quart = "=0.18.4"
-werkzeug = "=2.3.7"
+quart = "=0.19.4"
+werkzeug = "=3.0.1"
 moment = "*"
 datefinder = "*"
 python-ldap = "*"

build/Dockerfile

@@ -2,6 +2,8 @@
 FROM python:3.8-alpine
 RUN apk update
 RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
 RUN pip3 install -U pipenv
 WORKDIR /app
 COPY . /app/

build/docker-compose.yaml

@@ -37,4 +37,4 @@ networks:
     driver: bridge
     driver_opts:
       com.docker.network.enable_ipv6: "false"
-      com.docker.network.bridge.name: mlp_iface
+      com.docker.network.bridge.name: mlp_iface

deploy/Dockerfile

@@ -0,0 +1,13 @@
+# Dockerfile for deploy
+FROM python:3.8-alpine
+RUN apk update
+RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
+RUN pip3 install -U pipenv
+WORKDIR /app
+COPY Pipfile /app/
+# uncomment to append ypur ca cert for TLS connections i.e. LDAPS
+#COPY ca.crt /usr/local/share/ca-certificates/ca.crt
+#RUN cat /usr/local/share/ca-certificates/ca.crt >> /etc/ssl/certs/ca-certificates.crt
+RUN pipenv install

deploy/docker-compose.yaml

@@ -0,0 +1,37 @@
+# compose file for deploy
+version: '3.9'
+
+services:
+  mlp:
+    image: drlight17/mta-log-parser:latest
+    container_name: mta-log-parser
+    command: /app/run.sh ${COMMAND}
+    ports:
+      - ${PORT}:${PORT}
+    restart: always
+    volumes:
+      - ./.env:/app/.env
+      - /tmp:/tmp
+      - ${MAIL_LOG_PATH}:${MAIL_LOG_PATH}
+    depends_on:
+      - rethinkdb
+    networks:
+      - internal
+
+  rethinkdb:
+    image: rethinkdb:2.4
+    container_name: rethinkdb
+    ports:
+      - ${RETHINK_EXP_GUI_PORT}:8080
+    restart: always
+    volumes:
+      - ./:/data
+    networks:
+      - internal
+
+networks:
+  internal:
+    driver: bridge
+    driver_opts:
+      com.docker.network.enable_ipv6: "false"
+      com.docker.network.bridge.name: mlp_iface

example.env

@@ -41,9 +41,18 @@ PORT=8487
 # make date time format (default is "HH:mm:ss, DD.MM.YYYY") to use in GUI for emails list timestamps and date picker according to moment.js templates (https://momentjs.com/)
 #DATETIME_FORMAT='HH:mm:ss, DD.MM.YYYY'
 
+# convert related logs timestamp according to the DATETIME_FORMAT. High CPU demanding! false by default
+#MAIL_LOG_TIMESTAMP_CONVERT=false
+
 # set MTA which logs you waunt to parse; available values are exim, postfix (default), sendmail and exchange
 #MTA=postfix
 
+# set email addresses to exclude from top senders stats permanently (comma separated)
+#EXCLUDE_FROM_TOP_SENDERS='[email protected],[email protected]'
+
+# set email addresses to exclude from top recipients stats permanently (comma separated)
+#EXCLUDE_FROM_TOP_RECIPIENTS='[email protected],[email protected]'
+
 # set startup command
 # dev to run in python developer mode
 # prod to run in production mode (default)

mlp/main.py

@@ -38,7 +38,7 @@
 
 # !!! change version upon update !!!
 global VERSION
-VERSION ="1.5.5"
+VERSION ="1.6"
 
 # postf_match += r'([A-F0-9]{11})\:[ \t]+?(.*)'
 #postf_match = r'([A-Za-z]+[ \t]+[0-9]+[ \t]+[0-9]+\:[0-9]+:[0-9]+).*'

mlp/objects.py

@@ -10,6 +10,9 @@
 from mlp.settings import log_timezone
 from mlp import settings, api
 
+import logging
+log = logging.getLogger(__name__)
+
 @dataclass
 class PostfixLog(Dictable):
     timestamp: datetime
@@ -32,14 +35,19 @@ def __str__(self):
 
     def clean_dict(self, convert_time=str) -> dict:
         data = dict(self)
-        # log lines timestamp convert before append
-        data['timestamp'] = moment.date(str(self.timestamp)).format(settings.datetime_format)
+        # log lines timestamp convert before append (for message related log view)
+        # this data conversion uses only one thread and takes too long so do it according to mail_log_timestamp_convert variable value
+        # log.info('Converting date and time %s using moment library according to settings',  data['timestamp'])
+        if settings.mail_log_timestamp_convert:
+            data['timestamp'] = moment.date(str(self.timestamp)).format(settings.datetime_format)
+        else:
         #print(data)
 
-        '''if is_false(convert_time):
-            data['timestamp'] = self.timestamp
-        else:
-            data['timestamp'] = convert_time(self.timestamp)'''
+            if is_false(convert_time):
+                data['timestamp'] = self.timestamp
+            else:
+                data['timestamp'] = convert_time(self.timestamp)
+
         return data
 
 

mlp/settings.py

@@ -73,9 +73,18 @@
 # add datetime_format var from env
 datetime_format = env('DATETIME_FORMAT', 'HH:mm:ss, DD.MM.YYYY')
 
+# add mail_log_timestamp_convert var from env
+mail_log_timestamp_convert = env_bool('MAIL_LOG_TIMESTAMP_CONVERT', False)
+
 # add mta var from env
 mta = env('MTA', '')
 
+# add exclude from top senders
+exclude_from_top_senders = env('EXCLUDE_FROM_TOP_SENDERS', '')
+
+# add exclude from top recipients
+exclude_from_top_recipients = env('EXCLUDE_FROM_TOP_RECIPIENTS', '')
+
 log_timezone = pytz.timezone(env('LOG_TIMEZONE', 'UTC'))
 """
 If your mail.log isn't in UTC, set the LOG_TIMEZONE env var to a pytz compatible timezone

mlp/static/css/style.css

@@ -18,7 +18,16 @@
 	filter: brightness(1);
 }
 
-.stats_refresh:hover {
+
+.eye.slash.icon {
+    position: absolute;
+    right: 1vh;
+    cursor: pointer;
+    opacity: .7 !important;
+    text-decoration: none;
+}
+
+.stats_refresh:hover, .eye.slash.icon:hover {
     opacity: 1!important;
 }
 
@@ -422,14 +431,25 @@ summary {
     flex: 1;
 }
 
-#by {
+#by, #equal {
     display: flex;
     align-items: center;
     justify-content: center;
     width: initial;
     margin-bottom: initial;
 }
 
+ #equal {
+    opacity: .7;
+    transition: all 0.15s ease-in-out;
+    cursor: pointer;
+ }
+
+ #equal:hover {
+    opacity: 1;
+
+ }
+
 @media only screen and (max-width: 767px){
     #user-settings, #tips {
         padding-left: 0;
@@ -450,7 +470,7 @@ summary {
         cursor: pointer;
     }
 
-    #by {
+    #by, #equal {
         width: 100%;
         margin-bottom: 15px;
     }
@@ -669,23 +689,33 @@ i.close.icon::before {
     z-index: 1;
 }
 
-/*canvas.no-data {
-  background: radial-gradient(circle at center, rgba(0,0,0,0) 0, rgba(0,0,0,0) 55%, rgba(0,0,0,0.2) 56%, rgba(0,0,0,0.2) 60%, rgba(0,0,0,0.2) 64%, rgba(0,0,0,0) 65%,rgba(0,0,0,0) 100%);
-}*/
 
-/* chartist styling*/
-/*.ct-series-a .ct-bar, .ct-series-a .ct-line, .ct-series-a .ct-point, .ct-series-a .ct-slice-donut {
-    stroke: #0CC162;
+/*canvas context menu style*/
+#app.dark .contextMenu {
+    border:1px solid #dedede!important;
+    background-color:rgba(0,0,0,.9);
+}
+
+.contextMenu{
+  position: absolute;
+  border:1px solid black;
+  background:white;
+  list-style:none;
+  border-radius: 0 6px 6px;
+  padding: 0 2px;
+  width: fit-content;
+  z-index:99999;
+
 }
-.ct-series-b .ct-bar, .ct-series-b .ct-line, .ct-series-b .ct-point, .ct-series-b .ct-slice-donut {
-    stroke: #BBBBBB;
+.menu-item {
+    font-size: 14px;
+    line-height: 24px;
+    font-family: sans-serif;
+    padding: 4px 10px;
 }
-.ct-chart {
-    margin: auto;
-    width: 300px;
-    height: 300px;
-}*/
 
-/*.ct-label {
-    font-size: 13px!important;
-}*/
+.menu-item:hover {
+  /*background: #dddddd;*/
+  text-decoration: underline;
+  cursor: pointer;
+}