From 73dcbe023d7d911092503653a35c10edd49376a7 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Thu, 31 Mar 2022 12:11:41 +0200
Subject: [PATCH 1/8] [parser] Parse hostname instead of hard-coding it

---
 maillogger/parser.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/maillogger/parser.py b/maillogger/parser.py
index f423b12..a3aec03 100644
--- a/maillogger/parser.py
+++ b/maillogger/parser.py
@@ -5,7 +5,7 @@
 
 
 REGEX = r'(?P<month>[A-Z][a-z]{2}) (?P<day>[0-9]{,2}) ' \
-    + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) mail postfix/[a-z]+\[[0-9]+\]: ' \
+    + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) (?P<hostname>[A-Za-z0-9-]+) postfix/[a-z]+\[[0-9]+\]: ' \
     + r'(?P<mail_id>[A-Z0-9]+): to=<(?P<to_address>.*@.*)>, ' \
     + r'relay=(?P<relay>.*), delay=(?P<delay>[0-9.]+), ' \
     + r'delays=(?P<delays>[0-9][0-9/.]+), dsn=(?P<dsn>[0-9].[0-9].[0-9]), ' \
@@ -53,6 +53,7 @@ class ParseResult:
     month: InitVar[str]
     day: InitVar[str]
     time: InitVar[str]
+    hostname: InitVar[str]
 
     mail_id: str
     to_address: str
@@ -65,7 +66,7 @@ class ParseResult:
 
     datetime: str = field(init=False)
 
-    def __post_init__(self, month: str, day: str, time: str) -> None:
+    def __post_init__(self, month: str, day: str, time: str, hostname: str) -> None:
         self.datetime = self.convert2dateime(month, day, time)
 
     def to_dict(self) -> ParseResultType:

From 942b94bcee7a3f61ce6b847bb5bd1f3126a44cc6 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Thu, 31 Mar 2022 12:12:46 +0200
Subject: [PATCH 2/8] [parser] Allow / in postfix service name

Postfix uses a hierarchical service name by now, e.g. postfix/smtps/smtpd.
---
 maillogger/parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maillogger/parser.py b/maillogger/parser.py
index a3aec03..f3818f9 100644
--- a/maillogger/parser.py
+++ b/maillogger/parser.py
@@ -5,7 +5,7 @@
 
 
 REGEX = r'(?P<month>[A-Z][a-z]{2}) (?P<day>[0-9]{,2}) ' \
-    + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) (?P<hostname>[A-Za-z0-9-]+) postfix/[a-z]+\[[0-9]+\]: ' \
+    + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) (?P<hostname>[A-Za-z0-9-]+) postfix/[a-z/]+\[[0-9]+\]: ' \
     + r'(?P<mail_id>[A-Z0-9]+): to=<(?P<to_address>.*@.*)>, ' \
     + r'relay=(?P<relay>.*), delay=(?P<delay>[0-9.]+), ' \
     + r'delays=(?P<delays>[0-9][0-9/.]+), dsn=(?P<dsn>[0-9].[0-9].[0-9]), ' \

From cfa86033c0f52ba9feef761398dc013bb585c4cb Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Wed, 6 Apr 2022 13:30:21 +0200
Subject: [PATCH 3/8] [parser] Allow one-digit days

---
 maillogger/parser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/maillogger/parser.py b/maillogger/parser.py
index f3818f9..1d3fa58 100644
--- a/maillogger/parser.py
+++ b/maillogger/parser.py
@@ -4,7 +4,7 @@
 from typing import Dict, Optional
 
 
-REGEX = r'(?P<month>[A-Z][a-z]{2}) (?P<day>[0-9]{,2}) ' \
+REGEX = r'(?P<month>[A-Z][a-z]{2}) +(?P<day>[0-9]{,2}) ' \
     + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) (?P<hostname>[A-Za-z0-9-]+) postfix/[a-z/]+\[[0-9]+\]: ' \
     + r'(?P<mail_id>[A-Z0-9]+): to=<(?P<to_address>.*@.*)>, ' \
     + r'relay=(?P<relay>.*), delay=(?P<delay>[0-9.]+), ' \
@@ -74,5 +74,6 @@ def to_dict(self) -> ParseResultType:
 
     @staticmethod
     def convert2dateime(month: str, day: str, time: str) -> str:
+        day = day.rjust(2, '0')
         tmp = datetime.strptime(f'{month}{day}{time}', '%b%d%H:%M:%S')
         return tmp.replace(year=datetime.now().year).strftime('%Y%m%d%H%M%S')

From 090555f7652c6eb9a179685cbb6b0b678589c50a Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Thu, 31 Mar 2022 12:47:37 +0200
Subject: [PATCH 4/8] [parser] Also parse from=<> lines

---
 maillogger/parser.py | 66 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 16 deletions(-)

diff --git a/maillogger/parser.py b/maillogger/parser.py
index 1d3fa58..4dcb2f0 100644
--- a/maillogger/parser.py
+++ b/maillogger/parser.py
@@ -3,14 +3,22 @@
 from datetime import datetime
 from typing import Dict, Optional
 
-
-REGEX = r'(?P<month>[A-Z][a-z]{2}) +(?P<day>[0-9]{,2}) ' \
+REGEX_PREFIX = r'(?P<month>[A-Z][a-z]{2}) +(?P<day>[0-9]{,2}) ' \
     + r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}) (?P<hostname>[A-Za-z0-9-]+) postfix/[a-z/]+\[[0-9]+\]: ' \
-    + r'(?P<mail_id>[A-Z0-9]+): to=<(?P<to_address>.*@.*)>, ' \
+    + r'(?P<mail_id>[A-Z0-9]+): '
+
+REGEX_FROM = REGEX_PREFIX \
+    + r'from=<(?P<from_address>.*@.*)>, size=(?P<size>[0-9]+), ' \
+    + r'nrcpt=(?P<nrcpt>[0-9]+) \((?P<description>.*)\)' \
+
+REGEX_TO = REGEX_PREFIX \
+    + r'to=<(?P<to_address>.*@.*)>, ' \
     + r'relay=(?P<relay>.*), delay=(?P<delay>[0-9.]+), ' \
     + r'delays=(?P<delays>[0-9][0-9/.]+), dsn=(?P<dsn>[0-9].[0-9].[0-9]), ' \
     + r'status=(?P<status>(sent|deferred|bounced)) \((?P<description>.*)\)'
-PATTERN = re.compile(REGEX)
+
+PATTERN_FROM = re.compile(REGEX_FROM)
+PATTERN_TO = re.compile(REGEX_TO)
 
 ParseResultType = Dict[str, str]
 
@@ -22,7 +30,7 @@ def parse(target: str) -> Optional[ParseResultType]:
         target (str): maillog
 
     Returns:
-        Optional[ParseResultType]: return the following dict if match
+        Optional[ParseResultType]: return one of the following dict if match
 
         {
             'month': 'Aug',
@@ -37,15 +45,30 @@ def parse(target: str) -> Optional[ParseResultType]:
             'status': 'sent',
             'description': 'delivered to maildir'
         }
+
+        {
+            'month': 'Aug',
+            'day': '1',
+            'time': '10:00:00',
+            'mail_id': '677RGS0',
+            'from_address': 'dummy@gmail.com',
+            'size': '12345',
+            'nrcpt': '12',
+            'description': 'delivered to maildir'
+        }
     """
 
-    match_obj = re.search(PATTERN, target)
+    match_from = re.search(PATTERN_FROM, target)
+    match_to = re.search(PATTERN_TO, target)
 
-    if match_obj is None:
-        return None
+    if match_from:
+        result = match_from.groupdict()
+        return ParseResultFrom(**result).to_dict()
+    if match_to:
+        result = match_to.groupdict()
+        return ParseResultTo(**result).to_dict()
 
-    result = match_obj.groupdict()
-    return ParseResult(**result).to_dict()
+    return None
 
 
 @dataclass
@@ -56,12 +79,6 @@ class ParseResult:
     hostname: InitVar[str]
 
     mail_id: str
-    to_address: str
-    relay: str
-    delay: str
-    delays: str
-    dsn: str
-    status: str
     description: str
 
     datetime: str = field(init=False)
@@ -77,3 +94,20 @@ def convert2dateime(month: str, day: str, time: str) -> str:
         day = day.rjust(2, '0')
         tmp = datetime.strptime(f'{month}{day}{time}', '%b%d%H:%M:%S')
         return tmp.replace(year=datetime.now().year).strftime('%Y%m%d%H%M%S')
+
+
+@dataclass
+class ParseResultTo(ParseResult):
+    to_address: str
+    relay: str
+    delay: str
+    delays: str
+    dsn: str
+    status: str
+
+
+@dataclass
+class ParseResultFrom(ParseResult):
+    from_address: str
+    size: str
+    nrcpt: str

From 7b294d69a6fcd5b3acdf6a741eea0a3066015167 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Wed, 6 Apr 2022 12:51:37 +0200
Subject: [PATCH 5/8] Disable parsing from lines for now if writing CSV/TSV

The schema is entirely different, so we cannot put it in a uniform file.
---
 maillogger/main.py   | 7 ++++++-
 maillogger/parser.py | 6 +++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/maillogger/main.py b/maillogger/main.py
index 5ec960d..99058a4 100644
--- a/maillogger/main.py
+++ b/maillogger/main.py
@@ -10,9 +10,14 @@ def main() -> None:
     loader = Loader(options.source_file)
     contents = loader.handle()
 
+    parse_to = True
+    parse_from = True
+    if options.fmt in ("csv", "tsv"):
+        parse_from = False
+
     parsed_contents = []
     for c in contents:
-        result = parse(c)
+        result = parse(c, parse_to, parse_from)
         if result:
             parsed_contents.append(result)
 
diff --git a/maillogger/parser.py b/maillogger/parser.py
index 4dcb2f0..040b70a 100644
--- a/maillogger/parser.py
+++ b/maillogger/parser.py
@@ -23,7 +23,7 @@
 ParseResultType = Dict[str, str]
 
 
-def parse(target: str) -> Optional[ParseResultType]:
+def parse(target: str, parse_to: bool = True, parse_from: bool = True) -> Optional[ParseResultType]:
     """Parse postfix maillog including send status
 
     Args:
@@ -61,10 +61,10 @@ def parse(target: str) -> Optional[ParseResultType]:
     match_from = re.search(PATTERN_FROM, target)
     match_to = re.search(PATTERN_TO, target)
 
-    if match_from:
+    if parse_from and match_from:
         result = match_from.groupdict()
         return ParseResultFrom(**result).to_dict()
-    if match_to:
+    if parse_to and match_to:
         result = match_to.groupdict()
         return ParseResultTo(**result).to_dict()
 

From 15b9eb529906751eac0a1545df61a44044222776 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Wed, 6 Apr 2022 13:40:23 +0200
Subject: [PATCH 6/8] [analyze] Implement grouping of mails by mail id

---
 maillogger/analyze.py     | 27 +++++++++++++++++++++++++++
 maillogger/cli.py         | 13 ++++++++++++-
 maillogger/file/writer.py | 14 ++++++++------
 maillogger/main.py        |  4 ++++
 4 files changed, 51 insertions(+), 7 deletions(-)
 create mode 100644 maillogger/analyze.py

diff --git a/maillogger/analyze.py b/maillogger/analyze.py
new file mode 100644
index 0000000..49b6400
--- /dev/null
+++ b/maillogger/analyze.py
@@ -0,0 +1,27 @@
+from typing import Dict, List
+
+from maillogger.parser import ParseResultType
+
+
+def group_by_mail_id(results: List[ParseResultType]) -> Dict[str, List[ParseResultType]]:
+    """Group a list of parse results into a dict by mail_id
+
+    Args:
+        results (List[ParseResultType]): List of parse results as dicts
+
+    Returns:
+        Dict[str, List[ParseResultType]]: return a dictionary with the mail_id as key
+
+        {
+            '677RGS0': [
+                {...}, {...}
+            ]
+        }
+    """
+
+    groups = {}
+
+    for result in results:
+        groups.setdefault(result['mail_id'], []).append(result)
+
+    return groups
diff --git a/maillogger/cli.py b/maillogger/cli.py
index 1e30204..6fd3649 100644
--- a/maillogger/cli.py
+++ b/maillogger/cli.py
@@ -33,6 +33,12 @@ def setup_options(parser: argparse.ArgumentParser) -> None:
         help='File data format to write the parsed maillog (Default: csv)'
     )
 
+    parser.add_argument(
+        '-g', '--group',
+        action='store_true',
+        help='Group results by mail id (only available in JSON format)'
+    )
+
     parser.add_argument(
         '-c', '--compress',
         action='store_true',
@@ -50,4 +56,9 @@ def setup_options(parser: argparse.ArgumentParser) -> None:
 def parse_options(args: Optional[List[str]] = None) -> argparse.Namespace:
     parser = get_parser()
     setup_options(parser)
-    return parser.parse_args(args=args)
+
+    options = parser.parse_args(args=args)
+    if options.group and options.fmt in ("csv", "tsv"):
+        parser.error("Grouping by mail id can only be used with JSON output.")
+
+    return options
diff --git a/maillogger/file/writer.py b/maillogger/file/writer.py
index 36d0919..a7e9b26 100644
--- a/maillogger/file/writer.py
+++ b/maillogger/file/writer.py
@@ -1,12 +1,14 @@
 import csv
 import json
 from dataclasses import dataclass
-from typing import Any, ClassVar, List, Type
+from typing import Any, ClassVar, Dict, List, Type, Union
 
 from maillogger.exceptions import UnsupportedDataFormatError
 from maillogger.file.base import FileHandler
 from maillogger.parser import ParseResultType
 
+OutputResultType = Union[Dict[str, List[ParseResultType]], List[ParseResultType]]
+
 
 @dataclass
 class FileWriter(FileHandler):
@@ -32,7 +34,7 @@ def add_file_ext(self) -> None:
             ext = f'{self.ext}.{self.gz_ext}'
         self.filepath = f'{self.filepath}.{ext}'
 
-    def handle(self, records: List[ParseResultType]) -> None:
+    def handle(self, records: OutputResultType) -> None:
         if not records:
             return
 
@@ -51,7 +53,7 @@ class CsvWriter(FileWriter):
 
     newline: str = ''
 
-    def write(self, records: List[ParseResultType]) -> None:
+    def write(self, records: OutputResultType) -> None:
         writer = csv.DictWriter(self.fd, fieldnames=list(records[0].keys()))
         writer.writeheader()
         writer.writerows(records)
@@ -64,7 +66,7 @@ class JsonWriter(FileWriter):
     ensure_ascii: bool = False
     indent: int = 2
 
-    def write(self, records: List[ParseResultType]) -> None:
+    def write(self, records: OutputResultType) -> None:
         json.dump(
             records,
             self.fd,  # type: ignore
@@ -76,7 +78,7 @@ def write(self, records: List[ParseResultType]) -> None:
 class TsvWriter(FileWriter):
     ext = 'tsv'
 
-    def write(self, records: List[ParseResultType]) -> None:
+    def write(self, records: OutputResultType) -> None:
         header = '\t'.join(records[0].keys())
         self.fd.write(f'{header}\n')  # type: ignore
 
@@ -101,7 +103,7 @@ def get_writer(filepath: str, fmt: str, **kwargs: Any) -> Type[FileWriter]:
 
 def write(
         filepath: str,
-        records: List[ParseResultType],
+        records: OutputResultType,
         fmt: str,
         **kwargs: Any) -> None:
     fmt = fmt.lower()
diff --git a/maillogger/main.py b/maillogger/main.py
index 99058a4..be44ee4 100644
--- a/maillogger/main.py
+++ b/maillogger/main.py
@@ -1,3 +1,4 @@
+from maillogger.analyze import group_by_mail_id
 from maillogger.cli import parse_options
 from maillogger.file.loader import Loader
 from maillogger.file.writer import write
@@ -21,6 +22,9 @@ def main() -> None:
         if result:
             parsed_contents.append(result)
 
+    if options.group:
+        parsed_contents = group_by_mail_id(parsed_contents)
+
     write(
         filepath=options.target_file, records=parsed_contents,
         fmt=options.fmt, compress=options.compress)

From d78e9db994033b10f3a4b9460b21f4fda1d22e63 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Wed, 6 Apr 2022 13:44:54 +0200
Subject: [PATCH 7/8] Add myself as copyright holder

---
 LICENSE | 1 +
 1 file changed, 1 insertion(+)

diff --git a/LICENSE b/LICENSE
index 37aaa8a..612b3e8 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2020 homoluctus
+Copyright (c) 2022 Dominik George <nik@velocitux.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

From 6cee23694745aab4a4b64f2b3f7b34bbc7718081 Mon Sep 17 00:00:00 2001
From: Dominik George <nik@velocitux.com>
Date: Wed, 6 Apr 2022 14:14:05 +0200
Subject: [PATCH 8/8] [analyze] Implement basic aggregation by mail id

---
 maillogger/analyze.py     | 57 ++++++++++++++++++++++++++++++++++++---
 maillogger/cli.py         |  6 +++++
 maillogger/file/writer.py |  3 ++-
 maillogger/main.py        |  7 +++--
 4 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/maillogger/analyze.py b/maillogger/analyze.py
index 49b6400..4e7df3a 100644
--- a/maillogger/analyze.py
+++ b/maillogger/analyze.py
@@ -1,16 +1,20 @@
+from dataclasses import asdict, dataclass, field
 from typing import Dict, List
 
-from maillogger.parser import ParseResultType
+from maillogger.parser import ParseResultType, ParseResultTo
 
+GroupedResultType = Dict[str, List[ParseResultType]]
+AggregateResultType = List[Dict[str, str]]
 
-def group_by_mail_id(results: List[ParseResultType]) -> Dict[str, List[ParseResultType]]:
+
+def group_by_mail_id(results: List[ParseResultType]) -> GroupedResultType:
     """Group a list of parse results into a dict by mail_id
 
     Args:
         results (List[ParseResultType]): List of parse results as dicts
 
     Returns:
-        Dict[str, List[ParseResultType]]: return a dictionary with the mail_id as key
+        GroupedResultType: return a dictionary with the mail_id as key
 
         {
             '677RGS0': [
@@ -25,3 +29,50 @@ def group_by_mail_id(results: List[ParseResultType]) -> Dict[str, List[ParseResu
         groups.setdefault(result['mail_id'], []).append(result)
 
     return groups
+
+
+def aggregate(groups: GroupedResultType) -> AggregateResultType:
+    """Aggregate all results per mail id into one item
+
+    Args:
+        groups (GroupedResultType): dict of lists grouped by mail id
+
+    Returns:
+        AggregateResultType: dictionary with one dict keyed by mail id
+    """
+
+    aggregates = {}
+
+    for mail_id, records in groups.items():
+        for record in records:
+            aggregates.setdefault(mail_id, AggregateResult(mail_id)).update(record)
+
+    return [aggregate.to_dict() for aggregate in aggregates.values()]
+
+
+@dataclass
+class AggregateResult:
+    mail_id: str
+
+    from_address: str = ''
+    to_addresses: List[str] = field(default_factory=list)
+
+    size: str = '0'
+
+    def to_dict(self) -> ParseResultType:
+        return asdict(self)
+
+    def update(self, record: ParseResultType) -> None:
+        if not self.mail_id:
+            self.mail_id = record["mail_id"]
+        elif record["mail_id"] != self.mail_id:
+            raise ValueError("Trying to aggregate different mail ids!")
+
+        if "from_address" in record:
+            self.from_address = record["from_address"]
+
+        if "size" in record:
+            self.size = record["size"]
+
+        if "to_address" in record:
+            self.to_addresses.append(record["to_address"])
diff --git a/maillogger/cli.py b/maillogger/cli.py
index 6fd3649..c8ca45a 100644
--- a/maillogger/cli.py
+++ b/maillogger/cli.py
@@ -39,6 +39,12 @@ def setup_options(parser: argparse.ArgumentParser) -> None:
         help='Group results by mail id (only available in JSON format)'
     )
 
+    parser.add_argument(
+        '-a', '--aggregate',
+        action='store_true',
+        help='Aggregate results by mail id'
+    )
+
     parser.add_argument(
         '-c', '--compress',
         action='store_true',
diff --git a/maillogger/file/writer.py b/maillogger/file/writer.py
index a7e9b26..464c1d3 100644
--- a/maillogger/file/writer.py
+++ b/maillogger/file/writer.py
@@ -3,11 +3,12 @@
 from dataclasses import dataclass
 from typing import Any, ClassVar, Dict, List, Type, Union
 
+from maillogger.analyze import AggregateResultType, GroupedResultType
 from maillogger.exceptions import UnsupportedDataFormatError
 from maillogger.file.base import FileHandler
 from maillogger.parser import ParseResultType
 
-OutputResultType = Union[Dict[str, List[ParseResultType]], List[ParseResultType]]
+OutputResultType = Union[GroupedResultType, List[ParseResultType], AggregateResultType]
 
 
 @dataclass
diff --git a/maillogger/main.py b/maillogger/main.py
index be44ee4..f9c21b9 100644
--- a/maillogger/main.py
+++ b/maillogger/main.py
@@ -1,4 +1,4 @@
-from maillogger.analyze import group_by_mail_id
+from maillogger.analyze import aggregate, group_by_mail_id
 from maillogger.cli import parse_options
 from maillogger.file.loader import Loader
 from maillogger.file.writer import write
@@ -22,9 +22,12 @@ def main() -> None:
         if result:
             parsed_contents.append(result)
 
-    if options.group:
+    if options.group or options.aggregate:
         parsed_contents = group_by_mail_id(parsed_contents)
 
+    if options.aggregate:
+        parsed_contents = aggregate(parsed_contents)
+
     write(
         filepath=options.target_file, records=parsed_contents,
         fmt=options.fmt, compress=options.compress)