add do nothing option to dummy output (#503)

* add do nothing option to dummy output

CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Improvements
 
+* a do nothing option do dummy output to ensure dummy does not fill memory
 * make the s3 connector raise `FatalOutputError` instead of warnings
 * make the s3 connector blocking by removing threading
 

logprep/abc/output.py

@@ -93,3 +93,6 @@ def store_custom(self, document: dict, target: str):
     @abstractmethod
     def store_failed(self, error_message: str, document_received: dict, document_processed: dict):
         """Store an event when an error occurred during the processing."""
+
+    def _write_backlog(self):
+        """Write the backlog to the output destination."""

logprep/connector/dummy/output.py

@@ -37,6 +37,8 @@ class DummyOutput(Output):
     class Config(Output.Config):
         """Common Configurations"""
 
+        do_nothing: bool = field(default=False)
+
         exceptions: List[str] = field(
             validator=validators.deep_iterable(
                 member_validator=validators.instance_of((str, type(None))),
@@ -83,6 +85,8 @@ def store(self, document: dict):
         document : dict
            Processed log event that will be stored.
         """
+        if self._config.do_nothing:
+            return
         if self._exceptions:
             exception = self._exceptions.pop(0)
             if exception is not None:
@@ -98,6 +102,8 @@ def store_custom(self, document: dict, target: str):
 
     def store_failed(self, error_message: str, document_received: dict, document_processed: dict):
         """Store an event when an error occurred during the processing."""
+        if self._config.do_nothing:
+            return
         self.metrics.number_of_failed_events += 1
         self.failed_events.append((error_message, document_received, document_processed))