Remove object metrics

NationalSecurityAgency · Oct 4, 2023 · ba74e39 · ba74e39
1 parent 01ad754
commit ba74e39
Show file tree

Hide file tree

Showing 6 changed files with 0 additions and 134 deletions.
diff --git a/contrib/elasticsearch/sample_objectMetrics-filebeat-prospector.yml b/contrib/elasticsearch/sample_objectMetrics-filebeat-prospector.yml
diff --git a/src/main/config/emissary.output.DropOffPlace.cfg b/src/main/config/emissary.output.DropOffPlace.cfg
@@ -35,8 +35,3 @@ OUTPUT_FILTER = "JSON:emissary.output.filter.JsonOutputFilter"
 "OUTPUT_SPEC_DATA" = "%R%/%S%.%F%.out"
 
 IMPORT_FILE = "emissary.output.DropOffUtil.cfg"
-
-OUTPUT_OBJECT_METRICS = "@ENV{'LogObjectMetrics'}"
-OBJECT_METRICS_LATENCY = ""
-OBJECT_METRICS_LATENCY_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"
-OBJECT_METRICS_FIELDS = "FILETYPE"
diff --git a/src/main/config/logback.xml b/src/main/config/logback.xml
@@ -20,34 +20,6 @@
     </encoder>
   </appender>
 
-  <!-- Enable output metrics logging by creating an environment variable of "LogObjectMetrics=true" -->
-  <if condition="&quot;${LogObjectMetrics}&quot;.equals(&quot;true&quot;)">
-    <then>
-      <appender name="OBJECT-METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
-          <customFields>{"node":"${emissary.node.name}","port":"${emissary.node.port}","application":"server"}</customFields>
-          <includeMdc>false</includeMdc>
-          <fieldNames>
-            <version>[ignore]</version>
-            <logger>[ignore]</logger>
-            <thread>[ignore]</thread>
-            <levelValue>[ignore]</levelValue>
-            <level>[ignore]</level>
-            <message>[ignore]</message>
-          </fieldNames>
-        </encoder>
-        <file>logs/${emissary.node.name}-${emissary.node.port}-object-metrics.log</file>
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-          <fileNamePattern>logs/${emissary.node.name}-${emissary.node.port}-object-metrics.log.%d{yyyy-MM-dd-HH}</fileNamePattern>
-        </rollingPolicy>
-      </appender>
-
-      <logger name="objectMetrics" level="INFO" additivity="false">
-        <appender-ref ref="OBJECT-METRICS" />
-      </logger>
-    </then>
-  </if>
-
   <!-- Enable output metrics logging by creating an environment variable of "ObjectTrace=true" -->
   <if condition="&quot;${ObjectTrace}&quot;.equals(&quot;true&quot;)">
     <then>

diff --git a/src/main/java/emissary/output/DropOffPlace.java b/src/main/java/emissary/output/DropOffPlace.java
@@ -13,12 +13,7 @@
 import emissary.util.DisposeHelper;
 import emissary.util.ShortNameComparator;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.IOException;
-import java.time.Instant;
-import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -28,25 +23,18 @@
 import java.util.Map;
 import java.util.Set;
 
-import static net.logstash.logback.marker.Markers.appendEntries;
-
 /**
  * DropOffPlace manages the output from the system It has evolved into a controller of sorts with way too many options,
  * that controls which types of output are desired and called the appropriate output helper for the desired output.
  **/
 public class DropOffPlace extends ServiceProviderPlace implements EmptyFormPlace {
 
-    private static Logger objectMetricsLog = null;
     protected boolean doSynchronized = false;
     protected Set<String> elideContentForms;
     protected Set<String> noNukeForms;
     protected List<IDropOffFilter> outputFilters = new ArrayList<>();
     protected boolean failurePolicyTerminate = true;
     protected DropOffUtil dropOffUtil;
-    private boolean outputObjectMetrics = false;
-    private List<String> objectMetricsFields = new ArrayList<>();
-    private String outputObjectMetricsLatency;
-    private DateTimeFormatter latencyDateTimeFormatter;
     private boolean outputCompletionPayloadSize = false;
 
     /**
@@ -96,17 +84,11 @@ protected void configurePlace() {
         this.dropOffUtil = new DropOffUtil(configG);
         this.doSynchronized = configG.findBooleanEntry("SYNCHRONIZED_PROCESS", false);
         this.failurePolicyTerminate = configG.findBooleanEntry("FAILURE_TERMINATES_CHAIN", true);
-        this.outputObjectMetrics = configG.findBooleanEntry("OUTPUT_OBJECT_METRICS", false);
-        this.objectMetricsFields = configG.findEntries("OBJECT_METRICS_FIELDS");
-        this.outputObjectMetricsLatency = configG.findStringEntry("OBJECT_METRICS_LATENCY", "");
-        String outputTldMetricsLatencyFormat = configG.findStringEntry("OBJECT_METRICS_LATENCY_FORMAT", "yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
-        this.latencyDateTimeFormatter = DateTimeFormatter.ofPattern(outputTldMetricsLatencyFormat);
         this.outputCompletionPayloadSize = configG.findBooleanEntry("OUTPUT_COMPLETION_PAYLOAD_SIZE", false);
         // Build and store all the filter that are desired IN THE ORDER SPECIFIED
         final List<String> filterClasses = configG.findEntries("OUTPUT_FILTER");
         initializeFilters(filterClasses);
 
-        objectMetricsLog = LoggerFactory.getLogger("objectMetrics");
     }
 
     /**
@@ -242,9 +224,6 @@ public List<IBaseDataObject> agentProcessHeavyDuty(final List<IBaseDataObject> p
             // Just report the TLD object ID
             final IBaseDataObject tld = payloadList.get(0);
 
-            if (outputObjectMetrics) {
-                objectMetricsLog.info(appendEntries(outputObjectMetrics(tld, objectMetricsFields)), "Finished DropOff");
-            }
 
             if (outputCompletionPayloadSize && tld.data() != null) {
                 logger.info(
@@ -499,35 +478,6 @@ public IDropOffFilter getFilter(final String name) {
         return null;
     }
 
-    /**
-     * Create a one line json output of specified parameters
-     *
-     * @param ibdo IBaseDataObject for the Top Level Document
-     * @param outputParams List of parameters to output
-     */
-    protected Map<String, String> outputObjectMetrics(IBaseDataObject ibdo, List<String> outputParams) {
-        // Create output map
-        Map<String, String> jsonMap = new HashMap<>();
-        jsonMap.put("InternalId", ibdo.getInternalId().toString());
-
-        long processingLatency;
-        if (outputObjectMetricsLatency.isEmpty() || !ibdo.hasParameter(outputObjectMetricsLatency)) {
-            processingLatency = Instant.now().toEpochMilli() - ibdo.getCreationTimestamp().getTime();
-        } else {
-            processingLatency = Instant.now().toEpochMilli()
-                    - Instant.from(latencyDateTimeFormatter.parse(ibdo.getStringParameter(outputObjectMetricsLatency))).toEpochMilli();
-        }
-        jsonMap.put("ProcessingLatency", String.valueOf(processingLatency));
-
-        // Add other specified Parameters
-        for (String param : outputParams) {
-            if (ibdo.hasParameter(param)) {
-                jsonMap.put(param, ibdo.getStringParameter(param));
-            }
-        }
-        return jsonMap;
-    }
-
     public DropOffUtil getDropOffUtil() {
         return this.dropOffUtil;
     }

diff --git a/src/main/resources/emissary/output/DropOffPlace.cfg b/src/main/resources/emissary/output/DropOffPlace.cfg
@@ -34,9 +34,4 @@ OUTPUT_FILTER = "emissary.output.filter.DataFilter"
 
 IMPORT_FILE = "emissary.output.DropOffUtil.cfg"
 
-OUTPUT_OBJECT_METRICS = "@ENV{'LogObjectMetrics'}"
-OBJECT_METRICS_LATENCY = ""
-OBJECT_METRICS_LATENCY_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"
-OBJECT_METRICS_FIELDS = "FILETYPE"
-
 OUTPUT_COMPLETION_PAYLOAD_SIZE = "false"
diff --git a/src/test/java/emissary/output/DropOffPlaceTest.java b/src/test/java/emissary/output/DropOffPlaceTest.java
@@ -11,7 +11,6 @@
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.read.ListAppender;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -24,13 +23,8 @@
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
-import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -148,33 +142,6 @@ void testOutputMessageHandlesNullPayloadArray() throws Exception {
         }
     }
 
-    @Test
-    void testOutputObjectMetric() throws ParseException, JsonProcessingException {
-        // Setup
-        final IBaseDataObject tld = DataObjectFactory.getInstance();
-        tld.setId("TEST-UUID-VALUE");
-        tld.setFileType("test-type");
-        tld.setParameter("FLOW", "test-flow");
-        Date currentData = new Date();
-        Date upstreamDropOff = new Date(currentData.getTime() - (30 * 60000));
-        DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
-        tld.setParameter("UPSTREAM-DROPOFF", df.format(upstreamDropOff));
-        List<String> tldMetricsFields = new ArrayList<>();
-        tldMetricsFields.add("FILETYPE");
-        tldMetricsFields.add("FLOW");
-
-        // Run
-        Map<String, String> outputMap = place.outputObjectMetrics(tld, tldMetricsFields);
-
-        // Verify
-        assertTrue(outputMap.containsKey("FILETYPE"));
-        assertEquals("test-type", outputMap.get("FILETYPE"));
-        assertTrue(outputMap.containsKey("FLOW"));
-        assertEquals("test-flow", outputMap.get("FLOW"));
-        assertTrue(outputMap.containsKey("ProcessingLatency"));
-        assertTrue(outputMap.containsKey("InternalId"));
-    }
-
     public static void cleanupDirectoryRecursively(Path path) throws IOException {
         Files.walkFileTree(path, new SimpleFileVisitor<>() {
             @Override