Adding objectTrace logger to output configurable fields to a new log file from PickUpPlace

src/main/config/logback.xml

@@ -49,6 +49,35 @@
     </then>
   </if>
 
+  <!-- Enable output metrics logging by creating an environment variable of "ObjectTrace=true" -->
+  <if condition='${ObjectTrace} == true'>
+    <then>
+      <appender name="OBJECT-TRACE" class="emissary.util.FifteenMinuteLogbackAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
+          <customFields>{"node":"${emissary.node.name}"}</customFields>
+          <includeMdc>false</includeMdc>
+          <fieldNames>
+            <version>[ignore]</version>
+            <logValue>[ignore]</logValue>
+            <logger>[ignore]</logger>
+            <thread>[ignore]</thread>
+            <levelValue>[ignore]</levelValue>
+            <level>[ignore]</level>
+            <message>[ignore]</message>
+          </fieldNames>
+        </encoder>
+        <file>logs/${emissary.node.name}-${emissary.node.port}-object-trace.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+          <fileNamePattern>logs/${emissary.node.name}-${emissary.node.port}-object-trace.log.%d{yyyyMMdd-HHmm}</fileNamePattern>
+        </rollingPolicy>
+      </appender>
+
+      <logger name="objectTrace" level="INFO" additivity="false">
+        <appender-ref ref="OBJECT-TRACE" />
+      </logger>
+    </then>
+  </if>
+
   <root level="INFO">
     <!-- stop logging to a file in development man, redirect or tee to a file
     <appender-ref ref="TOFILE"/>

src/main/java/emissary/pickup/PickUpPlace.java

@@ -17,9 +17,12 @@
 import emissary.place.ServiceProviderPlace;
 import emissary.pool.AgentPool;
 import emissary.util.ClassComparator;
+import emissary.util.ObjectTracing;
 import emissary.util.TimeUtil;
 import emissary.util.shell.Executrix;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
 
 import java.io.File;
@@ -34,6 +37,7 @@
 
 import static emissary.core.constants.Parameters.FILE_DATE;
 import static emissary.core.constants.Parameters.FILE_NAME;
+import static net.logstash.logback.marker.Markers.appendEntries;
 
 /**
  * This class is the base class of those places that inject data into the system. This place knows a lot about
@@ -77,6 +81,9 @@ public abstract class PickUpPlace extends ServiceProviderPlace implements IPickU
     // Metadata items that should always be copied to children
     protected Set<String> ALWAYS_COPY_METADATA_VALS = new HashSet<>();
 
+    private boolean useObjectTraceLogger = true;
+    protected Logger objectTraceLogger;
+
     public PickUpPlace() throws IOException {
         super();
         configurePickUpPlace();
@@ -180,6 +187,15 @@ protected void configurePickUpPlace() {
         }
 
         ALWAYS_COPY_METADATA_VALS = configG.findEntriesAsSet("ALWAYS_COPY_METADATA");
+
+        // Setup objectTraceLogger
+        useObjectTraceLogger = configG.findBooleanEntry("USE_OBJECT_TRACE_LOGGER", useObjectTraceLogger);
+        if (useObjectTraceLogger) {
+            logger.info("Setting up the object trace logger");
+            objectTraceLogger = LoggerFactory.getLogger("objectTrace");
+
+            ObjectTracing.setUpFieldNames(configG.findEntries("OBJECT_TRACE_LOGGER_FIELD_NAME"));
+        }
     }
 
     /**
@@ -552,10 +568,27 @@ protected boolean processDataObject(IBaseDataObject d, String fixedName, File th
         dataObjectCreated(d, theFile);
         logger.info("**Deploying an agent for {} and object {} forms={} simple={}", fixedName, d.getInternalId(), d.getAllCurrentForms(),
                 (simpleMode ? "simple" : ""));
+
+        // If object tracing log that agent is being deployed for fixedName (filename)
+        if (useObjectTraceLogger) {
+            objectTraceLog(d);
+        }
+
         assignToPooledAgent(d, -1L);
         return true;
     }
 
+    /**
+     * Creates an entry in the object trace log. Can be overridden if desired.
+     * 
+     * @param d The IBDO
+     */
+    public void objectTraceLog(IBaseDataObject d) {
+        objectTraceLogger.info(appendEntries(
+                ObjectTracing.createTraceMessageMap(new String[] {"PickUp", d.getFilename(), String.valueOf(System.currentTimeMillis())})).toString(),
+                "");
+    }
+
     /**
      * Parse out sessions and process data from a file
      * 

src/main/java/emissary/util/FifteenMinuteLogbackAppender.java

@@ -0,0 +1,19 @@
+package emissary.util;
+
+import ch.qos.logback.core.rolling.RollingFileAppender;
+
+public class FifteenMinuteLogbackAppender<E> extends RollingFileAppender<E> {
+    private long start = System.currentTimeMillis(); // minutes
+    private static final int ROLLOVER_TIME_MINUTES = 15;
+
+    @Override
+    public void rollover() {
+        long currentTime = System.currentTimeMillis();
+        int maxIntervalSinceLastLoggingInMillis = ROLLOVER_TIME_MINUTES * 60 * 1000;
+
+        if ((currentTime - start) >= maxIntervalSinceLastLoggingInMillis) {
+            super.rollover();
+            start = System.currentTimeMillis();
+        }
+    }
+}

src/main/java/emissary/util/ObjectTracing.java

@@ -0,0 +1,38 @@
+package emissary.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ObjectTracing {
+
+    private ObjectTracing() {}
+
+    private static final List<String> fieldNames = new ArrayList<>();
+
+    protected static final Logger logger = LoggerFactory.getLogger(ObjectTracing.class);
+
+    public static void setUpFieldNames(List<String> newFieldNames) {
+        fieldNames.addAll(newFieldNames);
+    }
+
+    public static Map<String, String> createTraceMessageMap(String[] fieldValues) {
+
+        Map<String, String> jsonMap = new HashMap<>();
+
+        if (fieldValues.length != fieldNames.size()) {
+            logger.error("Cannot create log entry, the number of fields does not equals the number of values");
+            return Collections.emptyMap();
+        }
+
+        for (int i = 0; i < fieldValues.length; i++) {
+            jsonMap.put(fieldNames.get(i), fieldValues[i]);
+        }
+        return jsonMap;
+    }
+}

src/main/resources/emissary/pickup/file/FilePickUpPlace.cfg

@@ -36,4 +36,8 @@ WIN_OUT_ROOT = "@{OUTPUT_DATA}"
 MINIMUM_DATA_SIZE = "-1"
 MAXIMUM_DATA_SIZE = "-1"
 
+USE_OBJECT_TRACE_LOGGER = true
 
+OBJECT_TRACE_LOGGER_FIELD_NAME = "stage"
+OBJECT_TRACE_LOGGER_FIELD_NAME = "inputFileName"
+OBJECT_TRACE_LOGGER_FIELD_NAME = "timestamp"

src/main/resources/emissary/util/ObjectTracing.cfg

@@ -0,0 +1,3 @@
+FIELD_NAME = "stage"
+FIELD_NAME = "inputFileName"
+FIELD_NAME = "timestamp"