From 785b2c3c71aa7eb954bc1f7e3dc27f3dd0c726f5 Mon Sep 17 00:00:00 2001 From: Amanda P Date: Thu, 3 Aug 2023 17:04:46 +0000 Subject: [PATCH 1/8] Adding objectTrace logger to output configurable fields to a new log file from PickUpPlace --- src/main/config/logback.xml | 29 +++++++++ .../java/emissary/pickup/PickUpPlace.java | 33 +++++++++++ .../util/FifteenMinuteLogbackAppender.java | 19 ++++++ .../java/emissary/util/ObjectTracing.java | 59 +++++++++++++++++++ .../emissary/pickup/file/FilePickUpPlace.cfg | 4 ++ .../resources/emissary/util/ObjectTracing.cfg | 3 + 6 files changed, 147 insertions(+) create mode 100644 src/main/java/emissary/util/FifteenMinuteLogbackAppender.java create mode 100644 src/main/java/emissary/util/ObjectTracing.java create mode 100644 src/main/resources/emissary/util/ObjectTracing.cfg diff --git a/src/main/config/logback.xml b/src/main/config/logback.xml index e3e7aca330..5cb6ae97d9 100644 --- a/src/main/config/logback.xml +++ b/src/main/config/logback.xml @@ -49,6 +49,35 @@ + + + + + + {"node":"${emissary.node.name}"} + false + + [ignore] + [ignore] + [ignore] + [ignore] + [ignore] + [ignore] + [ignore] + + + logs/${emissary.node.name}-${emissary.node.port}-object-trace.log + + logs/${emissary.node.name}-${emissary.node.port}-object-trace.log.%d{yyyyMMdd-HHmm} + + + + + + + + + - + {"node":"${emissary.node.name}"} false diff --git a/src/main/java/emissary/pickup/PickUpPlace.java b/src/main/java/emissary/pickup/PickUpPlace.java index d24bfb2c8a..96e7a5a03f 100755 --- a/src/main/java/emissary/pickup/PickUpPlace.java +++ b/src/main/java/emissary/pickup/PickUpPlace.java @@ -193,8 +193,6 @@ protected void configurePickUpPlace() { if (useObjectTraceLogger) { logger.info("Setting up the object trace logger"); objectTraceLogger = LoggerFactory.getLogger("objectTrace"); - - ObjectTracing.setUpFieldNames(configG.findEntries("OBJECT_TRACE_LOGGER_FIELD_NAME")); } } @@ -584,9 +582,7 @@ protected boolean processDataObject(IBaseDataObject d, String fixedName, File th * @param d The IBDO */ public void objectTraceLog(IBaseDataObject d) { - objectTraceLogger.info(appendEntries( - ObjectTracing.createTraceMessageMap(new String[] {"PickUp", d.getFilename(), String.valueOf(System.currentTimeMillis())})).toString(), - ""); + objectTraceLogger.info(appendEntries(ObjectTracing.createTraceMessageMap(d)), ""); } /** diff --git a/src/main/java/emissary/util/FifteenMinuteLogbackAppender.java b/src/main/java/emissary/util/FifteenMinuteLogbackAppender.java deleted file mode 100644 index 909aff490e..0000000000 --- a/src/main/java/emissary/util/FifteenMinuteLogbackAppender.java +++ /dev/null @@ -1,19 +0,0 @@ -package emissary.util; - -import ch.qos.logback.core.rolling.RollingFileAppender; - -public class FifteenMinuteLogbackAppender extends RollingFileAppender { - private long start = System.currentTimeMillis(); // minutes - private static final int ROLLOVER_TIME_MINUTES = 15; - - @Override - public void rollover() { - long currentTime = System.currentTimeMillis(); - int maxIntervalSinceLastLoggingInMillis = ROLLOVER_TIME_MINUTES * 60 * 1000; - - if ((currentTime - start) >= maxIntervalSinceLastLoggingInMillis) { - super.rollover(); - start = System.currentTimeMillis(); - } - } -} diff --git a/src/main/java/emissary/util/ObjectTracing.java b/src/main/java/emissary/util/ObjectTracing.java index f0d25f1ecc..33622d09f9 100644 --- a/src/main/java/emissary/util/ObjectTracing.java +++ b/src/main/java/emissary/util/ObjectTracing.java @@ -1,38 +1,32 @@ package emissary.util; +import emissary.core.IBaseDataObject; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; public class ObjectTracing { private ObjectTracing() {} - private static final List fieldNames = new ArrayList<>(); - protected static final Logger logger = LoggerFactory.getLogger(ObjectTracing.class); - public static void setUpFieldNames(List newFieldNames) { - fieldNames.addAll(newFieldNames); - } - - public static Map createTraceMessageMap(String[] fieldValues) { + /** + * Given an IBDO, create a map with log entries we care about and return it + * + * @param d the IBDO + * @return the map of log entries to add to the log + */ + public static Map createTraceMessageMap(IBaseDataObject d) { Map jsonMap = new HashMap<>(); - if (fieldValues.length != fieldNames.size()) { - logger.error("Cannot create log entry, the number of fields does not equals the number of values"); - return Collections.emptyMap(); - } - - for (int i = 0; i < fieldValues.length; i++) { - jsonMap.put(fieldNames.get(i), fieldValues[i]); - } + // add our fields + jsonMap.put("inputFileName", d.getFilename()); + jsonMap.put("stage", "PickUpPlace"); return jsonMap; } } diff --git a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java new file mode 100644 index 0000000000..340b9f1ca2 --- /dev/null +++ b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java @@ -0,0 +1,40 @@ +package emissary.util; + +import emissary.config.ConfigUtil; +import emissary.config.Configurator; + +import ch.qos.logback.core.rolling.RollingFileAppender; + +import java.io.IOException; + +public class ObjectTracingLogbackAppender extends RollingFileAppender { + private long start = System.currentTimeMillis(); // minutes + private int rolloverTime = 15; + private boolean isConfigured = false; + + public void configure() { + try { + Configurator configG = ConfigUtil.getConfigInfo(this.getClass()); + rolloverTime = configG.findIntEntry("ROLLOVER_TIME_MINUTES", rolloverTime); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void rollover() { + + if (!isConfigured) { + configure(); + isConfigured = true; + } + + long currentTime = System.currentTimeMillis(); + int maxIntervalSinceLastLoggingInMillis = rolloverTime * 60 * 1000; + + if ((currentTime - start) >= maxIntervalSinceLastLoggingInMillis) { + super.rollover(); + start = System.currentTimeMillis(); + } + } +} diff --git a/src/main/resources/emissary/pickup/file/FilePickUpPlace.cfg b/src/main/resources/emissary/pickup/file/FilePickUpPlace.cfg index 4b09201dbc..a1560dc22c 100644 --- a/src/main/resources/emissary/pickup/file/FilePickUpPlace.cfg +++ b/src/main/resources/emissary/pickup/file/FilePickUpPlace.cfg @@ -37,7 +37,3 @@ MINIMUM_DATA_SIZE = "-1" MAXIMUM_DATA_SIZE = "-1" USE_OBJECT_TRACE_LOGGER = true - -OBJECT_TRACE_LOGGER_FIELD_NAME = "stage" -OBJECT_TRACE_LOGGER_FIELD_NAME = "inputFileName" -OBJECT_TRACE_LOGGER_FIELD_NAME = "timestamp" diff --git a/src/main/resources/emissary/util/ObjectTracing.cfg b/src/main/resources/emissary/util/ObjectTracing.cfg index aa7defef7b..c5d336b5a4 100644 --- a/src/main/resources/emissary/util/ObjectTracing.cfg +++ b/src/main/resources/emissary/util/ObjectTracing.cfg @@ -1,3 +1,2 @@ FIELD_NAME = "stage" FIELD_NAME = "inputFileName" -FIELD_NAME = "timestamp" diff --git a/src/main/resources/emissary/util/ObjectTracingLogbackAppender.cfg b/src/main/resources/emissary/util/ObjectTracingLogbackAppender.cfg new file mode 100644 index 0000000000..92da46ed54 --- /dev/null +++ b/src/main/resources/emissary/util/ObjectTracingLogbackAppender.cfg @@ -0,0 +1 @@ +ROLLOVER_TIME_MINUTES = 15 \ No newline at end of file From 1e64ac9a52d9df399fb17994a1b6d5fb27a0e376 Mon Sep 17 00:00:00 2001 From: Amanda Date: Thu, 17 Aug 2023 18:43:05 +0000 Subject: [PATCH 4/8] Minor changes --- src/main/java/emissary/pickup/PickUpPlace.java | 8 +++++--- src/main/java/emissary/util/ObjectTracing.java | 9 ++------- .../java/emissary/util/ObjectTracingLogbackAppender.java | 5 +++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/emissary/pickup/PickUpPlace.java b/src/main/java/emissary/pickup/PickUpPlace.java index 96e7a5a03f..4b5e1f321c 100755 --- a/src/main/java/emissary/pickup/PickUpPlace.java +++ b/src/main/java/emissary/pickup/PickUpPlace.java @@ -81,8 +81,9 @@ public abstract class PickUpPlace extends ServiceProviderPlace implements IPickU // Metadata items that should always be copied to children protected Set ALWAYS_COPY_METADATA_VALS = new HashSet<>(); - private boolean useObjectTraceLogger = true; + private boolean useObjectTraceLogger = false; protected Logger objectTraceLogger; + private ObjectTracing tracingUtil; public PickUpPlace() throws IOException { super(); @@ -194,6 +195,7 @@ protected void configurePickUpPlace() { logger.info("Setting up the object trace logger"); objectTraceLogger = LoggerFactory.getLogger("objectTrace"); } + tracingUtil = new ObjectTracing(); } /** @@ -577,12 +579,12 @@ protected boolean processDataObject(IBaseDataObject d, String fixedName, File th } /** - * Creates an entry in the object trace log. Can be overridden if desired. + * Creates an entry in the object trace log. * * @param d The IBDO */ public void objectTraceLog(IBaseDataObject d) { - objectTraceLogger.info(appendEntries(ObjectTracing.createTraceMessageMap(d)), ""); + objectTraceLogger.info(appendEntries(tracingUtil.createTraceMessageMap(d)), ""); } /** diff --git a/src/main/java/emissary/util/ObjectTracing.java b/src/main/java/emissary/util/ObjectTracing.java index 33622d09f9..388f33e51d 100644 --- a/src/main/java/emissary/util/ObjectTracing.java +++ b/src/main/java/emissary/util/ObjectTracing.java @@ -2,17 +2,12 @@ import emissary.core.IBaseDataObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.HashMap; import java.util.Map; public class ObjectTracing { - private ObjectTracing() {} - - protected static final Logger logger = LoggerFactory.getLogger(ObjectTracing.class); + public ObjectTracing() {} /** * Given an IBDO, create a map with log entries we care about and return it @@ -20,7 +15,7 @@ private ObjectTracing() {} * @param d the IBDO * @return the map of log entries to add to the log */ - public static Map createTraceMessageMap(IBaseDataObject d) { + public Map createTraceMessageMap(IBaseDataObject d) { Map jsonMap = new HashMap<>(); diff --git a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java index 340b9f1ca2..a5657720eb 100644 --- a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java +++ b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java @@ -8,15 +8,16 @@ import java.io.IOException; public class ObjectTracingLogbackAppender extends RollingFileAppender { - private long start = System.currentTimeMillis(); // minutes + private long start = System.currentTimeMillis(); private int rolloverTime = 15; private boolean isConfigured = false; public void configure() { try { - Configurator configG = ConfigUtil.getConfigInfo(this.getClass()); + Configurator configG = ConfigUtil.getConfigInfo(getClass()); rolloverTime = configG.findIntEntry("ROLLOVER_TIME_MINUTES", rolloverTime); } catch (IOException e) { + // TODO -- how to handle this? throw new RuntimeException(e); } } From 11eb6c2b21f8b3814e7fd799b43b1a72d793c039 Mon Sep 17 00:00:00 2001 From: Amanda Date: Thu, 17 Aug 2023 18:59:32 +0000 Subject: [PATCH 5/8] Handing error case in setting up ObjectTracingLogbackAppender --- .../java/emissary/util/ObjectTracingLogbackAppender.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java index a5657720eb..1f80ad59e5 100644 --- a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java +++ b/src/main/java/emissary/util/ObjectTracingLogbackAppender.java @@ -4,6 +4,8 @@ import emissary.config.Configurator; import ch.qos.logback.core.rolling.RollingFileAppender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; @@ -12,13 +14,14 @@ public class ObjectTracingLogbackAppender extends RollingFileAppender { private int rolloverTime = 15; private boolean isConfigured = false; + protected static final Logger logger = LoggerFactory.getLogger(ObjectTracingLogbackAppender.class); + public void configure() { try { Configurator configG = ConfigUtil.getConfigInfo(getClass()); rolloverTime = configG.findIntEntry("ROLLOVER_TIME_MINUTES", rolloverTime); } catch (IOException e) { - // TODO -- how to handle this? - throw new RuntimeException(e); + logger.error("Could not read from the config for the ObjectTracingLogbackAppender ", e); } } From adce41ba174dec833888d13a89a6292b9d4ead3e Mon Sep 17 00:00:00 2001 From: Amanda Date: Tue, 5 Sep 2023 14:35:05 +0000 Subject: [PATCH 6/8] Fix log configuration --- src/main/config/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/config/logback.xml b/src/main/config/logback.xml index 0c7933af16..8618f76c27 100644 --- a/src/main/config/logback.xml +++ b/src/main/config/logback.xml @@ -50,7 +50,7 @@ - + From 4c9c6c9e158a55338e8874b076eed0bc9ff93942 Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 6 Sep 2023 18:05:08 +0000 Subject: [PATCH 7/8] Addressing code review comments --- src/main/config/logback.xml | 3 +-- .../java/emissary/pickup/PickUpPlace.java | 20 +++------------- ...ava => CustomRolloverLogbackAppender.java} | 6 ++--- .../java/emissary/util/ObjectTracing.java | 24 ++++++++++++------- ....cfg => CustomRolloverLogbackAppender.cfg} | 0 .../resources/emissary/util/ObjectTracing.cfg | 2 -- 6 files changed, 22 insertions(+), 33 deletions(-) rename src/main/java/emissary/util/{ObjectTracingLogbackAppender.java => CustomRolloverLogbackAppender.java} (88%) rename src/main/resources/emissary/util/{ObjectTracingLogbackAppender.cfg => CustomRolloverLogbackAppender.cfg} (100%) delete mode 100644 src/main/resources/emissary/util/ObjectTracing.cfg diff --git a/src/main/config/logback.xml b/src/main/config/logback.xml index 8618f76c27..114dccef8d 100644 --- a/src/main/config/logback.xml +++ b/src/main/config/logback.xml @@ -29,7 +29,6 @@ false [ignore] - [ignore] [ignore] [ignore] [ignore] @@ -52,7 +51,7 @@ - + {"node":"${emissary.node.name}"} false diff --git a/src/main/java/emissary/pickup/PickUpPlace.java b/src/main/java/emissary/pickup/PickUpPlace.java index 4b5e1f321c..434ac3e023 100755 --- a/src/main/java/emissary/pickup/PickUpPlace.java +++ b/src/main/java/emissary/pickup/PickUpPlace.java @@ -21,8 +21,6 @@ import emissary.util.TimeUtil; import emissary.util.shell.Executrix; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.slf4j.MDC; import java.io.File; @@ -37,7 +35,6 @@ import static emissary.core.constants.Parameters.FILE_DATE; import static emissary.core.constants.Parameters.FILE_NAME; -import static net.logstash.logback.marker.Markers.appendEntries; /** * This class is the base class of those places that inject data into the system. This place knows a lot about @@ -82,8 +79,7 @@ public abstract class PickUpPlace extends ServiceProviderPlace implements IPickU protected Set ALWAYS_COPY_METADATA_VALS = new HashSet<>(); private boolean useObjectTraceLogger = false; - protected Logger objectTraceLogger; - private ObjectTracing tracingUtil; + protected ObjectTracing objectTracingUtil; public PickUpPlace() throws IOException { super(); @@ -192,10 +188,9 @@ protected void configurePickUpPlace() { // Setup objectTraceLogger useObjectTraceLogger = configG.findBooleanEntry("USE_OBJECT_TRACE_LOGGER", useObjectTraceLogger); if (useObjectTraceLogger) { + objectTracingUtil = new ObjectTracing(); logger.info("Setting up the object trace logger"); - objectTraceLogger = LoggerFactory.getLogger("objectTrace"); } - tracingUtil = new ObjectTracing(); } /** @@ -571,22 +566,13 @@ protected boolean processDataObject(IBaseDataObject d, String fixedName, File th // If object tracing log that agent is being deployed for fixedName (filename) if (useObjectTraceLogger) { - objectTraceLog(d); + objectTracingUtil.emitLifecycleEvent(d, ObjectTracing.Stage.PickUp); } assignToPooledAgent(d, -1L); return true; } - /** - * Creates an entry in the object trace log. - * - * @param d The IBDO - */ - public void objectTraceLog(IBaseDataObject d) { - objectTraceLogger.info(appendEntries(tracingUtil.createTraceMessageMap(d)), ""); - } - /** * Parse out sessions and process data from a file * diff --git a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java b/src/main/java/emissary/util/CustomRolloverLogbackAppender.java similarity index 88% rename from src/main/java/emissary/util/ObjectTracingLogbackAppender.java rename to src/main/java/emissary/util/CustomRolloverLogbackAppender.java index 1f80ad59e5..6a12616e2c 100644 --- a/src/main/java/emissary/util/ObjectTracingLogbackAppender.java +++ b/src/main/java/emissary/util/CustomRolloverLogbackAppender.java @@ -9,19 +9,19 @@ import java.io.IOException; -public class ObjectTracingLogbackAppender extends RollingFileAppender { +public class CustomRolloverLogbackAppender extends RollingFileAppender { private long start = System.currentTimeMillis(); private int rolloverTime = 15; private boolean isConfigured = false; - protected static final Logger logger = LoggerFactory.getLogger(ObjectTracingLogbackAppender.class); + protected static final Logger logger = LoggerFactory.getLogger(CustomRolloverLogbackAppender.class); public void configure() { try { Configurator configG = ConfigUtil.getConfigInfo(getClass()); rolloverTime = configG.findIntEntry("ROLLOVER_TIME_MINUTES", rolloverTime); } catch (IOException e) { - logger.error("Could not read from the config for the ObjectTracingLogbackAppender ", e); + logger.error("Could not read from the config for the CustomRolloverLogbackAppender ", e); } } diff --git a/src/main/java/emissary/util/ObjectTracing.java b/src/main/java/emissary/util/ObjectTracing.java index 388f33e51d..c12725f48d 100644 --- a/src/main/java/emissary/util/ObjectTracing.java +++ b/src/main/java/emissary/util/ObjectTracing.java @@ -2,26 +2,32 @@ import emissary.core.IBaseDataObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.Map; +import static net.logstash.logback.marker.Markers.appendEntries; + public class ObjectTracing { + protected static Logger objectTraceLogger = LoggerFactory.getLogger("objectTrace");; + + public enum Stage { + PickUp, DropOff + } + public ObjectTracing() {} - /** - * Given an IBDO, create a map with log entries we care about and return it - * - * @param d the IBDO - * @return the map of log entries to add to the log - */ - public Map createTraceMessageMap(IBaseDataObject d) { + public void emitLifecycleEvent(IBaseDataObject d, ObjectTracing.Stage stage) { Map jsonMap = new HashMap<>(); // add our fields jsonMap.put("inputFileName", d.getFilename()); - jsonMap.put("stage", "PickUpPlace"); - return jsonMap; + jsonMap.put("stage", String.valueOf(stage)); + + objectTraceLogger.info(appendEntries(jsonMap), ""); } } diff --git a/src/main/resources/emissary/util/ObjectTracingLogbackAppender.cfg b/src/main/resources/emissary/util/CustomRolloverLogbackAppender.cfg similarity index 100% rename from src/main/resources/emissary/util/ObjectTracingLogbackAppender.cfg rename to src/main/resources/emissary/util/CustomRolloverLogbackAppender.cfg diff --git a/src/main/resources/emissary/util/ObjectTracing.cfg b/src/main/resources/emissary/util/ObjectTracing.cfg deleted file mode 100644 index c5d336b5a4..0000000000 --- a/src/main/resources/emissary/util/ObjectTracing.cfg +++ /dev/null @@ -1,2 +0,0 @@ -FIELD_NAME = "stage" -FIELD_NAME = "inputFileName" From 389204a682e9ec067d85ff14412b80bb90983691 Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 6 Sep 2023 18:07:32 +0000 Subject: [PATCH 8/8] Correcting logback.xml --- src/main/config/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/config/logback.xml b/src/main/config/logback.xml index 114dccef8d..270f5444dd 100644 --- a/src/main/config/logback.xml +++ b/src/main/config/logback.xml @@ -29,6 +29,7 @@ false [ignore] + [ignore] [ignore] [ignore] [ignore] @@ -57,7 +58,6 @@ false [ignore] - [ignore] [ignore] [ignore] [ignore]