Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding objectTrace logger to output configurable fields to a new log file from PickUpPlace #530

Merged
merged 8 commits into from
Sep 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/main/config/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,34 @@
</then>
</if>

<!-- Enable output metrics logging by creating an environment variable of "ObjectTrace=true" -->
<if condition="&quot;${ObjectTrace}&quot;.equals(&quot;true&quot;)">
<then>
<appender name="OBJECT-TRACE" class="emissary.util.CustomRolloverLogbackAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<customFields>{"node":"${emissary.node.name}"}</customFields>
<includeMdc>false</includeMdc>
<fieldNames>
<version>[ignore]</version>
<logger>[ignore]</logger>
<thread>[ignore]</thread>
<levelValue>[ignore]</levelValue>
<level>[ignore]</level>
<message>[ignore]</message>
</fieldNames>
</encoder>
<file>logs/${emissary.node.name}-${emissary.node.port}-object-trace.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/${emissary.node.name}-${emissary.node.port}-object-trace.log.%d{yyyyMMdd-HHmm}</fileNamePattern>
</rollingPolicy>
</appender>

<logger name="objectTrace" level="INFO" additivity="false">
<appender-ref ref="OBJECT-TRACE" />
</logger>
</then>
</if>

<root level="INFO">
<!-- stop logging to a file in development man, redirect or tee to a file
<appender-ref ref="TOFILE"/>
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/emissary/pickup/PickUpPlace.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import emissary.place.ServiceProviderPlace;
import emissary.pool.AgentPool;
import emissary.util.ClassComparator;
import emissary.util.ObjectTracing;
import emissary.util.TimeUtil;
import emissary.util.shell.Executrix;

Expand Down Expand Up @@ -77,6 +78,9 @@ public abstract class PickUpPlace extends ServiceProviderPlace implements IPickU
// Metadata items that should always be copied to children
protected Set<String> ALWAYS_COPY_METADATA_VALS = new HashSet<>();

private boolean useObjectTraceLogger = false;
protected ObjectTracing objectTracingUtil;

public PickUpPlace() throws IOException {
super();
configurePickUpPlace();
Expand Down Expand Up @@ -180,6 +184,13 @@ protected void configurePickUpPlace() {
}

ALWAYS_COPY_METADATA_VALS = configG.findEntriesAsSet("ALWAYS_COPY_METADATA");

// Setup objectTraceLogger
useObjectTraceLogger = configG.findBooleanEntry("USE_OBJECT_TRACE_LOGGER", useObjectTraceLogger);
if (useObjectTraceLogger) {
objectTracingUtil = new ObjectTracing();
logger.info("Setting up the object trace logger");
}
}

/**
Expand Down Expand Up @@ -552,6 +563,12 @@ protected boolean processDataObject(IBaseDataObject d, String fixedName, File th
dataObjectCreated(d, theFile);
logger.info("**Deploying an agent for {} and object {} forms={} simple={}", fixedName, d.getInternalId(), d.getAllCurrentForms(),
(simpleMode ? "simple" : ""));

// If object tracing log that agent is being deployed for fixedName (filename)
if (useObjectTraceLogger) {
objectTracingUtil.emitLifecycleEvent(d, ObjectTracing.Stage.PickUp);
}

assignToPooledAgent(d, -1L);
return true;
}
Expand Down
44 changes: 44 additions & 0 deletions src/main/java/emissary/util/CustomRolloverLogbackAppender.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package emissary.util;

import emissary.config.ConfigUtil;
import emissary.config.Configurator;

import ch.qos.logback.core.rolling.RollingFileAppender;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

public class CustomRolloverLogbackAppender<E> extends RollingFileAppender<E> {
private long start = System.currentTimeMillis();
private int rolloverTime = 15;
private boolean isConfigured = false;

protected static final Logger logger = LoggerFactory.getLogger(CustomRolloverLogbackAppender.class);

public void configure() {
try {
Configurator configG = ConfigUtil.getConfigInfo(getClass());
rolloverTime = configG.findIntEntry("ROLLOVER_TIME_MINUTES", rolloverTime);
} catch (IOException e) {
logger.error("Could not read from the config for the CustomRolloverLogbackAppender ", e);
}
}

@Override
public void rollover() {

if (!isConfigured) {
configure();
isConfigured = true;
}

long currentTime = System.currentTimeMillis();
int maxIntervalSinceLastLoggingInMillis = rolloverTime * 60 * 1000;

if ((currentTime - start) >= maxIntervalSinceLastLoggingInMillis) {
super.rollover();
start = System.currentTimeMillis();
}
}
}
33 changes: 33 additions & 0 deletions src/main/java/emissary/util/ObjectTracing.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package emissary.util;

import emissary.core.IBaseDataObject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

import static net.logstash.logback.marker.Markers.appendEntries;

public class ObjectTracing {

protected static Logger objectTraceLogger = LoggerFactory.getLogger("objectTrace");;

public enum Stage {
PickUp, DropOff
}

public ObjectTracing() {}

public void emitLifecycleEvent(IBaseDataObject d, ObjectTracing.Stage stage) {

Map<String, String> jsonMap = new HashMap<>();

// add our fields
jsonMap.put("inputFileName", d.getFilename());
jsonMap.put("stage", String.valueOf(stage));

objectTraceLogger.info(appendEntries(jsonMap), "");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ WIN_OUT_ROOT = "@{OUTPUT_DATA}"
MINIMUM_DATA_SIZE = "-1"
MAXIMUM_DATA_SIZE = "-1"


USE_OBJECT_TRACE_LOGGER = true
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ROLLOVER_TIME_MINUTES = 15