Skip to content

Commit

Permalink
PerformanceQueries Changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@nbauernfeind
nbauernfeind committed Nov 9, 2023
1 parent 81dd004 commit 1ba592e
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 51 deletions.
4 changes: 2 additions & 2 deletions engine/table/src/main/java/io/deephaven/engine/table/impl/perf/UpdatePerformanceTracker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,10 @@ private synchronized void publish(
public UpdatePerformanceTracker(final UpdateGraph updateGraph) {
this.updateGraph = Objects.requireNonNull(updateGraph);
this.aggregatedSmallUpdatesEntry = new PerformanceEntry(
QueryConstants.NULL_LONG, QueryConstants.NULL_INT, QueryConstants.NULL_INT,
QueryConstants.NULL_LONG, QueryConstants.NULL_LONG, QueryConstants.NULL_INT,
"Aggregated Small Updates", null, updateGraph.getName());
this.flushEntry = new PerformanceEntry(
QueryConstants.NULL_LONG, QueryConstants.NULL_INT, QueryConstants.NULL_INT,
QueryConstants.NULL_LONG, QueryConstants.NULL_LONG, QueryConstants.NULL_INT,
"UpdatePerformanceTracker Flush", null, updateGraph.getName());
}

Expand Down
22 changes: 22 additions & 0 deletions ...ons/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueries.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.google.auto.service.AutoService;
import io.deephaven.engine.table.Table;

import io.deephaven.engine.table.hierarchical.TreeTable;
import io.deephaven.engine.util.GroovyDeephavenSession;
import io.deephaven.util.annotations.ScriptApi;

Expand Down Expand Up @@ -68,6 +69,27 @@ public static Table queryOperationPerformance(final long evaluationNumber) {
evaluationNumber);
}

/**
* Converts the query performance table into a tree table.
*
* @return query performance tree table.
*/
@ScriptApi
public static TreeTable queryPerformanceAsTreeTable() {
return PerformanceQueriesGeneral.queryPerformanceAsTreeTable(TableLoggers.queryPerformanceLog());
}

/**
* Merges the query performance and query operation performance tables into a single tree table.
*
* @return query operation performance tree table.
*/
@ScriptApi
public static TreeTable queryOperationPerformanceAsTreeTable() {
return PerformanceQueriesGeneral.queryOperationPerformanceAsTreeTable(
TableLoggers.queryPerformanceLog(), TableLoggers.queryOperationPerformanceLog());
}

/**
* Gets the information for a process.
*
Expand Down
131 changes: 82 additions & 49 deletions ...formance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,17 @@
package io.deephaven.engine.table.impl.util;

import io.deephaven.engine.table.Table;
import io.deephaven.engine.table.impl.DataAccessHelpers;
import io.deephaven.engine.table.hierarchical.TreeTable;
import io.deephaven.engine.util.TableTools;
import io.deephaven.plot.Figure;
import io.deephaven.plot.PlottingConvenience;
import io.deephaven.util.QueryConstants;
import org.jetbrains.annotations.NotNull;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.OptionalLong;
import java.util.stream.Stream;

import static io.deephaven.api.agg.Aggregation.AggFirst;
import static io.deephaven.api.agg.Aggregation.AggMax;
Expand All @@ -35,20 +37,21 @@ public static Table queryPerformance(Table queryPerformanceLog, final long evalu
queryPerformanceLog = queryPerformanceLog
.updateView(
"WorkerHeapSize = " + workerHeapSizeBytes + "L",
"TimeSecs = nanosToMillis(EndTime - StartTime) / 1000d", // How long this query ran for, in
// seconds
// How long this query ran for, in seconds
"TimeSecs = nanosToMillis(EndTime - StartTime) / 1000d",
"NetMemoryChange = FreeMemoryChange - TotalMemoryChange",
"QueryMemUsed = TotalMemory - FreeMemory", // Memory in use by the query. (Only
// includes active heap memory.)
"QueryMemUsedPct = QueryMemUsed / WorkerHeapSize", // Memory usage as a percenage of max heap
// size (-Xmx)
"QueryMemFree = WorkerHeapSize - QueryMemUsed" // Remaining memory until the query runs into the
// max heap size
)
.moveColumnsUp(
"ProcessUniqueId", "EvaluationNumber",
"QueryMemUsed", "QueryMemFree", "QueryMemUsedPct",
"EndTime", "TimeSecs", "NetMemoryChange");
// Memory in use by the query. (Only includes active heap memory.)
"QueryMemUsed = TotalMemory - FreeMemory",
// Memory usage as a percenage of max heap size (-Xmx)
"QueryMemUsedPct = QueryMemUsed / WorkerHeapSize",
// Remaining memory until the query runs into the max heap size
"QueryMemFree = WorkerHeapSize - QueryMemUsed");

queryPerformanceLog = maybeMoveColumnsUp(queryPerformanceLog,
"ProcessUniqueId", "EvaluationNumber", "ParentEvaluationNumber",
"QueryMemUsed", "QueryMemFree", "QueryMemUsedPct",
"EndTime", "TimeSecs", "NetMemoryChange");

if (formatPctColumns) {
queryPerformanceLog = formatColumnsAsPct(queryPerformanceLog, "QueryMemUsedPct");
}
Expand All @@ -64,15 +67,16 @@ public static Table queryOperationPerformance(Table queryOps, final long evaluat
queryOps = queryOps.where(whereConditionForEvaluationNumber(evaluationNumber));
}

return queryOps
queryOps = queryOps
.updateView(
"TimeSecs = nanosToMillis(EndTime - StartTime) / 1000d",
"NetMemoryChange = FreeMemoryChange - TotalMemoryChange" // Change in memory usage delta while
// this query was executing
)
.moveColumnsUp(
"ProcessUniqueId", "EvaluationNumber", "OperationNumber",
"EndTime", "TimeSecs", "NetMemoryChange");
// Change in memory usage delta while this query was executing
"NetMemoryChange = FreeMemoryChange - TotalMemoryChange");

return maybeMoveColumnsUp(queryOps,
"ProcessUniqueId", "EvaluationNumber", "ParentEvaluationNumber",
"OperationNumber", "ParentOperationNumber",
"EndTime", "TimeSecs", "NetMemoryChange");
}

public static Table queryOperationPerformance(final Table queryOps) {
Expand All @@ -84,11 +88,7 @@ public static String processInfo(Table processInfo, final String processInfoId,
processInfo = processInfo
.where("Id = `" + processInfoId + "`", "Type = `" + type + "`", "Key = `" + key + "`")
.select("Value");
try {
return (String) DataAccessHelpers.getColumn(processInfo, 0).get(0);
} catch (Exception e) {
return null;
}
return processInfo.<String>getColumnSource("Value").get(processInfo.getRowSet().firstRowKey());
}

public static Table queryUpdatePerformance(Table queryUpdatePerformance, final long evaluationNumber,
Expand All @@ -101,24 +101,26 @@ public static Table queryUpdatePerformance(Table queryUpdatePerformance, final l
queryUpdatePerformance = queryUpdatePerformance
.updateView(
"WorkerHeapSize = " + workerHeapSizeBytes + "L",
"Ratio = EntryIntervalUsage / IntervalDurationNanos", // % of time during this interval that the
// operation was using CPU
"QueryMemUsed = MaxTotalMemory - MinFreeMemory", // Memory in use by the query. (Only
// includes active heap memory.)
"QueryMemUsedPct = QueryMemUsed / WorkerHeapSize", // Memory usage as a percenage of the max
// heap size (-Xmx)
"QueryMemFree = WorkerHeapSize - QueryMemUsed", // Remaining memory until the query runs into
// the max heap size
"NRows = EntryIntervalAdded + EntryIntervalRemoved + EntryIntervalModified", // Total number of
// changed rows
"RowsPerSec = round(NRows / IntervalDurationNanos * 1.0e9)", // Average rate data is ticking at
"RowsPerCPUSec = round(NRows / EntryIntervalUsage * 1.0e9)" // Approximation of how fast CPU
// handles row changes
)
.moveColumnsUp(
"ProcessUniqueId", "EvaluationNumber", "OperationNumber",
"Ratio", "QueryMemUsed", "QueryMemUsedPct", "IntervalEndTime",
"RowsPerSec", "RowsPerCPUSec", "EntryDescription");
// % of time during this interval that the operation was using CPU
"Ratio = EntryIntervalUsage / IntervalDurationNanos",
// Memory in use by the query. (Only includes active heap memory.)
"QueryMemUsed = MaxTotalMemory - MinFreeMemory",
// Memory usage as a percentage of the max heap size (-Xmx)
"QueryMemUsedPct = QueryMemUsed / WorkerHeapSize",
// Remaining memory until the query runs into the max heap size
"QueryMemFree = WorkerHeapSize - QueryMemUsed",
// Total number of changed rows
"NRows = EntryIntervalAdded + EntryIntervalRemoved + EntryIntervalModified",
// Average rate data is ticking at
"RowsPerSec = round(NRows / IntervalDurationNanos * 1.0e9)",
// Approximation of how fast CPU handles row changes
"RowsPerCPUSec = round(NRows / EntryIntervalUsage * 1.0e9)");

queryUpdatePerformance = maybeMoveColumnsUp(queryUpdatePerformance,
"ProcessUniqueId", "EvaluationNumber", "OperationNumber",
"Ratio", "QueryMemUsed", "QueryMemUsedPct", "IntervalEndTime",
"RowsPerSec", "RowsPerCPUSec", "EntryDescription");

if (formatPctColumnsLocal && formatPctColumns) {
queryUpdatePerformance = formatColumnsAsPctUpdatePerformance(queryUpdatePerformance);
}
Expand Down Expand Up @@ -149,6 +151,7 @@ public static Map<String, Table> queryUpdatePerformanceMap(final Table queryUpda
"EntryIntervalAdded",
"EntryIntervalRemoved",
"EntryIntervalModified",
"EntryIntervalShifted",
"NRows");

// Create a table showing the 'worst' updates, i.e. the operations with the greatest 'Ratio'
Expand Down Expand Up @@ -270,7 +273,7 @@ public static Map<String, Object> serverStateWithPlots(final Table pml) {
final Table pm = serverState(pml);
resultMap.put("ServerState", pm);

int maxMemMiB = DataAccessHelpers.getColumn(pm, "MaxMemMiB").getInt(0);
int maxMemMiB = pm.getColumnSource("MaxMemMiB").getInt(pm.getRowSet().firstRowKey());
if (maxMemMiB == QueryConstants.NULL_INT) {
maxMemMiB = 4096;
}
Expand Down Expand Up @@ -310,6 +313,33 @@ public static Map<String, Object> serverStateWithPlots(final Table pml) {
return resultMap;
}

public static TreeTable queryPerformanceAsTreeTable(@NotNull final Table qpl) {
return qpl.tree("EvaluationNumber", "ParentEvaluationNumber");
}

public static TreeTable queryOperationPerformanceAsTreeTable(
@NotNull final Table qpl, @NotNull final Table qopl) {
Table mergeWithAggKeys = TableTools.merge(
qpl.updateView(
"EvalKey = `` + EvaluationNumber",
"ParentEvalKey = ParentEvaluationNumber == null ? null : (`` + ParentEvaluationNumber)",
"OperationNumber = NULL_INT",
"ParentOperationNumber = NULL_INT",
"Depth = NULL_INT",
"CallerLine = (String) null",
"IsCompilation = NULL_BOOLEAN",
"InputSizeLong = NULL_LONG"),
qopl.updateView(
"EvalKey = EvaluationNumber + `:` + OperationNumber",
"ParentEvalKey = EvaluationNumber + (ParentOperationNumber == null ? `` : (`:` + ParentOperationNumber))",
"Exception = (String) null"))
.moveColumnsUp("EvalKey", "ParentEvalKey")
.moveColumnsDown("EvaluationNumber", "ParentEvaluationNumber", "OperationNumber",
"ParentOperationNumber");

return mergeWithAggKeys.tree("EvalKey", "ParentEvalKey");
}

private static Table formatColumnsAsPct(final Table t, final String... cols) {
final String[] formats = new String[cols.length];
for (int i = 0; i < cols.length; ++i) {
Expand All @@ -323,11 +353,14 @@ private static Table formatColumnsAsPctUpdatePerformance(final Table updatePerfo
}

private static long getWorkerHeapSizeBytes() {
final OptionalLong opt = EngineMetrics.getProcessInfo().getMemoryInfo().heap().max();
return opt.orElse(0);
return EngineMetrics.getProcessInfo().getMemoryInfo().heap().max().orElse(0);
}

private static String whereConditionForEvaluationNumber(final long evaluationNumber) {
return "EvaluationNumber = " + evaluationNumber + "";
return "EvaluationNumber = " + evaluationNumber;
}

private static Table maybeMoveColumnsUp(final Table source, final String... cols) {
return source.moveColumnsUp(Stream.of(cols).filter(source::hasColumns).toArray(String[]::new));
}
}
3 changes: 3 additions & 0 deletions server/src/main/java/io/deephaven/server/session/SessionState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,9 @@ private synchronized void setQueryPerformanceRecorder(
}
this.qprIsForBatch = qprIsForBatch;
this.queryPerformanceRecorder = queryPerformanceRecorder;
if (!qprIsForBatch && QueryPerformanceRecorder.getInstance() == queryPerformanceRecorder) {
QueryPerformanceRecorder.resetInstance();
}
}

/**
Expand Down

0 comments on commit 1ba592e

Please sign in to comment.