Skip to content

Commit

Permalink
Merge pull request #34 from instana/metrics-naming
Browse files Browse the repository at this point in the history 
Sync with the metrics naming in Traceloop
  • Loading branch information
@jinsongo
jinsongo authored Jun 5, 2024
2 parents 2cdc83f + 437e5a0 commit f571fc3
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 102 deletions.
4 changes: 2 additions & 2 deletions llm/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ vi config/config.yaml
The following options are required：
- `otel.backend.url`：The OTel gRPC address of the agent, for example: http://localhost:4317
- `otel.service.name`：The Data Collector name, which can be any string you choose.
- `price.prompt.tokens.per.kilo`：The unit price per thousand prompt tokens.
- `price.complete.tokens.per.kilo`：The unit price per thousand complete tokens.
- `*.price.prompt.tokens.per.kilo`：The unit price per thousand prompt tokens.
- `*.price.complete.tokens.per.kilo`：The unit price per thousand complete tokens.


## Run ODCL
Expand Down
9 changes: 7 additions & 2 deletions llm/config/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,10 @@ instances:
- otel.backend.url: http://localhost:4317
otel.service.name: DC1
otel.service.port: 8000
price.prompt.tokens.per.kilo: 0.0
price.complete.tokens.per.kilo: 0.0
#Only configure the settings of the AI provider you are using
watsonx.price.prompt.tokens.per.kilo: 0.0
watsonx.price.complete.tokens.per.kilo: 0.0
openai.price.prompt.tokens.per.kilo: 0.0
openai.price.complete.tokens.per.kilo: 0.0
anthropic.price.prompt.tokens.per.kilo: 0.0
anthropic.price.complete.tokens.per.kilo: 0.0
8 changes: 6 additions & 2 deletions llm/src/main/java/com/instana/dc/llm/LLMDcUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,12 @@ public class LLMDcUtil {
public static final String DEFAULT_INSTRUMENTATION_SCOPE_VER = "1.0.0";
public static final String SERVICE_NAME = "service.name";
public static final String SERVICE_INSTANCE_ID = "service.instance.id";
public final static String PRICE_PROMPT_TOKES_PER_KILO = "price.prompt.tokens.per.kilo";
public final static String PRICE_COMPLETE_TOKES_PER_KILO = "price.complete.tokens.per.kilo";
public final static String WATSONX_PRICE_PROMPT_TOKES_PER_KILO = "watsonx.price.prompt.tokens.per.kilo";
public final static String WATSONX_PRICE_COMPLETE_TOKES_PER_KILO = "watsonx.price.complete.tokens.per.kilo";
public final static String OPENAI_PRICE_PROMPT_TOKES_PER_KILO = "openai.price.prompt.tokens.per.kilo";
public final static String OPENAI_PRICE_COMPLETE_TOKES_PER_KILO = "openai.price.complete.tokens.per.kilo";
public final static String ANTHROPIC_PRICE_PROMPT_TOKES_PER_KILO = "anthropic.price.prompt.tokens.per.kilo";
public final static String ANTHROPIC_PRICE_COMPLETE_TOKES_PER_KILO = "anthropic.price.complete.tokens.per.kilo";
public final static String SERVICE_LISTEN_PORT = "otel.service.port";

/* Configurations for Metrics:
Expand Down
204 changes: 134 additions & 70 deletions llm/src/main/java/com/instana/dc/llm/impl/llm/LLMDc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,12 @@ public class LLMDc extends AbstractLLMDc {
public static final String SENSOR_NAME = "com.instana.plugin.watsonx";
private HashMap<String, ModelAggregation> modelAggrMap = new HashMap<>();
private MetricsCollectorService metricsCollector = new MetricsCollectorService();
private Double pricePromptTokens = 0.0;
private Double priceCompleteTokens = 0.0;
private Double watsonxPricePromptTokens = 0.0;
private Double watsonxPriceCompleteTokens = 0.0;
private Double openaiPricePromptTokens = 0.0;
private Double openaiPriceCompleteTokens = 0.0;
private Double anthropicPricePromptTokens = 0.0;
private Double anthropicPriceCompleteTokens = 0.0;
private int listenPort = 0;

/**
Expand All @@ -40,104 +44,117 @@ public class LLMDc extends AbstractLLMDc {

private class ModelAggregation {
private final String modelId;
private final String userId;
private int deltaPromptTokens;
private int deltaCompleteTokens;
private int deltaDuration;
private int deltaReqCount;
private int maxDuration;
private int lastTotalPromptTokens;
private int lastTotalCompleteTokens;
private int lastTotalDuration;
private int lastTotalReqCount;
private final String aiSystem;
private long deltaPromptTokens;
private long deltaCompleteTokens;
private long deltaDuration;
private long deltaReqCount;
private long maxDuration;
private long lastTotalPromptTokens;
private long lastTotalCompleteTokens;
private long lastTotalDuration;
private long lastTotalReqCount;

public ModelAggregation(String modelId, String userId) {
public ModelAggregation(String modelId, String aiSystem) {
this.modelId = modelId;
this.userId = userId;
this.aiSystem = aiSystem;
}
public String getModelId() {
return modelId;
}
public String getUserId() {
return userId;
public String getAiSystem() {
return aiSystem;
}
public void addDeltaPromptTokens(int currTokens) {
public void addDeltaPromptTokens(long currTokens, long reqCount) {
if(currTokens == 0) {
return;
}
int diffPromptTokens = 0;
if(currTokens > lastTotalPromptTokens && lastTotalPromptTokens != 0) {
long diffPromptTokens = 0;
if(reqCount == 1) {
diffPromptTokens = currTokens;
} else if(currTokens > lastTotalPromptTokens && lastTotalPromptTokens != 0) {
diffPromptTokens = currTokens - lastTotalPromptTokens;
}
lastTotalPromptTokens = currTokens;
deltaPromptTokens += diffPromptTokens;
}

public int getDeltaPromptTokens() {
public long getDeltaPromptTokens() {
return deltaPromptTokens;
}
public void addDeltaCompleteTokens(int currTokens) {
public void addDeltaCompleteTokens(long currTokens, long reqCount) {
if(currTokens == 0) {
return;
}
int diffCompleteTokens = 0;
if(currTokens > lastTotalCompleteTokens && lastTotalCompleteTokens != 0) {
long diffCompleteTokens = 0;
if(reqCount == 1) {
diffCompleteTokens = currTokens;
} else if(currTokens > lastTotalCompleteTokens && lastTotalCompleteTokens != 0) {
diffCompleteTokens = currTokens - lastTotalCompleteTokens;
}
lastTotalCompleteTokens = currTokens;
deltaCompleteTokens += diffCompleteTokens;
}
public int getDeltaCompleteTokens() {
public long getDeltaCompleteTokens() {
return deltaCompleteTokens;
}
public void addDeltaDuration(int currDuration) {
public void addDeltaDuration(long currDuration, long reqCount) {
if(currDuration == 0) {
return;
}
int diffDuration = 0;
if(currDuration > lastTotalDuration && lastTotalDuration != 0) {
long diffDuration = 0;
if(reqCount == 1) {
diffDuration = currDuration;
} else if(currDuration > lastTotalDuration && lastTotalDuration != 0) {
diffDuration = currDuration - lastTotalDuration;
}
lastTotalDuration = currDuration;
deltaDuration += diffDuration;

if(deltaDuration > maxDuration) {
maxDuration = deltaDuration;
}
}
public int getDeltaDuration() {
public long getDeltaDuration() {
return deltaDuration;
}
public int getMaxDuration() {
public void setMaxDuration(long maxDuration) {
this.maxDuration = maxDuration;
}
public long getMaxDuration() {
return maxDuration;
}
public void addDeltaReqCount(int currCount) {
public void addDeltaReqCount(long currCount) {
if(currCount == 0) {
return;
}
int diffReqCount = 0;
if(currCount > lastTotalReqCount && lastTotalReqCount != 0) {
long diffReqCount = 0;
if(currCount == 1) {
diffReqCount = currCount;
} else if(currCount > lastTotalReqCount && lastTotalReqCount != 0) {
diffReqCount = currCount - lastTotalReqCount;
}
lastTotalReqCount = currCount;
deltaReqCount += diffReqCount;
}
public int getDeltaReqCount() {
public long getDeltaReqCount() {
return deltaReqCount;
}
public long getCurrentReqCount() {
return lastTotalReqCount;
}
public void resetMetrics() {
deltaPromptTokens = 0;
deltaCompleteTokens = 0;
deltaDuration = 0;
maxDuration = 0;
deltaReqCount = 0;
}
}

public LLMDc(Map<String, Object> properties, CustomDcConfig cdcConfig) throws Exception {
super(properties, cdcConfig);
pricePromptTokens = (Double) properties.getOrDefault(PRICE_PROMPT_TOKES_PER_KILO, 0.0);
priceCompleteTokens = (Double) properties.getOrDefault(PRICE_COMPLETE_TOKES_PER_KILO, 0.0);
watsonxPricePromptTokens = (Double) properties.getOrDefault(WATSONX_PRICE_PROMPT_TOKES_PER_KILO, 0.0);
watsonxPriceCompleteTokens = (Double) properties.getOrDefault(WATSONX_PRICE_COMPLETE_TOKES_PER_KILO, 0.0);
openaiPricePromptTokens = (Double) properties.getOrDefault(OPENAI_PRICE_PROMPT_TOKES_PER_KILO, 0.0);
openaiPriceCompleteTokens = (Double) properties.getOrDefault(OPENAI_PRICE_COMPLETE_TOKES_PER_KILO, 0.0);
anthropicPricePromptTokens = (Double) properties.getOrDefault(ANTHROPIC_PRICE_PROMPT_TOKES_PER_KILO, 0.0);
anthropicPriceCompleteTokens = (Double) properties.getOrDefault(ANTHROPIC_PRICE_COMPLETE_TOKES_PER_KILO, 0.0);
listenPort = (int) properties.getOrDefault(SERVICE_LISTEN_PORT, 8000);
}

Expand Down Expand Up @@ -187,66 +204,113 @@ public void collectData() {
metricsCollector.clearMetrics();
for (OtelMetric metric : otelMetrics) {
try {
String modelId = metric.getModelId();
long promptTokens = metric.getPromtTokens();
long completeTokens = metric.getCompleteTokens();
double duration = metric.getDuration();
if(duration == 0.0) {
continue;
}
String modelId = metric.getModelId();
String aiSystem = metric.getAiSystem();
long requestCount = metric.getReqCount();

ModelAggregation modelAggr = modelAggrMap.get(modelId);
if (modelAggr == null) {
modelAggr = new ModelAggregation(modelId, "llmUser");
modelAggr = new ModelAggregation(modelId, aiSystem);
modelAggrMap.put(modelId, modelAggr);
}

modelAggr.addDeltaPromptTokens((int)(promptTokens));
modelAggr.addDeltaCompleteTokens((int)(completeTokens));
modelAggr.addDeltaDuration((int)(duration*1000));
modelAggr.addDeltaReqCount((int)(requestCount));
modelAggr.addDeltaDuration((long)(duration*1000), requestCount);
modelAggr.addDeltaReqCount(requestCount);
} catch (Exception e) {
e.printStackTrace();
}
}
for (OtelMetric metric : otelMetrics) {
try {
String modelId = metric.getModelId();
String aiSystem = metric.getAiSystem();
long promptTokens = metric.getPromtTokens();
long completeTokens = metric.getCompleteTokens();
if(promptTokens == 0 && completeTokens == 0) {
continue;
}
ModelAggregation modelAggr = modelAggrMap.get(modelId);
if (modelAggr == null) {
modelAggr = new ModelAggregation(modelId, aiSystem);
modelAggrMap.put(modelId, modelAggr);
}
long currentReqCount = modelAggr.getCurrentReqCount();
if(promptTokens > 0) {
modelAggr.addDeltaPromptTokens(promptTokens, currentReqCount);
}
if(completeTokens > 0) {
modelAggr.addDeltaCompleteTokens(completeTokens, currentReqCount);
}
} catch (Exception e) {
e.printStackTrace();
}
}

System.out.println("-----------------------------------------");
logger.info("-----------------------------------------");
for(Map.Entry<String,ModelAggregation> entry : modelAggrMap.entrySet()){
ModelAggregation aggr = entry.getValue();
String modelId = aggr.getModelId();
String userId = aggr.getUserId();
int deltaRequestCount = aggr.getDeltaReqCount();
int deltaDuration = aggr.getDeltaDuration();
int deltaPromptTokens = aggr.getDeltaPromptTokens();
int deltaCompleteTokens = aggr.getDeltaCompleteTokens();
int maxDuration = aggr.getMaxDuration();

int avgDuration = deltaDuration/(deltaRequestCount==0?1:deltaRequestCount);
double intervalReqCount = (double)deltaRequestCount/LLM_POLL_INTERVAL;
double intervalPromptTokens = (double)deltaPromptTokens/LLM_POLL_INTERVAL;
double intervalCompleteTokens = (double)deltaCompleteTokens/LLM_POLL_INTERVAL;
String aiSystem = aggr.getAiSystem();
long deltaRequestCount = aggr.getDeltaReqCount();
long deltaDuration = aggr.getDeltaDuration();
long deltaPromptTokens = aggr.getDeltaPromptTokens();
long deltaCompleteTokens = aggr.getDeltaCompleteTokens();
long maxDuration = aggr.getMaxDuration();

long avgDuration = deltaDuration/(deltaRequestCount==0?1:deltaRequestCount);
if(avgDuration > maxDuration) {
maxDuration = avgDuration;
aggr.setMaxDuration(maxDuration);
}

int intervalSeconds = LLM_POLL_INTERVAL;
String agentLess = System.getenv("AGENTLESS_MODE_ENABLED");
if (agentLess != null) {
intervalSeconds = 1;
}

double pricePromptTokens = 0.0;
double priceCompleteTokens = 0.0;
if (aiSystem.compareTo("watsonx") == 0) {
pricePromptTokens = watsonxPricePromptTokens;
priceCompleteTokens = watsonxPriceCompleteTokens;
} else if (aiSystem.compareTo("openai") == 0) {
pricePromptTokens = openaiPricePromptTokens;
priceCompleteTokens = openaiPriceCompleteTokens;
} else if (aiSystem.compareTo("anthropic") == 0) {
pricePromptTokens = anthropicPricePromptTokens;
priceCompleteTokens = anthropicPriceCompleteTokens;
}
double intervalReqCount = (double)deltaRequestCount/intervalSeconds;
double intervalPromptTokens = (double)deltaPromptTokens/intervalSeconds;
double intervalCompleteTokens = (double)deltaCompleteTokens/intervalSeconds;
double intervalTotalTokens = intervalPromptTokens + intervalCompleteTokens;
double intervalPromptCost = (intervalPromptTokens/1000) * pricePromptTokens;
double intervalCompleteCost = (intervalCompleteTokens/1000) * priceCompleteTokens;
double intervalTotalCost = intervalPromptCost + intervalCompleteCost;
aggr.resetMetrics();

System.out.println("ModelId : " + modelId);
System.out.println("UserId : " + userId);
System.out.println("AvgDuration : " + avgDuration);
System.out.println("MaxDuration : " + maxDuration);
System.out.println("IntervalTokens : " + intervalTotalTokens);
System.out.println("IntervalCost : " + intervalTotalCost);
System.out.println("IntervalRequest : " + intervalReqCount);
logger.info("ModelId : " + modelId);
logger.info("AiSystem : " + aiSystem);
logger.info("AvgDuration : " + avgDuration);
logger.info("MaxDuration : " + maxDuration);
logger.info("IntervalTokens : " + intervalTotalTokens);
logger.info("IntervalCost : " + intervalTotalCost);
logger.info("IntervalRequest : " + intervalReqCount);

Map<String, Object> attributes = new HashMap<>();
attributes.put("model_id", modelId);
attributes.put("user_id", userId);
attributes.put("ai_system", aiSystem);
getRawMetric(LLM_STATUS_NAME).setValue(1);
getRawMetric(LLM_DURATION_NAME).getDataPoint(modelId).setValue(avgDuration, attributes);
getRawMetric(LLM_DURATION_MAX_NAME).getDataPoint(modelId).setValue(maxDuration, attributes);
getRawMetric(LLM_COST_NAME).getDataPoint(modelId).setValue(intervalTotalCost, attributes);
getRawMetric(LLM_TOKEN_NAME).getDataPoint(modelId).setValue(intervalTotalTokens, attributes);
getRawMetric(LLM_REQ_COUNT_NAME).getDataPoint(modelId).setValue(intervalReqCount, attributes);
}
System.out.println("-----------------------------------------");
logger.info("-----------------------------------------");
}
}
Loading

0 comments on commit f571fc3

Please sign in to comment.