Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into OPIK-653
Browse files Browse the repository at this point in the history
  • Loading branch information
japdubengsub committed Jan 9, 2025
2 parents 4e3dbdb + a9dfd69 commit a9f87ef
Show file tree
Hide file tree
Showing 133 changed files with 3,204 additions and 619 deletions.
36 changes: 36 additions & 0 deletions .github/workflows/backend_formatting_check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: "Backend Formatting Check"
run-name: "Backend Formatting Check on ${{ github.ref_name }} by @${{ github.actor }}"

on:
pull_request:
paths:
- "apps/opik-backend/**/*.java"
push:
branches:
- "main"
paths:
- "apps/opik-backend/**/*.java"

workflow_dispatch:

jobs:
run-backend-formatting-check:
runs-on: ubuntu-latest
defaults:
run:
working-directory: apps/opik-backend/
steps:
- name: Checkout
uses: actions/[email protected]
with:
fetch-depth: 1

- name: Set up JDK 21
uses: actions/setup-java@v4
with:
java-version: "21"
distribution: "corretto"
cache: maven

- name: Run Formatting Check for backend
run: mvn clean spotless:check
87 changes: 87 additions & 0 deletions .github/workflows/documentation_codeblock_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
name: Documentation - Test codeblocks
on:
workflow_dispatch:
inputs:
install_opik:
description: 'Enable opik installation from source files'
required: false
default: 'false'
type: choice
options:
- 'false'
- 'true'
pull_request:
paths:
- 'apps/opik-documentation/documentation/docs/*.md'
- 'apps/opik-documentation/documentation/docs/*.mdx'
- 'apps/opik-documentation/documentation/docs/**/*.md'
- 'apps/opik-documentation/documentation/docs/**/*.mdx'

jobs:
collect_test_paths:
runs-on: ubuntu-latest
outputs:
test_paths: ${{ steps.paths.outputs.paths }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # Fetch all history for git diff

- id: paths
working-directory: apps/opik-documentation/documentation
run: |
# Get list of changed files in docs directory
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
# For pull requests, compare with base branch
echo "paths=$(
git diff --name-only origin/${{ github.base_ref }} |
grep -E '^apps/opik-documentation/documentation/docs/.*\.(md|mdx)$' |
sed 's|apps/opik-documentation/documentation/||' |
jq -R -s -c 'split("\n")[:-1]'
)" >> $GITHUB_OUTPUT
else
# For manual runs and scheduled runs, check all files
echo "paths=$(
(
ls -d docs/*/ 2>/dev/null;
find docs -maxdepth 1 -type f -name "*.md" -o -name "*.mdx"
) | jq -R -s -c 'split("\n")[:-1]'
)" >> $GITHUB_OUTPUT
fi
test:
needs: collect_test_paths
runs-on: ubuntu-latest
env:
OPENAI_API_KEY: ${{ secrets.DOCS_OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPIK_WORKSPACE: ${{ secrets.COMET_WORKSPACE }}
OPIK_API_KEY: ${{ secrets.COMET_API_KEY }}
strategy:
matrix:
path: ${{ fromJson(needs.collect_test_paths.outputs.test_paths) }}
fail-fast: false
steps:
- uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'

- name: Install dependencies
working-directory: apps/opik-documentation/documentation
run: |
python -m pip install --upgrade pip
pip install pytest
pip install -r requirements.txt
if [ "${{ github.event.inputs.install_opik }}" = "true" ]; then
pip install -e .
fi
- name: Run tests
working-directory: apps/opik-documentation/documentation
run: |
if [ -n "${{ matrix.path }}" ]; then
pytest ${{ matrix.path }} -v --suppress-no-test-exit-code
fi
1 change: 1 addition & 0 deletions .github/workflows/documentation_cookbook_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ jobs:
- apps/opik-documentation/documentation/docs/cookbook/openai.ipynb
- apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb
- apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb
- apps/opik-documentation/documentation/docs/cookbook/dspy.ipynb
env:
NOTEBOOK_TO_TEST: ${{ matrix.notebooks }}
steps:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
target/
**/dependency-reduced-pom.xml

# BE related
/apps/opik-backend/redoc/openapi.yaml

# FE related
/apps/opik-frontend/dist
/apps/opik-frontend/build
Expand Down
17 changes: 17 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,23 @@ Replace `{project.pom.version}` with the version of the project in the pom file.

Once the backend is running, you can access the Opik API at `http://localhost:8080`.

#### Formatting the code

Before submitting a PR, please ensure that your code is formatted correctly.
Run the following command to automatically format your code:

```bash
mvn spotless:apply
```

Our CI will check that the code is formatted correctly and will fail if it is not by running the following command:

```bash
mvn spotless:check
```

#### Testing the backend

Before submitting a PR, please ensure that your code passes the test suite:

```bash
Expand Down
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,10 @@ The easiest way to get started is to use one of our integrations. Opik supports:
| OpenAI | Log traces for all OpenAI LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/openai/?utm_source=opik&utm_medium=github&utm_content=openai_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb) |
| LiteLLM | Call any LLM model using the OpenAI format | [Documentation](/tracing/integrations/litellm.md) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb) |
| LangChain | Log traces for all LangChain LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/langchain/?utm_source=opik&utm_medium=github&utm_content=langchain_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb) |
| Haystack | Log traces for all Haystack calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/haystack/?utm_source=opik&utm_medium=github&utm_content=haystack_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb) |
| Bedrock | Log traces for all Bedrock LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/bedrock?utm_source=opik&utm_medium=github&utm_content=bedrock_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb) |
| Haystack | Log traces for all Haystack calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/haystack/?utm_source=opik&utm_medium=github&utm_content=haystack_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb) |
| Anthropic | Log traces for all Anthropic LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/anthropic?utm_source=opik&utm_medium=github&utm_content=anthropic_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb) |
| Bedrock | Log traces for all Bedrock LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/bedrock?utm_source=opik&utm_medium=github&utm_content=bedrock_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb) |
| DSPy | Log traces for all DSPy runs | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/dspy?utm_source=opik&utm_medium=github&utm_content=dspy_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/dspy.ipynb) |
| Gemini | Log traces for all Gemini LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/gemini?utm_source=opik&utm_medium=github&utm_content=gemini_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb) |
| Groq | Log traces for all Groq LLM calls | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/groq?utm_source=opik&utm_medium=github&utm_content=groq_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb) |
| LangGraph | Log traces for all LangGraph executions | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/langgraph/?utm_source=opik&utm_medium=github&utm_content=langchain_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb) |
Expand Down
14 changes: 14 additions & 0 deletions apps/opik-backend/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,22 @@ llmProviderClient:
# Default: 60s
# Description: Write timeout for LLM providers
writeTimeout: ${LLM_PROVIDER_CLIENT_WRITE_TIMEOUT:-60s}
# Default: false
# Description: Whether or not to log requests
logRequests: ${LLM_PROVIDER_CLIENT_LOG_REQUESTS:-false}
# Default: false
# Description: Whether or not to log responses
logResponses: ${LLM_PROVIDER_CLIENT_LOG_RESPONSES:-false}
# Configuration for OpenAI client
openAiClient:
# Default:
# Description: OpenAI API URL
url: ${LLM_PROVIDER_OPENAI_URL:-}
# Configuration for Anthropic client
anthropicClient:
# Default: https://api.anthropic.com/v1/
# Description: Anthropic API URL
url: ${LLM_PROVIDER_ANTHROPIC_URL:-https://api.anthropic.com/v1/}
# Default: 2023-06-01
# Description: Anthropic API version https://docs.anthropic.com/en/api/versioning
version: ${LLM_PROVIDER_ANTHROPIC_VERSION:-'2023-06-01'}
10 changes: 10 additions & 0 deletions apps/opik-backend/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
<redisson.version>3.41.0</redisson.version>
<opentelmetry.version>2.10.0</opentelmetry.version>
<aws.java.sdk.version>2.29.9</aws.java.sdk.version>
<json-path.version>2.9.0</json-path.version>
<mainClass>com.comet.opik.OpikApplication</mainClass>
</properties>

Expand Down Expand Up @@ -207,10 +208,19 @@
<artifactId>java-uuid-generator</artifactId>
<version>${uuid.java.generator.version}</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>${json-path.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-open-ai</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-anthropic</artifactId>
</dependency>

<!-- Test -->

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ public void initialize(Bootstrap<OpikConfiguration> bootstrap) {
@Override
public void run(OpikConfiguration configuration, Environment environment) {
EncryptionUtils.setConfig(configuration);

// Resources
var jersey = environment.jersey();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,15 @@
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.JsonNode;
import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import lombok.experimental.SuperBuilder;

import java.beans.ConstructorProperties;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
Expand All @@ -24,44 +20,23 @@
@SuperBuilder(toBuilder = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type", visible = true)
@JsonSubTypes({
@JsonSubTypes.Type(value = AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge.class, name = "llm_as_judge")
@JsonSubTypes.Type(value = AutomationRuleEvaluatorLlmAsJudge.class, name = "llm_as_judge")
})
@Schema(name = "AutomationRuleEvaluator", discriminatorProperty = "type", discriminatorMapping = {
@DiscriminatorMapping(value = "llm_as_judge", schema = AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge.class)
@DiscriminatorMapping(value = "llm_as_judge", schema = AutomationRuleEvaluatorLlmAsJudge.class)
})
@AllArgsConstructor
public abstract sealed class AutomationRuleEvaluator<T> implements AutomationRule<T> {

@EqualsAndHashCode(callSuper = true)
@Data
@SuperBuilder(toBuilder = true)
@ToString(callSuper = true)
public static final class AutomationRuleEvaluatorLlmAsJudge extends AutomationRuleEvaluator<JsonNode> {

@NotNull @JsonView({View.Public.class, View.Write.class})
@Schema(accessMode = Schema.AccessMode.READ_WRITE)
JsonNode code;

@ConstructorProperties({"id", "projectId", "name", "samplingRate", "code", "createdAt", "createdBy", "lastUpdatedAt", "lastUpdatedBy"})
public AutomationRuleEvaluatorLlmAsJudge(UUID id, UUID projectId, @NotBlank String name, float samplingRate, @NotNull JsonNode code,
Instant createdAt, String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
super(id, projectId, name, samplingRate, createdAt, createdBy, lastUpdatedAt, lastUpdatedBy);
this.code = code;
}

@Override
public AutomationRuleEvaluatorType type() {
return AutomationRuleEvaluatorType.LLM_AS_JUDGE;
}
}
public abstract sealed class AutomationRuleEvaluator<T>
implements
AutomationRule<T>
permits AutomationRuleEvaluatorLlmAsJudge {

@JsonView({View.Public.class})
@Schema(accessMode = Schema.AccessMode.READ_ONLY)
UUID id;

@JsonView({View.Public.class, View.Write.class})
@NotNull
UUID projectId;
@NotNull UUID projectId;

@JsonView({View.Public.class, View.Write.class})
@Schema(accessMode = Schema.AccessMode.READ_WRITE)
Expand Down Expand Up @@ -100,17 +75,21 @@ public AutomationRuleAction getAction() {
}

public static class View {
public static class Write {}
public static class Public {}
public static class Write {
}
public static class Public {
}
}

@Builder(toBuilder = true)
public record AutomationRuleEvaluatorPage(
@JsonView({View.Public.class}) int page,
@JsonView( {
View.Public.class}) int page,
@JsonView({View.Public.class}) int size,
@JsonView({View.Public.class}) long total,
@JsonView({View.Public.class}) List<AutomationRuleEvaluatorLlmAsJudge> content)
implements Page<AutomationRuleEvaluatorLlmAsJudge>{
implements
Page<AutomationRuleEvaluatorLlmAsJudge>{

public static AutomationRuleEvaluator.AutomationRuleEvaluatorPage empty(int page) {
return new AutomationRuleEvaluator.AutomationRuleEvaluatorPage(page, 0, 0, List.of());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.comet.opik.api;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Builder;

import java.util.Set;
import java.util.UUID;

@Builder(toBuilder = true)
@JsonIgnoreProperties(ignoreUnknown = true)
public record AutomationRuleEvaluatorCriteria(
AutomationRuleEvaluatorType type,
String name,
Set<UUID> ids) {

public AutomationRule.AutomationRuleAction action() {
return AutomationRule.AutomationRuleAction.EVALUATOR;
}
}
Loading

0 comments on commit a9f87ef

Please sign in to comment.