diff --git a/10-observability/observability-models-ollama/README.md b/10-observability/observability-models-ollama/README.md new file mode 100644 index 0000000..beead97 --- /dev/null +++ b/10-observability/observability-models-ollama/README.md @@ -0,0 +1,85 @@ +# LLM Observability: Ollama + +LLM Observability for Ollama. + +## Running the application + +The application relies on Ollama for providing LLMs. The application also relies on Testcontainers to provision automatically +a Grafana LGTM observability stack. + +### Ollama as a native application + +First, make sure you have [Ollama](https://ollama.ai) installed on your laptop. +Then, use Ollama to run the _mistral_ and _nomic-embed-text_ models. Those are the ones we'll use in this example. + +```shell +ollama run mistral +ollama run nomic-embed-text +``` + +Finally, run the Spring Boot application. + +```shell +./gradlew bootTestRun +``` + +## Observability Platform + +Grafana is listening to port 3000. Check your container runtime to find the port to which is exposed to your localhost +and access Grafana from http://localhost:. The credentials are `admin`/`admin`. + +The application is automatically configured to export metrics and traces to the Grafana LGTM stack via OpenTelemetry. +In Grafana, you can query the traces from the "Explore" page, selecting the "Tempo" data source. You can also visualize metrics in "Explore > Metrics". + +## Calling the application + +You can now call the application to perform generative AI operations. +This example uses [httpie](https://httpie.io) to send HTTP requests. + +### Chat + +```shell +http :8080/chat +``` + +Try passing your custom prompt and check the result. + +```shell +http :8080/chat message=="What is the capital of Italy?" +``` + +The next request is configured with a custom temperature value to obtain a more creative, yet less precise answer. + +```shell +http :8080/chat/generic-options message=="Why is a raven like a writing desk? Give a short answer." +``` + +The next request is configured with Ollama-specific customizations. + +```shell +http :8080/chat/ollama-options message=="What can you see beyond what you can see? Give a short answer." +``` + +Finally, try a request which uses function calling. + +```shell +http :8080/chat/functions authorName=="Philip Pullman" +``` + +### Embedding + +```shell +http :8080/embed +``` + +Try passing your custom prompt and check the result. + +```shell +http :8080/embed message=="The capital of Italy is Rome" +``` + +The next request is configured with Ollama-specific customizations. + +```shell +http :8080/embed/ollama-options message=="The capital of Italy is Rome" +``` diff --git a/10-observability/observability-models-ollama/build.gradle b/10-observability/observability-models-ollama/build.gradle new file mode 100644 index 0000000..242b77d --- /dev/null +++ b/10-observability/observability-models-ollama/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'java' + id 'org.springframework.boot' + id 'io.spring.dependency-management' +} + +group = 'com.thomasvitale' +version = '0.0.1-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(22) + } +} + +repositories { + mavenLocal() + mavenCentral() + maven { url 'https://repo.spring.io/milestone' } + maven { url 'https://repo.spring.io/snapshot' } +} + +dependencies { + implementation platform("org.springframework.ai:spring-ai-bom:${springAiVersion}") + + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.ai:spring-ai-ollama-spring-boot-starter' + + implementation 'io.micrometer:micrometer-tracing-bridge-otel' + implementation 'io.opentelemetry:opentelemetry-exporter-otlp' + implementation 'io.micrometer:micrometer-registry-otlp' + implementation 'net.ttddyy.observation:datasource-micrometer-spring-boot:1.0.5' + + testAndDevelopmentOnly 'org.springframework.boot:spring-boot-devtools' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.springframework.boot:spring-boot-testcontainers' + testImplementation 'org.testcontainers:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/BookService.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/BookService.java new file mode 100644 index 0000000..4ea663e --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/BookService.java @@ -0,0 +1,40 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Service +public class BookService { + + private static final Map books = new ConcurrentHashMap<>(); + + static { + books.put(1, new Book("His Dark Materials", "Philip Pullman")); + books.put(2, new Book("Narnia", "C.S. Lewis")); + books.put(3, new Book("The Hobbit", "J.R.R. Tolkien")); + books.put(4, new Book("The Lord of The Rings", "J.R.R. Tolkien")); + books.put(5, new Book("The Silmarillion", "J.R.R. Tolkien")); + } + + List getBooksByAuthor(Author author) { + return books.values().stream() + .filter(book -> author.name().equals(book.author())) + .toList(); + } + + Book getBestsellerByAuthor(Author author) { + return switch (author.name()) { + case "J.R.R. Tolkien" -> books.get(4); + case "C.S. Lewis" -> books.get(2); + case "Philip Pullman" -> books.get(1); + default -> null; + }; + } + + public record Book(String title, String author) {} + public record Author(String name) {} + +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ChatController.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ChatController.java new file mode 100644 index 0000000..f62c513 --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ChatController.java @@ -0,0 +1,65 @@ +package com.thomasvitale.ai.spring; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.ChatOptionsBuilder; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.ollama.api.OllamaOptions; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Set; + +@RestController +class ChatController { + + private final Logger logger = LoggerFactory.getLogger(ChatController.class); + + private final ChatModel chatModel; + + ChatController(ChatModel chatModel) { + this.chatModel = chatModel; + } + + @GetMapping("/chat") + String chat(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + logger.info(message); + return chatModel.call(message); + } + + @GetMapping("/chat/generic-options") + String chatWithGenericOptions(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + return chatModel.call(new Prompt(message, ChatOptionsBuilder.builder() + .withTemperature(1.3f) + .build())) + .getResult().getOutput().getContent(); + } + + @GetMapping("/chat/ollama-options") + String chatWithOllamaOptions(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + return chatModel.call(new Prompt(message, OllamaOptions.builder() + .withFrequencyPenalty(1.3f) + .withNumPredict(1500) + .withPresencePenalty(1.0f) + .withStop(List.of("this-is-the-end", "addio")) + .withTemperature(0.7f) + .withTopK(1) + .withTopP(0f) + .build())) + .getResult().getOutput().getContent(); + } + + @GetMapping("/chat/functions") + String chatWithFunctions(@RequestParam(defaultValue = "Philip Pullman") String author) { + return chatModel.call(new Prompt("What books written by %s are available to read and what is their bestseller?".formatted(author), + OllamaOptions.builder() + .withTemperature(0.3f) + .withFunctions(Set.of("booksByAuthor", "bestsellerBookByAuthor")) + .build())) + .getResult().getOutput().getContent(); + } + +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java new file mode 100644 index 0000000..a4e0313 --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java @@ -0,0 +1,35 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.ollama.api.OllamaOptions; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +class EmbeddingController { + + private final EmbeddingModel embeddingModel; + + EmbeddingController(EmbeddingModel embeddingModel) { + this.embeddingModel = embeddingModel; + } + + @GetMapping("/embed") + String embed(@RequestParam(defaultValue = "And Gandalf yelled: 'You shall not pass!'") String message) { + var embeddings = embeddingModel.embed(message); + return "Size of the embedding vector: " + embeddings.length; + } + + @GetMapping("/embed/ollama-options") + String embedWithOllamaOptions(@RequestParam(defaultValue = "And Gandalf yelled: 'You shall not pass!'") String message) { + var embeddings = embeddingModel.call(new EmbeddingRequest(List.of(message), OllamaOptions.builder() + .build())) + .getResult().getOutput(); + return "Size of the embedding vector: " + embeddings.length; + } + +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/Functions.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/Functions.java new file mode 100644 index 0000000..da3d930 --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/Functions.java @@ -0,0 +1,25 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Description; + +import java.util.List; +import java.util.function.Function; + +@Configuration(proxyBeanMethods = false) +public class Functions { + + @Bean + @Description("Get the list of available books written by the given author") + public Function> booksByAuthor(BookService bookService) { + return bookService::getBooksByAuthor; + } + + @Bean + @Description("Get the bestseller book written by the given author") + public Function bestsellerBookByAuthor(BookService bookService) { + return bookService::getBestsellerByAuthor; + } + +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java new file mode 100644 index 0000000..71e3f35 --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java @@ -0,0 +1,27 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.web.client.ClientHttpRequestFactories; +import org.springframework.boot.web.client.ClientHttpRequestFactorySettings; +import org.springframework.boot.web.client.RestClientCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.BufferingClientHttpRequestFactory; + +import java.time.Duration; + +@Configuration(proxyBeanMethods = false) +public class HttpClientConfig { + + @Bean + RestClientCustomizer restClientCustomizer() { + return restClientBuilder -> { + restClientBuilder + .requestFactory(new BufferingClientHttpRequestFactory( + ClientHttpRequestFactories.get(ClientHttpRequestFactorySettings.DEFAULTS + .withConnectTimeout(Duration.ofSeconds(60)) + .withReadTimeout(Duration.ofSeconds(60)) + ))); + }; + } + +} diff --git a/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplication.java b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplication.java new file mode 100644 index 0000000..086023c --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplication.java @@ -0,0 +1,13 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ObservabilityModelsOllamaApplication { + + public static void main(String[] args) { + SpringApplication.run(ObservabilityModelsOllamaApplication.class, args); + } + +} diff --git a/10-observability/observability-models-ollama/src/main/resources/application.yml b/10-observability/observability-models-ollama/src/main/resources/application.yml new file mode 100644 index 0000000..1148f49 --- /dev/null +++ b/10-observability/observability-models-ollama/src/main/resources/application.yml @@ -0,0 +1,34 @@ +spring: + application: + name: observability-models-ollama + ai: + chat: + observations: + include-completion: true + include-prompt: true + image: + observations: + include-prompt: true + ollama: + chat: + options: + model: mistral + temperature: 0.7 + embedding: + options: + model: nomic-embed-text + +management: + endpoints: + web: + exposure: + include: "*" + metrics: + tags: + service.name: ${spring.application.name} + tracing: + sampling: + probability: 1.0 + otlp: + tracing: + endpoint: http://localhost:4318/v1/traces diff --git a/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplicationTests.java b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplicationTests.java new file mode 100644 index 0000000..1bc4392 --- /dev/null +++ b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOllamaApplicationTests.java @@ -0,0 +1,13 @@ +package com.thomasvitale.ai.spring; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class ObservabilityModelsOllamaApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOllamaApplication.java b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOllamaApplication.java new file mode 100644 index 0000000..2be8fda --- /dev/null +++ b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOllamaApplication.java @@ -0,0 +1,11 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.SpringApplication; + +public class TestObservabilityModelsOllamaApplication { + + public static void main(String[] args) { + SpringApplication.from(ObservabilityModelsOllamaApplication::main).with(TestcontainersConfiguration.class).run(args); + } + +} diff --git a/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java new file mode 100644 index 0000000..66e5c8f --- /dev/null +++ b/10-observability/observability-models-ollama/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java @@ -0,0 +1,29 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.devtools.restart.RestartScope; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Scope; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; + +@TestConfiguration(proxyBeanMethods = false) +class TestcontainersConfiguration { + + @Bean + @RestartScope + @Scope("singleton") + @ServiceConnection("otel/opentelemetry-collector-contrib") + GenericContainer lgtmContainer() { + return new GenericContainer<>("docker.io/grafana/otel-lgtm:0.7.1") + .withExposedPorts(3000, 4317, 4318) + .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "500") + .waitingFor(Wait.forLogMessage(".*The OpenTelemetry collector and the Grafana LGTM stack are up and running.*\\s", 1)) + .withStartupTimeout(Duration.ofMinutes(2)) + .withReuse(true); + } + +} diff --git a/README.md b/README.md index 4f87985..7486e4c 100644 --- a/README.md +++ b/README.md @@ -98,9 +98,10 @@ _Coming soon_ ### 10. Observability -| Project | Description | -|---------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| -| [observability-models-openai](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/10-observability/observability-models-openai) | LLM Observability for OpenAI. | +| Project | Description | +|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| +| [observability-models-ollama](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/10-observability/observability-models-ollama) | LLM Observability for Ollama. | +| [observability-models-openai](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/10-observability/observability-models-openai) | LLM Observability for OpenAI. | | [observability-vector-stores-pgvector](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/10-observability/observability-vector-stores-pgvector) | Vector Store Observability for PGVector. | ## References and Additional Resources diff --git a/settings.gradle b/settings.gradle index 1e5f4aa..1d76b30 100644 --- a/settings.gradle +++ b/settings.gradle @@ -47,5 +47,6 @@ include '08-image-models:image-models-openai' include '09-audio-models:audio-models-speech-openai' include '09-audio-models:audio-models-transcription-openai' +include '10-observability:observability-models-ollama' include '10-observability:observability-models-openai' include '10-observability:observability-vector-stores-pgvector'