Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/alert demo #3

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Feature/alert demo #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
059bbea
alert for epoch rewards distribution
Feb 4, 2020
bebcda9
Send alert into a topic
Feb 4, 2020
2b12adc
Send the alerts to a topic using the alerts schema
Feb 4, 2020
c014181
changed aggregation window
Feb 5, 2020
d96c069
changes
Feb 5, 2020
31797e0
config change
Feb 6, 2020
e034a61
Use avro schemas dependency
Feb 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
287 changes: 148 additions & 139 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,152 +1,161 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.keyko.monitoring</groupId>
<artifactId>keyko-event-streamer</artifactId>
<version>0.0.1</version>
<inceptionYear>2019</inceptionYear>
<properties>
<kafka.version>2.4.0</kafka.version>
<typesafe.config.version>1.4.0</typesafe.config.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<modelVersion>4.0.0</modelVersion>
<groupId>io.keyko.monitoring</groupId>
<artifactId>keyko-event-streamer</artifactId>
<version>0.0.1</version>
<inceptionYear>2019</inceptionYear>
<properties>
<kafka.version>2.4.0</kafka.version>
<confluent.version>5.3.2</confluent.version>
<avro.version>1.9.1</avro.version>
<log4j.version>2.13.0</log4j.version>
<org.json.version>20190722</org.json.version>
<typesafe.config.version>1.4.0</typesafe.config.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<keyko.schemas.version>0.1.0</keyko.schemas.version>
<junit.version>4.4</junit.version>
</properties>

</properties>

<repositories>
<repository>
<id>confluent</id>
<url>https://packages.confluent.io/maven/</url>
</repository>
</repositories>

<repositories>
<repository>
<id>confluent</id>
<url>https://packages.confluent.io/maven/</url>
</repository>
</repositories>

<dependencies>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>${typesafe.config.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-streams-avro-serde</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-schema-registry-client</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>common-config</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>common-utils</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.json/json -->
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>${org.json.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>io.keyko.monitoring</groupId>
<artifactId>schemas</artifactId>
<version>${keyko.schemas.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams-test-utils</artifactId>
<version>${kafka.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<dependencies>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>${typesafe.config.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-streams-avro-serde</artifactId>
<version>5.3.2</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-schema-registry-client</artifactId>
<version>5.3.2</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>common-config</artifactId>
<version>5.3.2</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>common-utils</artifactId>
<version>5.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.9.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.json/json -->
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20190722</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_2.12</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams-test-utils</artifactId>
<version>${kafka.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.2.4</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>

<build>
<plugins>

<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>allinone</shadedClassifierName>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>io.keyko.monitoring.KeykoMonitoring</Main-Class>
</manifestEntries>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<release>11</release>
</configuration>
</plugin>
</plugins>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>allinone</shadedClassifierName>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>io.keyko.monitoring.KeykoMonitoring</Main-Class>
</manifestEntries>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<release>11</release>
</configuration>
</plugin>
</plugins>

</build>
</build>

</project>
5 changes: 4 additions & 1 deletion src/main/java/io/keyko/monitoring/KeykoMonitoring.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
import com.typesafe.config.ConfigFactory;
import io.keyko.monitoring.config.StreamerConfig;
import io.keyko.monitoring.stream.EventStreamManager;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class KeykoMonitoring {
private static final Logger LOGGER = LogManager.getLogger(KeykoMonitoring.class);

public static void main(final String[] args) throws Exception {

Config config = args.length > 0 ? ConfigFactory.load(args[0]) : ConfigFactory.load();
StreamerConfig streamerConfig = StreamerConfig.getInstance(config);

LOGGER.info("Starting KeykoMonitoring...");
new EventStreamManager(streamerConfig).initStream();

}
}
11 changes: 11 additions & 0 deletions src/main/java/io/keyko/monitoring/config/StreamerConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ public class StreamerConfig {
private static final String EVENT_BLOCK_TOPIC = "kafka.event-block-topic";
private static final String FLAT_EVENT_TOPIC = "kafka.flat-event-topic";
private static final String ACCOUNTS_AGGREGATION_TOPIC = "kafka.accounts-aggregation-topic";
private static final String ALERTS_TOPIC = "kafka.alerts-topic";


private StreamerConfig() {
Expand All @@ -23,6 +24,7 @@ private StreamerConfig() {
private String eventBlockTopic;
private String flatEventTopic;
private String accountsAggregationTopic;
private String alertsTopic;


private static StreamerConfig streamerConfig;
Expand All @@ -41,6 +43,7 @@ public static StreamerConfig getInstance(Config config) {
streamerConfig.setEventBlockTopic(config.getString(EVENT_BLOCK_TOPIC));
streamerConfig.setFlatEventTopic(config.getString(FLAT_EVENT_TOPIC));
streamerConfig.setAccountsAggregationTopic(config.getString(ACCOUNTS_AGGREGATION_TOPIC));
streamerConfig.setAlertsTopic(config.getString(ALERTS_TOPIC));


return streamerConfig;
Expand Down Expand Up @@ -104,4 +107,12 @@ public void setAccountsAggregationTopic(String accountsAggregationTopic) {
this.accountsAggregationTopic = accountsAggregationTopic;
}

public String getAlertsTopic() {
return alertsTopic;
}

public void setAlertsTopic(String alertsTopic) {
this.alertsTopic = alertsTopic;
}

}
8 changes: 8 additions & 0 deletions src/main/java/io/keyko/monitoring/model/AccountCreatedAggregation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package io.keyko.monitoring.model;

import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;

public class AccountCreatedAggregation extends BaseAggregation {

private static final String DATE_PATTERN = "dd/MM/yyyy HH:mm:ss";
private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DATE_PATTERN);

private String type;

Expand All @@ -13,6 +16,11 @@ public AccountCreatedAggregation(ZonedDateTime dateTime, Long count, String type
}


@Override
protected String formatDateTime(ZonedDateTime datetime) {
return datetime.format(formatter);
}

public String getType() {
return type;
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/io/keyko/monitoring/model/BaseAggregation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public BaseAggregation(ZonedDateTime dateTime, Long count) {
}


private String formatDateTime(ZonedDateTime datetime) {
protected String formatDateTime(ZonedDateTime datetime) {
return datetime.format(formatter);
}

Expand Down
Loading