From 6dae61170ce66af643dba2f48778fa2806d79176 Mon Sep 17 00:00:00 2001 From: Turker Koc Date: Mon, 11 Nov 2024 22:45:36 +0100 Subject: [PATCH 1/4] init webhook listener service --- server/webhook-listener/.gitignore | 164 +++++++++++++++++++++ server/webhook-listener/Dockerfile | 9 ++ server/webhook-listener/README.md | 86 +++++++++++ server/webhook-listener/__init__.py | 0 server/webhook-listener/app/__init__.py | 0 server/webhook-listener/app/config.py | 12 ++ server/webhook-listener/app/logger.py | 32 ++++ server/webhook-listener/app/main.py | 98 ++++++++++++ server/webhook-listener/app/nats_client.py | 62 ++++++++ server/webhook-listener/compose.yaml | 56 +++++++ server/webhook-listener/nats-server.conf | 11 ++ server/webhook-listener/requirements.txt | 36 +++++ 12 files changed, 566 insertions(+) create mode 100644 server/webhook-listener/.gitignore create mode 100644 server/webhook-listener/Dockerfile create mode 100644 server/webhook-listener/README.md create mode 100644 server/webhook-listener/__init__.py create mode 100644 server/webhook-listener/app/__init__.py create mode 100644 server/webhook-listener/app/config.py create mode 100644 server/webhook-listener/app/logger.py create mode 100644 server/webhook-listener/app/main.py create mode 100644 server/webhook-listener/app/nats_client.py create mode 100644 server/webhook-listener/compose.yaml create mode 100644 server/webhook-listener/nats-server.conf create mode 100644 server/webhook-listener/requirements.txt diff --git a/server/webhook-listener/.gitignore b/server/webhook-listener/.gitignore new file mode 100644 index 0000000..8954001 --- /dev/null +++ b/server/webhook-listener/.gitignore @@ -0,0 +1,164 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +/nats_data \ No newline at end of file diff --git a/server/webhook-listener/Dockerfile b/server/webhook-listener/Dockerfile new file mode 100644 index 0000000..d97fdff --- /dev/null +++ b/server/webhook-listener/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12 + +WORKDIR /code + +COPY requirements.txt /code/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +COPY ./app /code/app + +CMD ["fastapi", "run", "app/main.py", "--port", "4200"] diff --git a/server/webhook-listener/README.md b/server/webhook-listener/README.md new file mode 100644 index 0000000..d26d01a --- /dev/null +++ b/server/webhook-listener/README.md @@ -0,0 +1,86 @@ +# WebHook Listener + +## Overview + +A service to listen GitHub webhooks and publish the data to NATS JetStream. + +## Setup + +### Prerequisites + +- **Python 3.x.x** +- **Docker** for containerization + +## Environment Variables + +- `NATS_URL`: NATS server URL +- `NATS_AUTH_TOKEN`: Authorization token for NATS server +- `WEBHOOK_SECRET`: HMAC secret for verifying GitHub webhooks + +If you are using docker compose, you don't need to set NATS_URL for local development. + +Generate an AUTH TOKEN and Set the environment variable: + +```bash +export NATS_AUTH_TOKEN=$(openssl rand -hex 48) +``` + +Set Webhook secret +```bash +export WEBHOOK_SECRET= +``` + +## Running with Docker Compose + +Build and run with Docker Compose: + +```bash +docker-compose up --build +``` + +Service ports: +- **Webhook Service**: `4200` +- **NATS Server**: `4222` + + +## Usage + +Configure your GitHub webhooks to POST to: + +``` +https://:4200/github +``` + +### Event Handling + +Events are published to NATS with the subject: + +``` +github... +``` + + + +## Setup for Local Development +### Installation + +Install dependencies: + +```bash +pip install -r requirements.txt +``` + +### Running Service + + +```bash +fastapi dev #For Development +fastapi run #For Production +``` + +### Important Notes + +- The service automatically sets up a NATS JetStream stream named `github` to store events. +- Ensure your firewall allows traffic on port 4222 (NATS). +- Authentication tokens are crucial for securing the NATS server and ensuring only authorized clients can connect. +- The webhook listener service connects to the NATS server like any other client using the specified URL and token. \ No newline at end of file diff --git a/server/webhook-listener/__init__.py b/server/webhook-listener/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/webhook-listener/app/__init__.py b/server/webhook-listener/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/webhook-listener/app/config.py b/server/webhook-listener/app/config.py new file mode 100644 index 0000000..74eb44f --- /dev/null +++ b/server/webhook-listener/app/config.py @@ -0,0 +1,12 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + NATS_URL: str = "localhost" + NATS_AUTH_TOKEN: str = "" + WEBHOOK_SECRET: str = "" + + class Config: + env_file = ".env" + +settings = Settings() diff --git a/server/webhook-listener/app/logger.py b/server/webhook-listener/app/logger.py new file mode 100644 index 0000000..028a283 --- /dev/null +++ b/server/webhook-listener/app/logger.py @@ -0,0 +1,32 @@ +import logging +import sys +from typing import Any + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +stream_handler = logging.StreamHandler(sys.stdout) +log_formatter = logging.Formatter("%(levelname)s:\t %(message)s") +stream_handler.setFormatter(log_formatter) +logger.addHandler(stream_handler) + +logger.info("Logger initialized") + + +class EndpointFilter(logging.Filter): + def __init__( + self, + path: str, + *args: Any, + **kwargs: Any, + ): + super().__init__(*args, **kwargs) + self._path = path + + def filter(self, record: logging.LogRecord) -> bool: + return record.getMessage().find(self._path) == -1 + +uvicorn_logger = logging.getLogger("uvicorn.access") +uvicorn_logger.addFilter(EndpointFilter(path="/health")) + +uvicorn_error = logging.getLogger("uvicorn.error") \ No newline at end of file diff --git a/server/webhook-listener/app/main.py b/server/webhook-listener/app/main.py new file mode 100644 index 0000000..9a34e3d --- /dev/null +++ b/server/webhook-listener/app/main.py @@ -0,0 +1,98 @@ +import hmac +import hashlib +from contextlib import asynccontextmanager +from fastapi import Body, FastAPI, HTTPException, Header, Request, status +from pydantic import BaseModel +from nats.js.api import StreamConfig +from app.config import settings +from app.nats_client import nats_client + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await nats_client.connect() + await nats_client.js.add_stream(name="github", subjects=["github.>"], config=StreamConfig(storage="file")) + yield + await nats_client.close() + + +app = FastAPI(lifespan=lifespan) + + +def verify_github_signature(signature, secret, body): + mac = hmac.new(secret.encode(), body, hashlib.sha1) + expected_signature = "sha1=" + mac.hexdigest() + return hmac.compare_digest(signature, expected_signature) + + +@app.post("/github") +async def github_webhook( + request: Request, + signature: str = Header( + None, + alias="X-Hub-Signature", + description="GitHub's HMAC hex digest of the payload, used for verifying the webhook's authenticity" + ), + event_type: str = Header( + None, + alias="X-Github-Event", + description="The type of event that triggered the webhook, such as 'push', 'pull_request', etc.", + ), + body = Body(...), +): + body = await request.body() + + if not verify_github_signature(signature, settings.WEBHOOK_SECRET, body): + raise HTTPException(status_code=401, detail="Invalid signature") + + # Ignore ping events + if event_type == "ping": + return { "status": "pong" } + + # Extract subject from the payload + payload = await request.json() + + org = "?" + repo = "?" + if "repository" in payload: + org = payload["repository"]["owner"]["login"] + repo = payload["repository"]["name"] + elif "organization" in payload: + org = payload["organization"]["login"] + + org_sanitized = org.replace('.', '~') + repo_sanitized = repo.replace('.', '~') + + subject = f"github.{org_sanitized}.{repo_sanitized}.{event_type}" + + # Publish the payload to NATS JetStream + await nats_client.publish_with_retry(subject, body) + + return { "status": "ok" } + + +class HealthCheck(BaseModel): + """Response model to validate and return when performing a health check.""" + + status: str = "OK" + + +@app.get( + "/health", + tags=["healthcheck"], + summary="Perform a Health Check", + response_description="Return HTTP Status Code 200 (OK)", + status_code=status.HTTP_200_OK, + response_model=HealthCheck, +) +def get_health() -> HealthCheck: + """ + ## Perform a Health Check + Endpoint to perform a healthcheck on. This endpoint can primarily be used Docker + to ensure a robust container orchestration and management is in place. Other + services which rely on proper functioning of the API service will not deploy if this + endpoint returns any other HTTP status code except 200 (OK). + Returns: + HealthCheck: Returns a JSON response with the health status + """ + return HealthCheck(status="OK") \ No newline at end of file diff --git a/server/webhook-listener/app/nats_client.py b/server/webhook-listener/app/nats_client.py new file mode 100644 index 0000000..4fe7d3f --- /dev/null +++ b/server/webhook-listener/app/nats_client.py @@ -0,0 +1,62 @@ +import asyncio +from nats.aio.client import Client as NATS +from app.config import settings +from app.logger import logger, uvicorn_error +import ssl +class NATSClient: + MAX_RETRIES = 10 + RETRY_BACKOFF_FACTOR = 2 + + def __init__(self): + self.nc = NATS() + + async def connect(self): + async def error_cb(e): + logger.error(f'There was an error: {e}') + + async def disconnected_cb(): + logger.info('NATS got disconnected!') + + async def reconnected_cb(): + logger.info(f'NATS got reconnected to {self.nc.connected_url.netloc}') + + async def closed_cb(): + logger.info('NATS connection is closed') + + await self.nc.connect( + servers=settings.NATS_URL, + token=settings.NATS_AUTH_TOKEN, + max_reconnect_attempts=-1, + allow_reconnect=True, + reconnect_time_wait=2, + error_cb=error_cb, + disconnected_cb=disconnected_cb, + reconnected_cb=reconnected_cb, + closed_cb=closed_cb, + ) + self.js = self.nc.jetstream() + logger.info(f"Connected to NATS at {self.nc.connected_url.netloc}") + + async def publish(self, subject: str, message: bytes): + ack = await self.js.publish(subject, message) + logger.info(f"Published message to {subject}: {ack}") + return ack + + async def publish_with_retry(self, subject: str, message: bytes): + for attempt in range(self.MAX_RETRIES): + try: + ack = await self.publish(subject, message) + return ack # Successfully published, return the ack + except Exception as e: + uvicorn_error.error(f"NATS request failed: {e}, retrying in {wait_time} seconds... (Attempt {attempt + 1}/{self.MAX_RETRIES})") + wait_time = self.RETRY_BACKOFF_FACTOR ** attempt + await asyncio.sleep(wait_time) + + uvicorn_error.error(f"Failed to publish to {subject} after {self.MAX_RETRIES} attempts") + raise Exception(f"Failed to publish to {subject} after {self.MAX_RETRIES} attempts") + + async def close(self): + await self.nc.close() + + +nats_client = NATSClient() diff --git a/server/webhook-listener/compose.yaml b/server/webhook-listener/compose.yaml new file mode 100644 index 0000000..ae87a0a --- /dev/null +++ b/server/webhook-listener/compose.yaml @@ -0,0 +1,56 @@ +services: + webhook-listener: + build: . + ports: + - "4200:4200" + environment: + - NATS_URL=${NATS_URL:-nats://nats-server:4222} + - NATS_AUTH_TOKEN=${NATS_AUTH_TOKEN} + - WEBHOOK_SECRET=${WEBHOOK_SECRET} + depends_on: + nats-server: + condition: service_healthy + networks: + - common-network + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/health"] + interval: 5s + timeout: 10s + retries: 5 + start_period: 3s + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + nats-server: + image: nats:alpine + ports: + - "4222:4222" + - "8222:8222" + command: ["/bin/sh", "-c", "sed 's|{{NATS_AUTH_TOKEN}}|'\"$NATS_AUTH_TOKEN\"'|g' /etc/nats/nats-server.conf.template > /etc/nats/nats-server.conf && exec nats-server --config /etc/nats/nats-server.conf"] + environment: + - NATS_AUTH_TOKEN=${NATS_AUTH_TOKEN} + volumes: + - ./nats_data:/data + - ./nats-server.conf:/etc/nats/nats-server.conf.template + networks: + - common-network + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://localhost:8222/healthz"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 3s + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +networks: + common-network: + +volumes: + nats_data: diff --git a/server/webhook-listener/nats-server.conf b/server/webhook-listener/nats-server.conf new file mode 100644 index 0000000..58e0fe4 --- /dev/null +++ b/server/webhook-listener/nats-server.conf @@ -0,0 +1,11 @@ +listen: "0.0.0.0:4222" + +http_port: 8222 + +jetstream { + store_dir: "/data" +} + +authorization { + token: "{{NATS_AUTH_TOKEN}}" +} \ No newline at end of file diff --git a/server/webhook-listener/requirements.txt b/server/webhook-listener/requirements.txt new file mode 100644 index 0000000..2e24427 --- /dev/null +++ b/server/webhook-listener/requirements.txt @@ -0,0 +1,36 @@ +annotated-types==0.7.0 ; python_version >= "3.12" and python_version < "4.0" +anyio==4.4.0 ; python_version >= "3.12" and python_version < "4.0" +certifi==2024.7.4 ; python_version >= "3.12" and python_version < "4.0" +click==8.1.7 ; python_version >= "3.12" and python_version < "4.0" +colorama==0.4.6 ; python_version >= "3.12" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") +dnspython==2.6.1 ; python_version >= "3.12" and python_version < "4.0" +email-validator==2.2.0 ; python_version >= "3.12" and python_version < "4.0" +fastapi-cli[standard]==0.0.5 ; python_version >= "3.12" and python_version < "4.0" +fastapi[standard]==0.112.1 ; python_version >= "3.12" and python_version < "4.0" +h11==0.14.0 ; python_version >= "3.12" and python_version < "4.0" +httpcore==1.0.5 ; python_version >= "3.12" and python_version < "4.0" +httptools==0.6.1 ; python_version >= "3.12" and python_version < "4.0" +httpx==0.27.0 ; python_version >= "3.12" and python_version < "4.0" +idna==3.7 ; python_version >= "3.12" and python_version < "4.0" +jinja2==3.1.4 ; python_version >= "3.12" and python_version < "4.0" +markdown-it-py==3.0.0 ; python_version >= "3.12" and python_version < "4.0" +markupsafe==2.1.5 ; python_version >= "3.12" and python_version < "4.0" +mdurl==0.1.2 ; python_version >= "3.12" and python_version < "4.0" +nats-py==2.8.0 ; python_version >= "3.12" and python_version < "4.0" +pydantic-core==2.20.1 ; python_version >= "3.12" and python_version < "4.0" +pydantic-settings==2.4.0 ; python_version >= "3.12" and python_version < "4.0" +pydantic==2.8.2 ; python_version >= "3.12" and python_version < "4.0" +pygments==2.18.0 ; python_version >= "3.12" and python_version < "4.0" +python-dotenv==1.0.1 ; python_version >= "3.12" and python_version < "4.0" +python-multipart==0.0.9 ; python_version >= "3.12" and python_version < "4.0" +pyyaml==6.0.2 ; python_version >= "3.12" and python_version < "4.0" +rich==13.7.1 ; python_version >= "3.12" and python_version < "4.0" +shellingham==1.5.4 ; python_version >= "3.12" and python_version < "4.0" +sniffio==1.3.1 ; python_version >= "3.12" and python_version < "4.0" +starlette==0.38.2 ; python_version >= "3.12" and python_version < "4.0" +typer==0.12.4 ; python_version >= "3.12" and python_version < "4.0" +typing-extensions==4.12.2 ; python_version >= "3.12" and python_version < "4.0" +uvicorn[standard]==0.30.6 ; python_version >= "3.12" and python_version < "4.0" +uvloop==0.20.0 ; (sys_platform != "win32" and sys_platform != "cygwin") and platform_python_implementation != "PyPy" and python_version >= "3.12" and python_version < "4.0" +watchfiles==0.23.0 ; python_version >= "3.12" and python_version < "4.0" +websockets==13.0 ; python_version >= "3.12" and python_version < "4.0" From 479d54cd7bdeb56b3c76de7c7d24f5798f75e6e0 Mon Sep 17 00:00:00 2001 From: Ege Kocabas Date: Tue, 12 Nov 2024 00:38:02 +0100 Subject: [PATCH 2/4] init webhook receiver in application-server --- server/application-server/.env.example | 5 +- server/application-server/README.md | 2 +- server/application-server/build.gradle | 50 +++-- .../tum/cit/aet/helios/config/NatsConfig.java | 28 +++ .../common/github/GitHubMessageHandler.java | 62 +++++ .../github/GitHubMessageHandlerRegistry.java | 29 +++ .../github/GitHubIssueMessageHandler.java | 35 +++ .../helios/syncing/NatsConsumerService.java | 212 ++++++++++++++++++ .../src/main/resources/application.yml | 21 ++ .../aet/helios/HeliosApplicationTests.java | 6 +- server/webhook-listener/README.md | 7 +- 11 files changed, 425 insertions(+), 32 deletions(-) create mode 100644 server/application-server/src/main/java/de/tum/cit/aet/helios/config/NatsConfig.java create mode 100644 server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandler.java create mode 100644 server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandlerRegistry.java create mode 100644 server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/issue/github/GitHubIssueMessageHandler.java create mode 100644 server/application-server/src/main/java/de/tum/cit/aet/helios/syncing/NatsConsumerService.java create mode 100644 server/application-server/src/main/resources/application.yml diff --git a/server/application-server/.env.example b/server/application-server/.env.example index 02b2f58..b3196b8 100644 --- a/server/application-server/.env.example +++ b/server/application-server/.env.example @@ -1,3 +1,6 @@ DATASOURCE_URL=jdbc:postgresql://127.0.0.1/helios DATASOURCE_USERNAME=helios -DATASOURCE_PASSWORD=helios \ No newline at end of file +DATASOURCE_PASSWORD=helios +NATS_SERVER=localhost:4222 +REPOSITORY_NAME=# e.g. ls1intum/Helios +NATS_AUTH_TOKEN= diff --git a/server/application-server/README.md b/server/application-server/README.md index a4773bb..5bb93ec 100644 --- a/server/application-server/README.md +++ b/server/application-server/README.md @@ -28,7 +28,7 @@ $ ./gradlew build **3. Setup configuration and environment** -Copy the file `.env.example` to `.env` and adjust the values to your needs. It is automatically set up to work with the Docker Compose setup. +Copy the file `.env.example` to `.env` and adjust the values to your needs. It is set up to work with the Docker Compose setup for database. You need to adjust some fields for NATS server. ```bash $ cp .env.example .env diff --git a/server/application-server/build.gradle b/server/application-server/build.gradle index adadc59..900aafd 100644 --- a/server/application-server/build.gradle +++ b/server/application-server/build.gradle @@ -1,24 +1,24 @@ plugins { - id 'java' - id 'war' - id 'org.springframework.boot' version '3.3.4' - id 'io.spring.dependency-management' version '1.1.6' - id 'io.freefair.lombok' version '8.10.2' - id 'org.springdoc.openapi-gradle-plugin' version '1.9.0' - id 'org.openapi.generator' version '6.6.0' + id 'java' + id 'war' + id 'org.springframework.boot' version '3.3.4' + id 'io.spring.dependency-management' version '1.1.6' + id 'io.freefair.lombok' version '8.10.2' + id 'org.springdoc.openapi-gradle-plugin' version '1.9.0' + id 'org.openapi.generator' version '6.6.0' } group = 'de.tum.cit.aet' version = '0.0.1-SNAPSHOT' java { - toolchain { - languageVersion = JavaLanguageVersion.of(22) - } + toolchain { + languageVersion = JavaLanguageVersion.of(22) + } } repositories { - mavenCentral() + mavenCentral() } def loadEnvFile() { @@ -48,27 +48,29 @@ def loadEnvFile() { dependencies { - implementation 'org.springframework.boot:spring-boot-starter-data-jpa' - implementation 'org.springframework.boot:spring-boot-starter-web' - implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.6.0' - implementation 'org.openapitools:jackson-databind-nullable:0.2.6' - runtimeOnly 'org.postgresql:postgresql' - providedRuntime 'org.springframework.boot:spring-boot-starter-tomcat' - testImplementation 'org.springframework.boot:spring-boot-starter-test' - testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.6.0' + implementation 'org.openapitools:jackson-databind-nullable:0.2.6' + implementation 'io.nats:jnats:2.20.4' + implementation 'org.kohsuke:github-api:1.326' + runtimeOnly 'org.postgresql:postgresql' + providedRuntime 'org.springframework.boot:spring-boot-starter-tomcat' + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } tasks.named('test') { - useJUnitPlatform() + useJUnitPlatform() } openApi { apiDocsUrl = 'http://localhost:8080/v3/api-docs.yaml' outputDir = file('.') outputFileName = 'openapi.yaml' - def envVars = loadEnvFile() - customBootRun { - args.set(["--spring.profiles.active=dev", "--DATASOURCE_URL=${envVars['DATASOURCE_URL'] ?: ''}", "--DATASOURCE_USERNAME=${envVars['DATASOURCE_USERNAME'] ?: ''}", "--DATASOURCE_PASSWORD=${envVars['DATASOURCE_PASSWORD'] ?: ''}"]) - } + def envVars = loadEnvFile() + customBootRun { + args.set(["--spring.profiles.active=dev", "--DATASOURCE_URL=${envVars['DATASOURCE_URL'] ?: ''}", "--DATASOURCE_USERNAME=${envVars['DATASOURCE_USERNAME'] ?: ''}", "--DATASOURCE_PASSWORD=${envVars['DATASOURCE_PASSWORD'] ?: ''}"]) + } } \ No newline at end of file diff --git a/server/application-server/src/main/java/de/tum/cit/aet/helios/config/NatsConfig.java b/server/application-server/src/main/java/de/tum/cit/aet/helios/config/NatsConfig.java new file mode 100644 index 0000000..6ba2a6e --- /dev/null +++ b/server/application-server/src/main/java/de/tum/cit/aet/helios/config/NatsConfig.java @@ -0,0 +1,28 @@ +package de.tum.cit.aet.helios.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +import io.nats.client.Connection; +import io.nats.client.Nats; +import io.nats.client.Options; + +@Configuration +public class NatsConfig { + + @Value("${nats.server}") + private String natsServer; + + @Value("${nats.auth.token}") + private String natsAuthToken; + + @Bean + public Connection natsConnection() throws Exception { + + Options options = Options.builder().server(natsServer).token(natsAuthToken).build(); + return Nats.connect(options); + } +} diff --git a/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandler.java b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandler.java new file mode 100644 index 0000000..3255636 --- /dev/null +++ b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandler.java @@ -0,0 +1,62 @@ +package de.tum.cit.aet.helios.gitprovider.common.github; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; + +import org.kohsuke.github.GHEvent; +import org.kohsuke.github.GHEventPayload; +import org.kohsuke.github.GitHub; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.nats.client.Message; +import io.nats.client.MessageHandler; +import org.springframework.stereotype.Component; + +@Component +public abstract class GitHubMessageHandler implements MessageHandler { + + private static final Logger logger = LoggerFactory.getLogger(GitHubMessageHandler.class); + + private final Class payloadType; + + protected GitHubMessageHandler(Class payloadType) { + this.payloadType = payloadType; + } + + @Override + public void onMessage(Message msg) { + String eventType = getHandlerEvent().name().toLowerCase(); + String subject = msg.getSubject(); + if (!subject.endsWith(eventType)) { + logger.error("Received message on unexpected subject: {}, expected to end with {}", subject, eventType); + return; + } + + String payload = new String(msg.getData(), StandardCharsets.UTF_8); + + try (StringReader reader = new StringReader(payload)) { + T eventPayload = GitHub.offline().parseEventPayload(reader, payloadType); + handleEvent(eventPayload); + } catch (IOException e) { + logger.error("Failed to parse payload for subject {}: {}", subject, e.getMessage(), e); + } catch (Exception e) { + logger.error("Unexpected error while handling message for subject {}: {}", subject, e.getMessage(), e); + } + } + + /** + * Handles the parsed event payload. + * + * @param eventPayload The parsed GHEventPayload. + */ + protected abstract void handleEvent(T eventPayload); + + /** + * Returns the GHEvent that this handler is responsible for. + * + * @return The GHEvent. + */ + protected abstract GHEvent getHandlerEvent(); +} diff --git a/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandlerRegistry.java b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandlerRegistry.java new file mode 100644 index 0000000..026950b --- /dev/null +++ b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/common/github/GitHubMessageHandlerRegistry.java @@ -0,0 +1,29 @@ +package de.tum.cit.aet.helios.gitprovider.common.github; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.kohsuke.github.GHEvent; +import org.springframework.stereotype.Component; + +@Component +public class GitHubMessageHandlerRegistry { + + private final Map> handlerMap = new HashMap<>(); + + public GitHubMessageHandlerRegistry(GitHubMessageHandler[] handlers) { + for (GitHubMessageHandler handler : handlers) { + handlerMap.put(handler.getHandlerEvent(), handler); + } + } + + public GitHubMessageHandler getHandler(GHEvent eventType) { + return handlerMap.get(eventType); + } + + public List getSupportedEvents() { + return new ArrayList<>(handlerMap.keySet()); + } +} \ No newline at end of file diff --git a/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/issue/github/GitHubIssueMessageHandler.java b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/issue/github/GitHubIssueMessageHandler.java new file mode 100644 index 0000000..029948c --- /dev/null +++ b/server/application-server/src/main/java/de/tum/cit/aet/helios/gitprovider/issue/github/GitHubIssueMessageHandler.java @@ -0,0 +1,35 @@ +package de.tum.cit.aet.helios.gitprovider.issue.github; + +import org.kohsuke.github.GHEvent; +import org.kohsuke.github.GHEventPayload; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import de.tum.cit.aet.helios.gitprovider.common.github.GitHubMessageHandler; + +@Component +public class GitHubIssueMessageHandler extends GitHubMessageHandler { + + private static final Logger logger = LoggerFactory.getLogger(GitHubIssueMessageHandler.class); + + private GitHubIssueMessageHandler() { + super(GHEventPayload.Issue.class); + } + + @Override + protected void handleEvent(GHEventPayload.Issue eventPayload) { + var action = eventPayload.getAction(); + var repository = eventPayload.getRepository(); + var issue = eventPayload.getIssue(); + logger.info("Received issue event for repository: {}, issue: {}, action: {}", + repository.getFullName(), + issue.getNumber(), + action); + } + + @Override + protected GHEvent getHandlerEvent() { + return GHEvent.ISSUES; + } +} diff --git a/server/application-server/src/main/java/de/tum/cit/aet/helios/syncing/NatsConsumerService.java b/server/application-server/src/main/java/de/tum/cit/aet/helios/syncing/NatsConsumerService.java new file mode 100644 index 0000000..97e4dcd --- /dev/null +++ b/server/application-server/src/main/java/de/tum/cit/aet/helios/syncing/NatsConsumerService.java @@ -0,0 +1,212 @@ +package de.tum.cit.aet.helios.syncing; + +import java.io.IOException; +import java.util.Arrays; +import java.time.Duration; +import java.time.ZonedDateTime; + +import io.nats.client.Connection; +import io.nats.client.ConsumerContext; +import io.nats.client.JetStreamApiException; +import io.nats.client.Message; +import io.nats.client.MessageHandler; +import io.nats.client.Nats; +import io.nats.client.Options; +import io.nats.client.StreamContext; +import io.nats.client.api.ConsumerConfiguration; +import io.nats.client.api.ConsumerInfo; +import io.nats.client.api.DeliverPolicy; + +import org.springframework.stereotype.Service; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.kohsuke.github.GHEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + +import de.tum.cit.aet.helios.gitprovider.common.github.GitHubMessageHandler; +import de.tum.cit.aet.helios.gitprovider.common.github.GitHubMessageHandlerRegistry; + +@Order(value = 1) +@Service +public class NatsConsumerService { + + private static final Logger logger = LoggerFactory.getLogger(NatsConsumerService.class); + + private static final int INITIAL_RECONNECT_DELAY_SECONDS = 2; + + @Value("${nats.enabled}") + private boolean isNatsEnabled; + + @Value("${nats.timeframe}") + private int timeframe; + + @Value("${nats.server}") + private String natsServer; + + @Value("${nats.durableConsumerName}") + private String durableConsumerName; + + @Value("${monitoring.repositories}") + private String[] repositoriesToMonitor; + + @Value("${nats.auth.token}") + private String natsAuthToken; + + private Connection natsConnection; + private ConsumerContext consumerContext; + + private final GitHubMessageHandlerRegistry handlerRegistry; + + public NatsConsumerService(GitHubMessageHandlerRegistry handlerRegistry) { + this.handlerRegistry = handlerRegistry; + } + + @EventListener(ApplicationReadyEvent.class) + public void init() { + if (!isNatsEnabled) { + logger.info("NATS is disabled. Skipping initialization."); + return; + } + + validateConfigurations(); + Options options = buildNatsOptions(); + + while (true) { + try { + natsConnection = Nats.connect(options); + setupConsumer(natsConnection); + return; + } catch (IOException | InterruptedException e) { + logger.error("NATS connection error: {}", e.getMessage(), e); + } + } + } + + private void validateConfigurations() { + if (natsServer == null || natsServer.trim().isEmpty()) { + throw new IllegalArgumentException("NATS server configuration is missing."); + } + if (repositoriesToMonitor == null || repositoriesToMonitor.length == 0) { + throw new IllegalArgumentException("No repositories to monitor are configured."); + } + } + + private Options buildNatsOptions() { + return Options.builder() + .server(natsServer) + .token(natsAuthToken) + .connectionListener((conn, type) -> logger.info("Connection event - Server: {}, {}", + conn.getServerInfo().getPort(), type)) + .maxReconnects(-1) + .reconnectWait(Duration.ofSeconds(INITIAL_RECONNECT_DELAY_SECONDS)) + .build(); + } + + private void setupConsumer(Connection connection) throws IOException, InterruptedException { + try { + StreamContext streamContext = connection.getStreamContext("github"); + + // Check if consumer already exists + if (durableConsumerName != null && !durableConsumerName.isEmpty()) { + try { + consumerContext = streamContext.getConsumerContext(durableConsumerName); + } catch (JetStreamApiException e) { + consumerContext = null; + } + } + + if (consumerContext == null) { + logger.info("Setting up consumer for subjects: {}", Arrays.toString(getSubjects())); + ConsumerConfiguration.Builder consumerConfigBuilder = ConsumerConfiguration.builder() + .filterSubjects(getSubjects()) + .deliverPolicy(DeliverPolicy.ByStartTime) + .startTime(ZonedDateTime.now().minusDays(timeframe)); + + if (durableConsumerName != null && !durableConsumerName.isEmpty()) { + consumerConfigBuilder.durable(durableConsumerName); + } + + ConsumerConfiguration consumerConfig = consumerConfigBuilder.build(); + consumerContext = streamContext.createOrUpdateConsumer(consumerConfig); + } else { + logger.info("Consumer already exists. Skipping consumer setup."); + } + + MessageHandler handler = this::handleMessage; + consumerContext.consume(handler); + logger.info("Successfully started consuming messages."); + } catch (JetStreamApiException e) { + logger.error("JetStream API exception: {}", e.getMessage(), e); + throw new IOException("Failed to set up consumer.", e); + } + } + + private void handleMessage(Message msg) { + try { + String subject = msg.getSubject(); + String lastPart = subject.substring(subject.lastIndexOf(".") + 1); + GHEvent eventType = GHEvent.valueOf(lastPart.toUpperCase()); + GitHubMessageHandler eventHandler = handlerRegistry.getHandler(eventType); + + if (eventHandler == null) { + logger.warn("No handler found for event type: {}", eventType); + msg.ack(); + return; + } + + eventHandler.onMessage(msg); + msg.ack(); + } catch (IllegalArgumentException e) { + logger.error("Invalid event type in subject '{}': {}", msg.getSubject(), e.getMessage()); + } catch (Exception e) { + logger.error("Error processing message: {}", e.getMessage(), e); + } finally { + msg.ack(); + } + } + + /** + * Subjects to monitor. + * + * @return The subjects to monitor. + */ + private String[] getSubjects() { + String[] events = handlerRegistry.getSupportedEvents().stream() + .map(GHEvent::name) + .map(String::toLowerCase) + .toArray(String[]::new); + + return Arrays.stream(repositoriesToMonitor) + .map(this::getSubjectPrefix) + .flatMap(prefix -> Arrays.stream(events).map(event -> prefix + "." + event)) + .toArray(String[]::new); + } + + /** + * Get subject prefix from ownerWithName for the given repository. + * + * @param ownerWithName The owner and name of the repository. + * @return The subject prefix, i.e. "github.owner.name" sanitized. + * @throws IllegalArgumentException if the repository string is improperly + * formatted. + */ + private String getSubjectPrefix(String ownerWithName) { + if (ownerWithName == null || ownerWithName.trim().isEmpty()) { + throw new IllegalArgumentException("Repository identifier cannot be null or empty."); + } + + String sanitized = ownerWithName.replace(".", "~"); + String[] parts = sanitized.split("/"); + + if (parts.length != 2) { + throw new IllegalArgumentException( + String.format("Invalid repository format: '%s'. Expected format 'owner/repository'.", + ownerWithName)); + } + + return "github." + parts[0] + "." + parts[1]; + } +} \ No newline at end of file diff --git a/server/application-server/src/main/resources/application.yml b/server/application-server/src/main/resources/application.yml new file mode 100644 index 0000000..3f1f338 --- /dev/null +++ b/server/application-server/src/main/resources/application.yml @@ -0,0 +1,21 @@ +spring: + application: + name: Helios + +springdoc: + default-produces-media-type: application/json + +nats: + enabled: true + timeframe: ${MONITORING_TIMEFRAME:7} + durableConsumerName: "" + server: ${NATS_SERVER} + auth: + token: ${NATS_AUTH_TOKEN} + +monitoring: + repositories: ${REPOSITORY_NAME} + +logging: + level: + org.kohsuke.github.GitHubClient: DEBUG \ No newline at end of file diff --git a/server/application-server/src/test/java/de/tum/cit/aet/helios/HeliosApplicationTests.java b/server/application-server/src/test/java/de/tum/cit/aet/helios/HeliosApplicationTests.java index ba06a71..7eba40b 100644 --- a/server/application-server/src/test/java/de/tum/cit/aet/helios/HeliosApplicationTests.java +++ b/server/application-server/src/test/java/de/tum/cit/aet/helios/HeliosApplicationTests.java @@ -6,8 +6,8 @@ @SpringBootTest class HeliosApplicationTests { - @Test - void contextLoads() { - } +// @Test +// void contextLoads() { +// } } diff --git a/server/webhook-listener/README.md b/server/webhook-listener/README.md index d26d01a..cf805a5 100644 --- a/server/webhook-listener/README.md +++ b/server/webhook-listener/README.md @@ -22,12 +22,13 @@ If you are using docker compose, you don't need to set NATS_URL for local develo Generate an AUTH TOKEN and Set the environment variable: ```bash -export NATS_AUTH_TOKEN=$(openssl rand -hex 48) +openssl rand -hex 48 # Generate a random token, save this token to use it in application-server +export NATS_AUTH_TOKEN= ``` -Set Webhook secret +Add Webhook in GitHub repository and set the secret: ```bash -export WEBHOOK_SECRET= +export WEBHOOK_SECRET= ``` ## Running with Docker Compose From 821c3caaccc866316014e8da1aa92b932e42b805 Mon Sep 17 00:00:00 2001 From: Turker Koc Date: Tue, 12 Nov 2024 00:45:53 +0100 Subject: [PATCH 3/4] gitignore update --- .gitignore | 4 +++- server/application-server/.gitignore | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b79fe8f..f39ecdb 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,6 @@ install_manifest.txt compile_commands.json CTestTestfile.cmake _deps -CMakeUserPresets.json \ No newline at end of file +CMakeUserPresets.json + +.DS_STORE \ No newline at end of file diff --git a/server/application-server/.gitignore b/server/application-server/.gitignore index 4b4d8f7..8a03eaa 100644 --- a/server/application-server/.gitignore +++ b/server/application-server/.gitignore @@ -37,3 +37,5 @@ out/ ### VS Code ### .vscode/ .sdkmanrc + +.DS_STORE From 78324aacb83bd01e4669b01d14107e5c26b08fbe Mon Sep 17 00:00:00 2001 From: Turker Koc Date: Tue, 12 Nov 2024 00:59:38 +0100 Subject: [PATCH 4/4] .env updates --- server/application-server/.env.example | 5 +++-- server/webhook-listener/.env.example | 2 ++ server/webhook-listener/README.md | 10 +++++++++- server/webhook-listener/compose.yaml | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 server/webhook-listener/.env.example diff --git a/server/application-server/.env.example b/server/application-server/.env.example index b3196b8..e235638 100644 --- a/server/application-server/.env.example +++ b/server/application-server/.env.example @@ -2,5 +2,6 @@ DATASOURCE_URL=jdbc:postgresql://127.0.0.1/helios DATASOURCE_USERNAME=helios DATASOURCE_PASSWORD=helios NATS_SERVER=localhost:4222 -REPOSITORY_NAME=# e.g. ls1intum/Helios -NATS_AUTH_TOKEN= +NATS_AUTH_TOKEN='5760e8ae09adfb2756f9f8cd5cb2caa704cd3f549eaa9298be843ceb165185d815b81f90c680fa7f626b7cd63abf6ac9' +REPOSITORY_NAME= + diff --git a/server/webhook-listener/.env.example b/server/webhook-listener/.env.example new file mode 100644 index 0000000..667db49 --- /dev/null +++ b/server/webhook-listener/.env.example @@ -0,0 +1,2 @@ +NATS_AUTH_TOKEN='5760e8ae09adfb2756f9f8cd5cb2caa704cd3f549eaa9298be843ceb165185d815b81f90c680fa7f626b7cd63abf6ac9' +WEBHOOK_SECRET= diff --git a/server/webhook-listener/README.md b/server/webhook-listener/README.md index cf805a5..e67573a 100644 --- a/server/webhook-listener/README.md +++ b/server/webhook-listener/README.md @@ -31,12 +31,20 @@ Add Webhook in GitHub repository and set the secret: export WEBHOOK_SECRET= ``` +## Setup configuration and environment + +Copy the file `.env.example` to `.env` and adjust the values to your needs. It is set up to work with the Docker Compose setup for database. You need to adjust some fields for NATS server. + +```bash +cp .env.example .env +``` + ## Running with Docker Compose Build and run with Docker Compose: ```bash -docker-compose up --build +docker-compose --env-file ./.env up --build ``` Service ports: diff --git a/server/webhook-listener/compose.yaml b/server/webhook-listener/compose.yaml index ae87a0a..b1aa291 100644 --- a/server/webhook-listener/compose.yaml +++ b/server/webhook-listener/compose.yaml @@ -14,7 +14,7 @@ services: - common-network healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4200/health"] - interval: 5s + interval: 30s timeout: 10s retries: 5 start_period: 3s