Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(engine): Move registry management endpoints into api service + restructure registry as executor service #590

Merged
merged 9 commits into from
Dec 7, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ PUBLIC_APP_URL=http://localhost
PUBLIC_API_URL=http://localhost/api
SAML_SP_ACS_URL=${PUBLIC_API_URL}/auth/saml/acs
INTERNAL_API_URL=http://api:8000
INTERNAL_REGISTRY_URL=http://registry:8000
INTERNAL_EXECUTOR_URL=http://executor:8000
# -- Caddy env vars ---
BASE_DOMAIN=:80
# Note: replace with your server's IP address
7 changes: 3 additions & 4 deletions .github/workflows/test-python.yml
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ on:
- pyproject.toml
- .github/workflows/test-python.yml
pull_request:
branches: ["main"]
branches: ["main", "staging"]
paths:
- tracecat/**
- registry/**
@@ -21,8 +21,7 @@ on:
inputs:
git-ref:
description: "Git Ref (Optional)"
required: false
default: "main"
required: true

permissions:
contents: read
@@ -126,7 +125,7 @@ jobs:
- name: Start Docker services
env:
TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true"
run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker registry postgres_db caddy
run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker executor postgres_db caddy

- name: Install dependencies
run: |
4 changes: 2 additions & 2 deletions Caddyfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{$BASE_DOMAIN} {
bind {$ADDRESS} # Binds to all available network interfaces if not specified
handle_path /api/registry* {
reverse_proxy http://registry:8000
handle_path /api/executor* {
reverse_proxy http://executor:8000
}
handle_path /api* {
reverse_proxy http://api:8000
14 changes: 7 additions & 7 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ services:
TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES}
TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS}
TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH}
TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}
OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}
USER_AUTH_SECRET: ${USER_AUTH_SECRET}
@@ -63,6 +63,7 @@ services:
- ./alembic:/app/alembic
depends_on:
- ollama
- executor

worker:
build:
@@ -78,7 +79,7 @@ services:
TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE}
TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive
TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL}
TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive
TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive
# Temporal
@@ -87,9 +88,9 @@ services:
volumes:
- ./tracecat:/app/tracecat
- ./registry:/app/registry
entrypoint: ["python", "tracecat/dsl/worker.py"]
command: ["python", "tracecat/dsl/worker.py"]

registry:
executor:
build:
context: .
dockerfile: Dockerfile.dev
@@ -112,13 +113,12 @@ services:
OLLAMA__API_URL: ${OLLAMA__API_URL}
volumes:
- ./tracecat:/app/tracecat
- ./registry:/app/registry
entrypoint:
command:
[
"python",
"-m",
"uvicorn",
"tracecat.api.registry:app",
"tracecat.api.executor:app",
"--host",
"0.0.0.0",
"--port",
8 changes: 4 additions & 4 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ services:
TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES}
TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS}
TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH}
TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}
OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}
USER_AUTH_SECRET: ${USER_AUTH_SECRET}
@@ -75,15 +75,15 @@ services:
TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE}
TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive
TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL}
TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive
TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive
# Temporal
TEMPORAL__CLUSTER_URL: ${TEMPORAL__CLUSTER_URL}
TEMPORAL__CLUSTER_QUEUE: ${TEMPORAL__CLUSTER_QUEUE}
command: ["python", "tracecat/dsl/worker.py"]

registry:
executor:
image: ghcr.io/tracecathq/tracecat:${TRACECAT__IMAGE_TAG:-0.16.0}
restart: unless-stopped
networks:
@@ -108,7 +108,7 @@ services:
"python",
"-m",
"uvicorn",
"tracecat.api.registry:app",
"tracecat.api.executor:app",
"--host",
"0.0.0.0",
"--port",
154 changes: 13 additions & 141 deletions frontend/src/client/schemas.gen.ts
Original file line number Diff line number Diff line change
@@ -173,110 +173,7 @@ export const $ActionRetryPolicy = {
title: 'ActionRetryPolicy'
} as const;

export const $ActionStatement_Input = {
properties: {
id: {
anyOf: [
{
type: 'string'
},
{
type: 'null'
}
],
title: 'Id',
description: 'The action ID. If this is populated means there is a corresponding actionin the database `Action` table.'
},
ref: {
type: 'string',
pattern: '^[a-z0-9_]+$',
title: 'Ref',
description: 'Unique reference for the task'
},
description: {
type: 'string',
title: 'Description',
default: ''
},
action: {
type: 'string',
pattern: '^[a-z0-9_.]+$',
title: 'Action',
description: 'Action type. Equivalent to the UDF key.'
},
args: {
type: 'object',
title: 'Args',
description: 'Arguments for the action'
},
depends_on: {
items: {
type: 'string'
},
type: 'array',
title: 'Depends On',
description: 'Task dependencies'
},
run_if: {
anyOf: [
{
type: 'string'
},
{
type: 'null'
}
],
title: 'Run If',
description: 'Condition to run the task'
},
for_each: {
anyOf: [
{
type: 'string'
},
{
items: {
type: 'string'
},
type: 'array'
},
{
type: 'null'
}
],
title: 'For Each',
description: 'Iterate over a list of items and run the task for each item.'
},
retry_policy: {
allOf: [
{
'$ref': '#/components/schemas/ActionRetryPolicy'
}
],
description: 'Retry policy for the action.'
},
start_delay: {
type: 'number',
title: 'Start Delay',
description: 'Delay before starting the action in seconds.',
default: 0
},
join_strategy: {
allOf: [
{
'$ref': '#/components/schemas/JoinStrategy'
}
],
description: 'The strategy to use when joining on this task. By default, all branches must complete successfully before the join task can complete.',
default: 'all'
}
},
type: 'object',
required: ['ref', 'action'],
title: 'ActionStatement'
} as const;

export const $ActionStatement_Output = {
export const $ActionStatement = {
properties: {
ref: {
type: 'string',
@@ -758,10 +655,15 @@ export const $DSLContext = {
},
ENV: {
'$ref': '#/components/schemas/DSLEnvironment'
},
SECRETS: {
type: 'object',
title: 'Secrets'
}
},
type: 'object',
title: 'DSLContext'
title: 'DSLContext',
description: 'DSL Context. Contains all the context needed to execute a DSL workflow.'
} as const;

export const $DSLEntrypoint = {
@@ -837,7 +739,7 @@ export const $DSLInput = {
},
actions: {
items: {
'$ref': '#/components/schemas/ActionStatement-Output'
'$ref': '#/components/schemas/ActionStatement'
},
type: 'array',
title: 'Actions'
@@ -1118,7 +1020,7 @@ export const $EventGroup = {
action_input: {
anyOf: [
{
'$ref': '#/components/schemas/RunActionInput-Output'
'$ref': '#/components/schemas/RunActionInput'
},
{
'$ref': '#/components/schemas/DSLRunArgs'
@@ -1333,7 +1235,7 @@ export const $GetWorkflowDefinitionActivityInputs = {
task: {
anyOf: [
{
'$ref': '#/components/schemas/ActionStatement-Output'
'$ref': '#/components/schemas/ActionStatement'
},
{
type: 'null'
@@ -1868,18 +1770,6 @@ export const $RegistryActionUpdate = {
description: 'API update model for a registered action.'
} as const;

export const $RegistryActionValidate = {
properties: {
args: {
type: 'object',
title: 'Args'
}
},
type: 'object',
required: ['args'],
title: 'RegistryActionValidate'
} as const;

export const $RegistryActionValidateResponse = {
properties: {
ok: {
@@ -2086,7 +1976,7 @@ export const $Role = {
},
service_id: {
type: 'string',
enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service'],
enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service', 'tracecat-executor'],
title: 'Service Id'
}
},
@@ -2119,28 +2009,10 @@ Service roles
- A service's \`user_id\` is the user it's acting on behalf of. This can be None for internal services.`
} as const;

export const $RunActionInput_Input = {
properties: {
task: {
'$ref': '#/components/schemas/ActionStatement-Input'
},
exec_context: {
'$ref': '#/components/schemas/DSLContext'
},
run_context: {
'$ref': '#/components/schemas/RunContext'
}
},
type: 'object',
required: ['task', 'exec_context', 'run_context'],
title: 'RunActionInput',
description: 'This object contains all the information needed to execute an action.'
} as const;

export const $RunActionInput_Output = {
export const $RunActionInput = {
properties: {
task: {
'$ref': '#/components/schemas/ActionStatement-Output'
'$ref': '#/components/schemas/ActionStatement'
},
exec_context: {
'$ref': '#/components/schemas/DSLContext'
Loading