Merge pull request #28 from theredguild/docker-compose

[on-hold] use docker compose for our devcontainer

.devcontainer/devcontainer.json

@@ -1,9 +1,9 @@
 {
     "name": "OSS Security Tools",
-
-    "build": {
-        "dockerfile": "../Dockerfile"
-    },
+    "dockerComposeFile": "../compose.yml",
+    "service": "toolbox",
+    "workspaceFolder": "/workspace",
+    "shutdownAction": "stopCompose",
 
     "features": {
     },
@@ -23,4 +23,4 @@
     },
 
     "remoteUser": "wanderer"
-}
+}

Dockerfile

@@ -219,9 +219,9 @@ RUN wget -qO - https://github.com/checkmarx/2ms/releases/latest/download/linux-$
     funzip - | sudo tee /usr/local/bin/2ms > /dev/null \
     && sudo chmod +x /usr/local/bin/2ms
 
-# # Install clair
-# RUN sudo wget -qO /usr/local/bin/clair https://github.com/quay/clair/releases/download/v4.7.4/clairctl-linux-$(dpkg --print-architecture) \
-#     && sudo chmod +x /usr/local/bin/clair
+# Install clair
+RUN sudo wget -qO /usr/local/bin/clair https://github.com/quay/clair/releases/download/v4.8.0/clairctl-linux-$(dpkg --print-architecture) \
+    && sudo chmod +x /usr/local/bin/clair
 
 # Install Grype
 RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin

clair-compose.yml

@@ -0,0 +1,75 @@
+---
+version: "3.7"
+# This is just to hold a bunch of yaml anchors and try to consolidate parts of
+# the config.
+x-anchors:
+  postgres: &postgres-image docker.io/library/postgres:12
+  traefik: &traefik-image docker.io/library/traefik:v2.2
+  clair: &clair-image quay.io/projectquay/clair:4.8.0
+
+services:
+  clair-indexer:
+    image: *clair-image
+    container_name: clair-indexer
+    depends_on:
+      clair-database:
+        condition: service_healthy
+    environment:
+      CLAIR_MODE: "indexer"
+    volumes:
+      - "./clair_config/local-dev/clair:/config:ro"
+    networks:
+      - toolbox-net
+
+  clair-matcher:
+    image: *clair-image
+    container_name: clair-matcher
+    depends_on:
+      clair-database:
+        condition: service_healthy
+    environment:
+      CLAIR_MODE: "matcher"
+    volumes:
+      - "./clair_config/local-dev/clair:/config:ro"
+    networks:
+      - toolbox-net
+
+  clair-database:
+    container_name: clair-database
+    image: *postgres-image
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+    volumes:
+      - type: bind
+        source: ./clair_config/local-dev/clair/init.sql
+        target: /docker-entrypoint-initdb.d/init.sql
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - "pg_isready -U postgres"
+      interval: 5s
+      timeout: 4s
+      retries: 12
+      start_period: 10s
+    networks:
+      - toolbox-net
+
+  clair-traefik:
+    container_name: clair-traefik
+    image: *traefik-image
+    depends_on:
+      - clair-matcher
+      - clair-indexer
+    ports:
+      - '6060:6060'
+      - '8080:8080'
+      - '8443'
+      - '5432'
+    volumes:
+      - './clair_config/local-dev/traefik/:/etc/traefik/:ro'
+    networks:
+      - toolbox-net
+
+networks:
+  toolbox-net:
+    external: true

clair_config/local-dev/clair/.gitignore

@@ -0,0 +1 @@
+quay.yaml

clair_config/local-dev/clair/config.yaml

@@ -0,0 +1,58 @@
+---
+log_level: debug-color
+introspection_addr: ":8089"
+http_listen_addr: ":6060"
+updaters:
+  sets:
+    - ubuntu
+    - debian
+    - rhel-vex
+    - alpine
+    - osv
+auth:
+  psk:
+    key: 'c2VjcmV0'
+    iss:
+      - quay
+      - clairctl
+indexer:
+  connstring: host=clair-database user=clair dbname=indexer sslmode=disable
+  scanlock_retry: 10
+  layer_scan_concurrency: 5
+  migrations: true
+matcher:
+  indexer_addr: http://clair-indexer:6060/
+  connstring: host=clair-database user=clair dbname=matcher sslmode=disable
+  max_conn_pool: 100
+  migrations: true
+matchers: {}
+notifier:
+  indexer_addr: http://clair-indexer:6060/
+  matcher_addr: http://clair-matcher:6060/
+  connstring: host=clair-database user=clair dbname=notifier sslmode=disable
+  migrations: true
+  delivery_interval: 30s
+  poll_interval: 1m
+  webhook:
+    target: "http://webhook-target/"
+    callback: "http://clair-notifier:6060/notifier/api/v1/notification/"
+  # amqp:
+  #   direct: true
+  #   exchange:
+  #     name: ""
+  #     type: "direct"
+  #     durable: true
+  #     auto_delete: false
+  #   uris: ["amqp://guest:guest@clair-rabbitmq:5672/"]
+  #   routing_key: "notifications"
+  #   callback: "http://clair-notifier/notifier/api/v1/notification"
+# tracing and metrics config
+trace:
+  name: "jaeger"
+#  probability: 1
+  jaeger:
+    agent:
+      endpoint: "clair-jaeger:6831"
+    service_name: "clair"
+metrics:
+  name: "prometheus"

clair_config/local-dev/clair/config.yaml.d/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

clair_config/local-dev/clair/init.sql

@@ -0,0 +1,12 @@
+CREATE USER clair WITH PASSWORD 'clair';
+CREATE USER quay WITH PASSWORD 'quay';
+CREATE DATABASE indexer WITH OWNER clair;
+CREATE DATABASE matcher WITH OWNER clair;
+CREATE DATABASE notifier WITH OWNER clair;
+CREATE DATABASE quay WITH OWNER quay;
+\connect matcher
+CREATE EXTENSION "uuid-ossp";
+\connect notifier
+CREATE EXTENSION "uuid-ossp";
+\connect quay
+CREATE EXTENSION "pg_trgm";

clair_config/local-dev/clair/quay.yaml.d/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

clair_config/local-dev/traefik/config/clair.yaml

@@ -0,0 +1,40 @@
+---
+http:
+  entrypoint:
+    clair:
+      address: ':6060'
+  routers:
+    indexer:
+      entryPoints: [clair]
+      rule: 'PathPrefix(`/indexer`)'
+      service: indexer
+    matcher:
+      entryPoints: [clair]
+      rule: 'PathPrefix(`/matcher`)'
+      service: matcher
+    notifier:
+      entryPoints: [clair]
+      rule: 'PathPrefix(`/notifier`)'
+      service: notifier
+  services:
+    indexer:
+      loadBalancer:
+        servers:
+        - url: "http://clair-indexer:6060/"
+        healthCheck:
+          path: /healthz
+          port: 8089
+    matcher:
+      loadBalancer:
+        servers:
+        - url: "http://clair-matcher:6060/"
+        healthCheck:
+          path: /healthz
+          port: 8089
+    notifier:
+      loadBalancer:
+        servers:
+        - url: "http://clair-notifier:6060/"
+        healthCheck:
+          path: /healthz
+          port: 8089

clair_config/local-dev/traefik/config/dashboard.yaml

@@ -0,0 +1,17 @@
+---
+http:
+  routers:
+    api:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/api`) || PathPrefix(`/dashboard`)'
+      service: 'api@internal'
+    dashboard-redirect:
+      entryPoints: [traefik]
+      rule: 'Path(`/`)'
+      middlewares: [dashboard-redirect]
+      service: 'api@internal'
+  middlewares:
+    dashboard-redirect:
+      redirectRegex:
+        regex: '.*'
+        replacement: '${1}/dashboard/'

clair_config/local-dev/traefik/config/grafana.yaml

@@ -0,0 +1,14 @@
+---
+http:
+  routers:
+    grafana:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/grafana`)'
+      service: grafana
+  services:
+    grafana:
+      loadBalancer:
+        servers:
+        - url: "http://clair-grafana:3000/"
+        healthCheck:
+          path: /grafana/api/health

clair_config/local-dev/traefik/config/jaeger.yaml

@@ -0,0 +1,14 @@
+---
+http:
+  routers:
+    jaeger:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/jaeger`)'
+      service: jaeger
+  services:
+    jaeger:
+      loadBalancer:
+        servers:
+        - url: "http://clair-jaeger:16686/"
+        healthCheck:
+          path: /

clair_config/local-dev/traefik/config/pgadmin.yaml

@@ -0,0 +1,14 @@
+---
+http:
+  routers:
+    pgadmin:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/pgadmin`)'
+      service: pgadmin
+  services:
+    pgadmin:
+      loadBalancer:
+        servers:
+        - url: "http://clair-pgadmin/"
+        healthCheck:
+          path: /pgadmin

clair_config/local-dev/traefik/config/postgresql.yaml

@@ -0,0 +1,13 @@
+---
+tcp:
+  routers:
+    postgresql:
+      entryPoints: [postgresql]
+      service: postgresql
+      # Traefik docs say this hack is needed if not using TLS.
+      rule: 'HostSNI(`*`)'
+  services:
+    postgresql:
+      loadBalancer:
+        servers:
+          - address: 'clair-database:5432'

clair_config/local-dev/traefik/config/prom.yaml

@@ -0,0 +1,14 @@
+---
+http:
+  routers:
+    prom:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/prom`)'
+      service: prom
+  services:
+    prom:
+      loadBalancer:
+        servers:
+        - url: "http://clair-prometheus:9090/"
+        healthCheck:
+          path: /prom/-/healthy

clair_config/local-dev/traefik/config/pyroscope.yaml

@@ -0,0 +1,21 @@
+---
+http:
+  routers:
+    pyroscope:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/pyroscope`)'
+      service: pyroscope
+      middlewares:
+      - pyroscope-stripprefix
+  middlewares:
+    pyroscope-stripprefix:
+      stripPrefix:
+        prefixes:
+        - /pyroscope
+  services:
+    pyroscope:
+      loadBalancer:
+        servers:
+        - url: "http://clair-pyroscope:4040/"
+        healthCheck:
+          path: /

clair_config/local-dev/traefik/config/quay.yaml

@@ -0,0 +1,20 @@
+---
+http:
+  routers:
+    quay:
+      entryPoints: [quay]
+      rule: 'PathPrefix(`/`)'
+      service: quay
+    quay-api:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/v2`)'
+      service: quay
+  services:
+    quay:
+      loadBalancer:
+        passHostHeader: false
+        servers:
+        - url: "http://clair-quay:8080/"
+        healthCheck:
+          path: /health
+          port: 8080

clair_config/local-dev/traefik/config/rabbitmq.yaml

@@ -0,0 +1,26 @@
+---
+http:
+  routers:
+    rabbitmq:
+      entryPoints: [traefik]
+      rule: 'PathPrefix(`/rabbitmq`)'
+      middlewares:
+      - rewrite-api
+      - rewrite
+      service: rabbitmq
+  services:
+    rabbitmq:
+      loadBalancer:
+        servers:
+        - url: "http://clair-rabbitmq:15672/"
+        healthCheck:
+          path: /
+  middlewares:
+    rewrite-api:
+      replacePathRegex:
+        regex: '^/rabbitmq/api/(.*?)/(.*)'
+        replacement: '/api/%2F/$2'
+    rewrite:
+      replacePathRegex:
+        regex: '^/rabbitmq/(.*)$'
+        replacement: '/$1'