RHINENG-8201 - docker-compose based dev-setup (#143)

* fix: RHINENG-8201 docker-compose based dev-setup

* fix: url api version and doc

* fix: rename command

.gitignore

@@ -1 +1,4 @@
 **.env
+.DS_Store
+bin
+tmp

Makefile

@@ -0,0 +1,57 @@
+mqtt_msg='{ "type": "connection-status", "message_id": "3a57b1ad-5163-47ee-9e57-3bb6d90bdfff", "version": 1, "sent": "2023-12-04T17:22:24+00:00", "content": { "canonical_facts": { "insights_id": "a319580a-2321-47c7-b061-886548bea067", "machine_id": "044afe91-3b12-4439-9e8f-f531462d26b9", "bios_uuid": "ec2c79ab-02ff-d09b-31a9-583a902dd74b", "subscription_manager_id": "4b1efbbe-a447-48d0-98c3-3594aae1d2c5", "ip_addresses": ["172.31.28.69"], "mac_addresses": ["0a:94:1d:36:ff:21", "00:00:00:00:00:00"], "fqdn": "ip-172-31-28-69.ec2.internal" }, "dispatchers": { "playbook": { "ansible-runner-version": "1.2.3" }, "package-manager": null, "rhc-worker-playbook":null }, "state": "online" } }'
+send_mqtt_msg:
+	mqtt pub -V 3 -t redhat/insights/4b1efbbe-a447-48d0-98c3-3594aae1d2c5/control/out -h 127.0.0.1 -p 8883 -d -v -m ${mqtt_msg}
+
+
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+ifeq (,$(wildcard $(LOCALBIN)))
+	@echo "🤖 Ensuring $(LOCALBIN) is available"
+	mkdir -p $(LOCALBIN)
+	@echo "✅ Done"
+endif
+
+.PHONY: golangci-lint
+GOLANGCILINT := $(LOCALBIN)/golangci-lint
+GOLANGCI_URL := https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh
+golangci-lint: $(LOCALBIN)
+ifeq (,$(wildcard $(GOLANGCILINT)))
+	@ echo "📥 Downloading golangci-lint"
+	curl -sSfL $(GOLANGCI_URL) | sh -s -- -b $(LOCALBIN) $(GOLANGCI_VERSION)
+	@ echo "✅ Done"
+endif
+
+.PHONY: lint
+lint: golangci-lint
+	$(GOLANGCILINT) run --timeout=3m ./...
+
+.PHONY: test
+test:
+	go test -v ./...
+
+start-httpapi:
+	go run main.go \
+		--log-level trace \
+		--metrics-port 9007 \
+		--inventory-host http://127.0.0.1:8001 \
+		http-api
+
+start-inventory-consumer:
+	go run main.go \
+		--log-level=trace \
+		--kafka-brokers=localhost:29092 \
+		--cloud-connector-host=http://127.0.0.1:8084/api/cloud-connector/ \
+		--cloud-connector-client-id=suraj \
+		--cloud-connector-psk=surajskey \
+		--dispatcher-psk=surajskey \
+		--playbook-host=https://random-host \
+		--metrics-port=9008 \
+		--dispatcher-host=http://127.0.0.1:8002 \
+		inventory-consumer
+
+
+configure-xjoin:
+	@./scripts/xjoin-config/configure-xjoin.sh
+
+get_host_from_inventory: 
+	curl http://127.0.0.1:8001/api/inventory/v1/hosts -H "x-rh-identity: eyJpZGVudGl0eSI6IHsiYWNjb3VudF9udW1iZXIiOiAiMTExMDAwIiwgIm9yZ19pZCI6ICIwMDAyIiwgImF1dGhfdHlwZSI6ICJqd3QtYXV0aCIsICJ0eXBlIjogIlVzZXIiLCJ1c2VyIjogeyJ1c2VybmFtZSI6ICJ0dXNlckByZWRoYXQuY29tIiwiZW1haWwiOiAidHVzZXJAcmVkaGF0LmNvbSIsImZpcnN0X25hbWUiOiAidGVzdCIsImxhc3RfbmFtZSI6ICJ1c2VyIiwiaXNfYWN0aXZlIjogdHJ1ZSwiaXNfb3JnX2FkbWluIjogZmFsc2UsICJsb2NhbGUiOiAiZW5fVVMifX19Cg==" | jq

docs/dev-setup.md

@@ -0,0 +1,70 @@
+# Development environment
+
+This is docker-compose based dev-setup which helps to deploy dependent services like cloud-connector, playbook-dispatcher and host-inventory locally. 
+
+## Installation
+
+Before starting the development make sure below packages/binaries are installed on the system. 
+
+| Package/binary                 | Version  | Documentation                                                                  |
+|--------------------------------|----------|--------------------------------------------------------------------------------|
+| golang                         | >=1.17   | https://go.dev/doc/install                                                     |
+| mqtt cli                       | >=4.24.0 | https://hivemq.github.io/mqtt-cli/docs/installation/                           |
+| docker & docker-compose        | latest   | https://docs.docker.com/desktop/install/fedora/                                |
+| librdkafka &  librdkafka-devel | latest   | Use package-manager to install it.  dnf install -y librdkafka librdkafka-devel |
+
+
+## Clone the repository
+```bash
+git clone [email protected]:RedHatInsights/config-manager.git
+```
+
+Update /etc/hosts with below. 
+
+```bash
+127.0.0.1       kafka
+127.0.0.1       minio
+```
+
+## Usage
+
+### Run dependent services
+Use below command to start kafka, cloud-connector, playbook-dispatcher and other dependent services. 
+
+```bash
+cd scripts
+docker-compose up
+```
+Note - If you are unable to pull image from quay.io then try to login quay.io using docker login quay.io and run docker-compose up again.
+
+After running `docker-compose up` run the below command give a minute for all the container to come up and then run below command and make sure there are **>=20** containers running. 
+```bash
+docker ps | wc -l 
+```
+
+Now we are ready to start config-manager locally. 
+
+### Running config-manager locally. 
+
+Use the below make command to start inventory-consumer-service. 
+
+```bash
+make start-inventory-consumer
+```
+
+### Sending data to local config-manager. 
+```bash
+make send_mqtt_msg
+```
+Above command sends host-registration request to host-inventory and also send the connection status to cloud-connector service. 
+
+On the terminal where you ran `make start-inventory-consumer` you will see logs populating. At this point config-manager tries to setup the host by installing `rhc-worker-playbook` package and applies current org profile to the host.
+
+### Running config-manager API.
+
+Use the below make command to start api server.
+
+```
+make configure-xjoin
+make start-httpapi
+```

scripts/client_id_to_account_id_map.json

@@ -0,0 +1,10 @@
+{
+    "8a823918-f801-474c-9f79-7cc7e9d79a7f": {
+        "accountId": "010101",
+        "orgId": "10001"
+    },
+    "c111000lient-1": {
+        "accountId": "111000",
+        "orgId": "0002"
+    }
+}

scripts/connector-local.json

@@ -0,0 +1,38 @@
+{
+    "name": "playbook-dispatcher-source",
+    "config": {
+        "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+        "tasks.max": "1",
+        "database.hostname": "db",
+        "database.port": "5432",
+        "database.user": "insights",
+        "database.password": "insights",
+        "database.dbname": "insights",
+        "slot.name": "debezium",
+        "plugin.name": "pgoutput",
+        "slot.max.retries": 999999999,
+        "database.server.name": "playbook-dispatcher",
+        "table.include.list": "public.runs,public.run_hosts",
+        "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+        "value.converter": "org.apache.kafka.connect.storage.StringConverter",
+        "topic.creation.enable": true,
+        "topic.creation.default.replication.factor": 1,
+        "topic.creation.default.partitions": 1,
+        "topic.creation.default.cleanup.policy": "compact",
+        "transforms": "transformRunEvent,transformRunHostEvent",
+        "transforms.transformRunEvent.type": "com.redhat.cloud.platform.playbook_dispatcher.RunEventTransform",
+        "transforms.transformRunHostEvent.type": "com.redhat.cloud.platform.playbook_dispatcher.RunHostEventTransform",
+        "transforms.transformRunEvent.table": "runs",
+        "transforms.transformRunHostEvent.table": "run_hosts",
+        "transforms.transformRunHostEvent.topic": "platform.playbook-dispatcher.run-hosts",
+        "transforms.transformRunEvent.topic": "platform.playbook-dispatcher.runs",
+        "errors.tolerance": "all",
+        "errors.retry.delay.max.ms": 30000,
+        "errors.retry.timeout": -1,
+        "errors.log.enable": true,
+        "errors.log.include.messages": true,
+        "heartbeat.interval.ms": 60000,
+        "heartbeat.topics.prefix": "__debezium-heartbeat-pd",
+        "heartbeat.action.query": "INSERT INTO public.runs (id, org_id, recipient, correlation_id, url, service, timeout, created_at, updated_at) VALUES ('98875b33-b37e-4c35-be8b-d74f321bac28', '5318290', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'https://redhat.com', 'heartbeat', 3600, NOW(), NOW()) ON CONFLICT(id) DO UPDATE SET updated_at=NOW();"
+    }
+}