diff --git a/installer/compose/README.md b/installer/compose/README.md index 93dc48dfca..14cc853c65 100644 --- a/installer/compose/README.md +++ b/installer/compose/README.md @@ -48,6 +48,7 @@ We provide three options to get you going: - **default**: the standard installation, uses Kafka as internal message broker (recommended) - **nats**: the standard installation which uses Nats as message broker (recommended for new installations) - **full**: contains experimental Flink wrappers +- **quickstart**: contains pre-configured sample assets, including pipelines, dashboards, and data views. We recommend first-time StreamPipes users to use the Quickstart mode to experience the convenience of StreamPipes in IIoT! (Recommended for first-time users) The ``nats`` version will become the default version in a later release. You can already try it for new installations, but there's not yet an automatic migration from current Kafka-based installations to Nats. @@ -57,7 +58,7 @@ but there's not yet an automatic migration from current Kafka-based installation ```bash docker-compose up -d -# go to after all services are started http://localhost +# go to `http://localhost` after all services are started ``` After all containers are successfully started just got to your browser and visit http://localhost to finish the installation. Once finished, switch to the pipeline editor and start the interactive tour or check the [documentation](https://streampipes.apache.org/docs/docs/user-guide-introduction.html) to learn more about StreamPipes! @@ -71,23 +72,32 @@ docker-compose down Starting the **nats** option works as follows: ```bash docker-compose -f docker-compose.nats.yml up -d -# go to after all services are started http://localhost +# go to `http://localhost` after all services are started ``` Stopping the **nats** option: ```bash docker-compose -f docker-compose.nats.yml down -#docker-compose -f docker-compose.nats.yml down ``` Starting the **full** option is almost the same, just specify the `docker-compose.full.yml` file: ```bash docker-compose -f docker-compose.full.yml up -d -# go to after all services are started http://localhost +# go to `http://localhost` after all services are started ``` Stopping the **full** option: ```bash docker-compose -f docker-compose.full.yml down -#docker-compose -f docker-compose.full.yml down +``` + +We introduce quickstart deployment that comes with a set of predefined StreamPipes assets, to use the quickstart mode, just build the Docker image and start **quickstart** option: +```bash +docker-compose -f docker-compose.quickstart.yml build script-runner +docker-compose -f docker-compose.quickstart.yml up -d +# go to `http://localhost` after all services are started +``` +Stopping the **quickstart** option: +```bash +docker-compose -f docker-compose.quickstart.yml down ``` ## Update services diff --git a/installer/compose/docker-compose.quickstart.yml b/installer/compose/docker-compose.quickstart.yml new file mode 100644 index 0000000000..c676359ea4 --- /dev/null +++ b/installer/compose/docker-compose.quickstart.yml @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +version: "3.4" + +# global logging +x-logging: + &default-logging + options: + max-size: '12m' + max-file: '5' + driver: json-file + +services: + + # StreamPipes services + backend: + image: "${SP_DOCKER_REGISTRY}/backend:${SP_VERSION}" + depends_on: + - consul + - couchdb + volumes: + - backend:/root/.streampipes + - files:/spImages + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + ui: + image: "${SP_DOCKER_REGISTRY}/ui:${SP_VERSION}" + ports: + - "80:8088" + depends_on: + - couchdb + - consul + - backend + volumes: + - nginx:/etc/nginx/ + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + consul: + image: consul:1.14.3 + environment: + - "CONSUL_LOCAL_CONFIG={\"disable_update_check\": true}" + - "CONSUL_BIND_INTERFACE=eth0" + - "CONSUL_HTTP_ADDR=0.0.0.0" + entrypoint: + - consul + - agent + - -server + - -bootstrap-expect=1 + - -data-dir=/consul/data + - -node=consul-one + - -bind={{ GetInterfaceIP "eth0" }} + - -client=0.0.0.0 + - -enable-script-checks=true + - -ui + volumes: + - consul:/consul/data + logging: *default-logging + restart: unless-stopped + networks: + spnet: + ipv4_address: ${SP_CONSUL_CONTAINER_IP} + + couchdb: + image: couchdb:3.3.1 + environment: + - COUCHDB_USER=admin + - COUCHDB_PASSWORD=admin + volumes: + - couchdb:/opt/couchdb/data + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + kafka: + image: fogsyio/kafka:2.2.0 + hostname: kafka + depends_on: + - zookeeper + environment: + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # Replace localhost with your external address if Kafka should be reachable from external systems. + KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_MESSAGE_MAX_BYTES: 5000012 + KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012 + KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000 + volumes: + - kafka:/kafka + - /var/run/docker.sock:/var/run/docker.sock + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + zookeeper: + image: fogsyio/zookeeper:3.4.13 + volumes: + - zookeeper:/opt/zookeeper-3.4.13 + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + influxdb: + image: influxdb:2.6 + environment: + - INFLUXDB_DATA_ENGINE=tsm1 + - INFLUXDB_REPORTING_DISABLED=false + - INFLUXDB_ADMIN_ENABLED=true + - DOCKER_INFLUXDB_INIT_USERNAME=admin + - DOCKER_INFLUXDB_INIT_PASSWORD=sp-admin + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=sp-admin + - DOCKER_INFLUXDB_INIT_ORG=sp + - DOCKER_INFLUXDB_INIT_BUCKET=sp + - DOCKER_INFLUXDB_INIT_MODE=${SP_INFLUX_INIT_MODE} + volumes: + - influxdb:/var/lib/influxdb + - influxdb2:/var/lib/influxdb2 + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + # Lite version pipeline elements + extensions-all-jvm: + image: "${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}" + depends_on: + - consul + volumes: + - files:/spImages + logging: *default-logging + restart: unless-stopped + networks: + spnet: + + script-runner: + build: ./quickstart + depends_on: + - backend + volumes: + - ./quickstart/zip_folder:/zip_folder + networks: + spnet: + + + +volumes: + backend: + consul: + connect: + couchdb: + kafka: + zookeeper: + influxdb: + influxdb2: + files: + nginx: + + +networks: + spnet: + driver: bridge + ipam: + config: + - subnet: ${SP_SUBNET} \ No newline at end of file diff --git a/installer/compose/quickstart/Dockerfile b/installer/compose/quickstart/Dockerfile new file mode 100644 index 0000000000..b1be08b658 --- /dev/null +++ b/installer/compose/quickstart/Dockerfile @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Use an official base image +FROM ubuntu:latest + +RUN apt-get update \ + && apt-get install -y curl jq + +COPY upload.sh /usr/local/bin/upload.sh +RUN chmod +x /usr/local/bin/upload.sh + +CMD ["/usr/local/bin/upload.sh"] diff --git a/installer/compose/quickstart/upload.sh b/installer/compose/quickstart/upload.sh new file mode 100755 index 0000000000..6c0a2b8d4b --- /dev/null +++ b/installer/compose/quickstart/upload.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +if [ -z "$SP_INITIAL_ADMIN_EMAIL" ]; then + USERNAME="admin@streampipes.apache.org" +else + USERNAME="$SP_INITIAL_ADMIN_EMAIL" +fi + +if [ -z "$SP_INITIAL_ADMIN_PASSWORD" ]; then + PASSWORD="admin" +else + PASSWORD="$SP_INITIAL_ADMIN_PASSWORD" +fi + +while true; do + if curl -s "http://backend:8030/streampipes-backend/api/v2/auth/login" --max-time 10 &> /dev/null; then + echo "StreamPipes backend is now ready!" + break + else + echo "StreamPipes backend not ready, waiting for 3 seconds..." + sleep 3 + fi +done + +JSON_TOKEN_RESPONSE=$(curl -s -X POST "http://backend:8030/streampipes-backend/api/v2/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\"}") + +TOKEN=$(echo "$JSON_TOKEN_RESPONSE" | jq -r '.accessToken') +RESPONSE_TOKEN="Bearer $TOKEN" + +JSON_RESOURCE_ITEMS=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/rdfendpoints/items" \ + -H "Content-Type: application/json" \ + -H "Authorization: $RESPONSE_TOKEN") + +ARRAY_LENGTH=$(echo $JSON_RESOURCE_ITEMS | jq '. | length') + +while [ $ARRAY_LENGTH -le 100 ]; do + echo "StreamPipes Extensions Service not ready, waiting for 3 seconds..." + sleep 3 + JSON_RESOURCE_ITEMS=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/rdfendpoints/items" \ + -H "Content-Type: application/json" \ + -H "Authorization: $RESPONSE_TOKEN") + ARRAY_LENGTH=$(echo $JSON_RESOURCE_ITEMS | jq '. | length') +done + +ITEM_MAP_NAME_URI=$(echo $JSON_RESOURCE_ITEMS | jq -r 'map({(.name): .uri}) | add') + +ITEM_MAP_NAME_URI=$(echo $JSON_RESOURCE_ITEMS | jq -r 'map({(.name): .uri}) | add') + +for ZIP_FILE in /zip_folder/*.zip; do + JSON_RESPONSE=$(curl --compressed -X POST "http://backend:8030/streampipes-backend/api/v2/import/preview" \ + -H "Authorization: $RESPONSE_TOKEN" \ + -F "file_upload=@$ZIP_FILE") + + PIPELINE_IDS=($(echo "$JSON_RESPONSE" | jq -r '.pipelines[].resourceId')) + + for id in "${PIPELINE_IDS[@]}"; do + + ITEMS_PIPELINES=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/pipelines/$id" \ + -H "Content-Type: application/json" \ + -H "Authorization: $RESPONSE_TOKEN") + + KEYS=$(echo $ITEMS_PIPELINES | jq -r 'keys[]') + + for key in $KEYS; do + if [ "$(echo $ITEMS_PIPELINES | jq ".${key} | type")" == '"array"' ]; then + key=$(echo $ITEMS_PIPELINES | jq -r ".${key}[]? | .name") + value=$(echo $ITEM_MAP_NAME_URI | jq -r ".\"$key\"") + + curl -s -X POST "http://backend:8030/streampipes-backend/api/v2/element" \ + -H "Content-Type: application/x-www-form-urlencoded;charset=UTF-8" \ + -H "Authorization: $RESPONSE_TOKEN" \ + --data-urlencode "uri=$value" \ + --data-urlencode "publicElement=true" + fi + done + done + + JSON_PAYLOAD="$JSON_RESPONSE" + + curl -i -X POST "http://backend:8030/streampipes-backend/api/v2/import" \ + -H "Authorization: $RESPONSE_TOKEN" \ + -F "file_upload=@$ZIP_FILE" \ + -F "configuration=@-;type=application/json" <<< "$JSON_PAYLOAD" +done diff --git a/installer/compose/quickstart/zip_folder/Example Asset - Smart Factory Monitoring and Analysis.zip b/installer/compose/quickstart/zip_folder/Example Asset - Smart Factory Monitoring and Analysis.zip new file mode 100644 index 0000000000..4c0895a614 Binary files /dev/null and b/installer/compose/quickstart/zip_folder/Example Asset - Smart Factory Monitoring and Analysis.zip differ diff --git a/tools/maven/installer.xml b/tools/maven/installer.xml index cca59a5e52..b88b2300ac 100644 --- a/tools/maven/installer.xml +++ b/tools/maven/installer.xml @@ -30,6 +30,7 @@ true %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.gitignore] + ${project.basedir}/compose/quickstart/zip_folder/*.zip ${project.basedir}/installer .