Bootstrap integration - Travis upgrade (#508)

Bootstrap integration with rest
Travis release process and dev branch improvements
Updated readme to describe the new way of running the server.

Co-authored-by: Xavi Artigas <[email protected]>

.gitignore

@@ -15,3 +15,5 @@ jsconfig.json
 rest-*.json
 
 rest/coverage
+/rest/target/
+/rest/logs.log

.travis.yml

@@ -4,43 +4,39 @@ node_js:
   - "12"
 services:
   - docker
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    - sourceline: 'deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.2 multiverse'
-      key_url: 'https://www.mongodb.org/static/pgp/server-4.2.asc'
-    packages:
-    - libzmq3-dev
-    - mongodb-org
-    - mongodb-org-mongos
-    - mongodb-org-server
-    - mongodb-org-shell
-    - mongodb-org-tools
-
 env:
+  global:
+    - DEV_BRANCH=dev
+    - RELEASE_BRANCH=main
+    - POST_RELEASE_BRANCH=main
+    - RELEASE_MESSAGE=release
   matrix:
     - SUBPROJECT=catapult-sdk
     - SUBPROJECT=rest
     - SUBPROJECT=spammer
-
 cache: yarn
 
 before_script:
-- sudo systemctl start mongod
+- . ./travis/travis-functions.sh && log_env_variables
+- echo '$SUBPROJECT'
 - sh yarn_setup.sh
 
 script: cd ${SUBPROJECT} && yarn run lint && yarn run test:travis && cd ..
 
 jobs:
   include:
-    - stage: docker alpha deploy
-      env:
-        - SUBPROJECT=rest
-      script: /bin/sh docker.sh
-      if: branch = main AND type != pull_request
-    - stage: docker release deploy
-      env:
-        - SUBPROJECT=rest
-      script: /bin/sh docker.sh --release
-      if: tag IS present
+    - stage: test
+      name: docker test build
+      script: /bin/bash travis/docker-functions.sh docker_build_version_file
+    - stage: publish
+      name: docker publish alpha
+      script: /bin/bash travis/docker-functions.sh docker_build_version_file publish
+      if: branch = env(DEV_BRANCH) AND type = push
+    - stage: release
+      name: docker publish release
+      script: /bin/bash travis/docker-functions.sh docker_build_version_file release
+      if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE)
+    - stage: post release
+      name: tag and version upgrade
+      script: /bin/bash travis/travis-functions.sh post_release_version_file
+      if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE)

README.md

@@ -9,57 +9,60 @@ Catapult REST gateway combines HTTP and WebSockets to perform read and write act
 
 - Node.js 12 LTS
 - [yarn][yarn] dependency manager
-- [catapult-server][catapult-server] configured as an [API or Dual node][api-node]
-- MongoDB 4.2
+- [docker][docker]
+- [symbol-bootstrap][symbol-bootstrap] 
 
 ## Installation
 
-1. Make sure the configuration file ``rest/resources/rest.json`` matches the API node connection details.
+1. Validate you are able to run Bootstrap by following the [requirements](https://github.com/nemtech/symbol-bootstrap#requirements).
 
-| Parameter | Description | Example  |
-|-|-|-|
-| db.url | MongoDB connection URL. | mongodb://localhost:27017/ |
-| apiNode.host | API node connection host. | 127.0.0.1 |
-| apiNode.port | API node connection port. | 7900 |
-| websocket.mq.host | ZeroMQ connection host. |  127.0.0.1 |
-| websocket.mq.port | ZeroMQ connection port. |  7902 |
+2. Install the project dependencies:
 
-*Note:* catapult-rest has to reach the API node, ZeroMQ, and MongoDB ports. If you are running catapult-server on a VPS, you can bind the ports to your local development environment creating an **SSH tunnel**: ``ssh -L 27017:localhost:27017 -L 7900:localhost:7900 -L 7902:localhost:7902 -p 2357 <USER>@<VPS_IP>``
+```
+./yarn_setup.sh
+```
 
-2. Catapult uses **TLS 1.3** to provide secure connections and identity assurance between the nodes.
-To generate and self sign the certificates, you can download and run the script [cert-generate.sh](https://github.com/tech-bureau/catapult-service-bootstrap/blob/master/common/ruby/script/cert-generate.sh).
+3. Run a Symbol private network using Bootstrap:
 
-```ssh
-mkdir certificate
-cd certificate
-curl https://raw.githubusercontent.com/tech-bureau/catapult-service-bootstrap/master/common/ruby/script/cert-generate.sh --output cert-generate.sh
-chmod 777 cert-generate.sh
-./cert-generate.sh
 ```
-Alternatively, you could use a certificate issued by a certification authority (CA).
-Then, edit ``rest/resources/rest.json`` TLS configuration:
+cd rest
+yarn build
+yarn bootstrap-start
+```
 
-| Parameter | Description | Example  |
-|-|-|-|
-| apiNode.tlsClientCertificatePath | TLS client certificate path. | /certificate/node.crt.pem |
-| apiNode.tlsClientKeyPath | TLS client key certificate path.  | /certificate/node.key.pem |
-| apiNode.tlsCaCertificatePath| TLS CA certificate path. | /certificate/ca.cert.pem |
+This Symbol network is a [light](https://github.com/nemtech/symbol-bootstrap#out-of-the-box-presets) preset network without rest. 
+Rest will be running from source code, so you can test your changes! 
+Mongo DB (27017), Server (7900) and Broker (7902) ports are open to localhost.
+
+4. Run catapult-rest:
 
-3. Install the project dependencies:
+In another terminal:
 
 ```
-./yarn_setup.sh
+yarn start
 ```
 
-4. Run catapult-rest:
+If everything goes well, you should see catapult-rest running by opening ``http://localhost:3000/node/info`` in a new browser tab.
+
+Alternatively, you can run bootstrap in `detached` mode to avoid opening a new terminal. 
 
 ```
 cd rest
 yarn build
-yarn start resources/rest.json
+yarn bootstrap-start-detached
+yarn start
+yarn bootstrap-stop
 ```
 
-If everything goes well, you should see catapult-rest running by opening ``localhost:3000/node/info`` in a new browser tab.
+Useful for test automation: 
+
+```
+cd rest
+yarn build
+yarn bootstrap-start-detached
+yarn test
+yarn bootstrap-stop
+```
 
 ## Usage
 
@@ -95,4 +98,6 @@ Copyright (c) 2018 Jaguar0625, gimre, BloodyRookie, Tech Bureau, Corp Licensed u
 
 [yarn]: https://yarnpkg.com/lang/en/
 [catapult-server]: https://github.com/nemtech/catapult-server
+[symbol-bootstrap]: https://github.com/nemtech/symbol-bootstrap
+[docker]: https://www.docker.com
 [api-node]: https://nemtech.github.io/server.html#installation

catapult-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catapult-sdk",
-  "version": "2.1.0",
+  "version": "0.0.0",
   "description": "Catapult SDK core",
   "main": "_build/index.js",
   "scripts": {

rest/bootstrap-preset.yml

@@ -0,0 +1,14 @@
+#nodeEqualityStrategy: public-key
+nodes:
+  - repeat: 0
+  - trustedHosts: '127.0.0.1, 172.20.0.25, 172.20.0.1'
+    localNetworks: '127.0.0.1, 172.20.0.25, 172.20.0.1'
+    brokerOpenPort: 7902
+gateways:
+  - excludeDockerService: true
+    name: rest
+    apiNodeConfigPath: target/gateways/rest/api-node-config
+    restLoggingFilename: target/rest.log
+    databaseHost: localhost
+    apiNodeHost: localhost
+    apiNodeBrokerHost: localhost

rest/package.json

@@ -1,21 +1,30 @@
 {
   "name": "catapult-api-rest",
-  "version": "2.1.0",
+  "version": "0.0.0",
   "description": "",
   "main": "_build/index.js",
   "scripts": {
     "clean": "rimraf _build && mkdir _build",
     "build": "ncp src/ _build",
     "rebuild": "npm run clean && npm run build",
     "version": "echo $npm_package_version",
-    "start": "node _build/index.js",
-    "start:debug": "node src/index.js",
+    "start": "node _build/index.js target/gateways/rest/rest.json",
+    "start:debug": "node src/index.js target/gateways/rest/rest.json",
     "test": "mocha --full-trace --recursive",
     "test:coverage": "nyc npm test && nyc report --reporter=text-lcov",
     "test:jenkins": "cross-env JUNIT_REPORT_PATH=test-results.xml mocha --reporter mocha-jenkins-reporter --mongoHost db --forbid-only --full-trace --recursive test || exit 0",
-    "test:travis": "nyc npm test && nyc report --reporter=text-lcov | coveralls",
+    "test:travis": "npm run bootstrap-start-detached && nyc npm test && nyc report --reporter=text-lcov | coveralls && npm run bootstrap-stop ",
     "lint": "eslint src test",
-    "lint:jenkins": "eslint -o tests.catapult.lint.xml -f junit src test || exit 0"
+    "lint:fix": "eslint src test --fix",
+    "lint:jenkins": "eslint -o tests.catapult.lint.xml -f junit src test || exit 0",
+    "bootstrap-clean": "symbol-bootstrap clean",
+    "bootstrap-start": "symbol-bootstrap start -a light -c bootstrap-preset.yml --healthCheck",
+    "bootstrap-start-detached": "symbol-bootstrap start -a light -c bootstrap-preset.yml --detached --healthCheck",
+    "bootstrap-stop": "symbol-bootstrap stop",
+    "bootstrap-run": "symbol-bootstrap run",
+    "bootstrap-run-detached": "symbol-bootstrap run --detached --healthCheck",
+    "bootstrap-compose": "symbol-bootstrap compose",
+    "bootstrap-config": "symbol-bootstrap config -a light -c bootstrap-preset.yml"
   },
   "keywords": [],
   "author": "",
@@ -35,7 +44,8 @@
     "mocha-jenkins-reporter": "^0.4.3",
     "ncp": "^2.0.0",
     "rimraf": "^2.6.3",
-    "sinon": "^7.3.2"
+    "sinon": "^7.3.2",
+    "symbol-bootstrap": "^0.2.0"
   },
   "dependencies": {
     "catapult-sdk": "link:../catapult-sdk",

rest/src/routes/nodeRoutes.js

@@ -20,17 +20,22 @@
  */
 
 const routeResultTypes = require('./routeResultTypes');
-const { version: sdkVersion } = require('../../../catapult-sdk/package.json');
-const { version: restVersion } = require('../../package.json');
 const nodeInfoCodec = require('../sockets/nodeInfoCodec');
 const nodePeersCodec = require('../sockets/nodePeersCodec');
 const nodeTimeCodec = require('../sockets/nodeTimeCodec');
 const catapult = require('catapult-sdk');
+const fs = require('fs');
+const path = require('path');
 
 const packetHeader = catapult.packet.header;
 const { PacketType } = catapult.packet;
 const { BinaryParser } = catapult.parser;
 
+// ATM, both rest and rest sdk share the same version. In the future,
+// we will have an open api and sdk dependencies with their given versions.
+const restVersion = fs.readFileSync(path.resolve(__dirname, '../../../version.txt'), 'UTF-8').trim();
+const sdkVersion = restVersion;
+
 const buildResponse = (packet, codec, resultType) => {
 	const binaryParser = new BinaryParser();
 	binaryParser.push(packet.payload);

rest/test/routes/nodeRoutes_spec.js

@@ -20,11 +20,16 @@
  */
 
 const { MockServer, test } = require('./utils/routeTestUtils');
-const { version: sdkVersion } = require('../../../catapult-sdk/package.json');
-const { version: restVersion } = require('../../package.json');
 const nodeRoutes = require('../../src/routes/nodeRoutes');
 const errors = require('../../src/server/errors');
 const { expect } = require('chai');
+const fs = require('fs');
+const path = require('path');
+
+// ATM, both rest and rest sdk share the same version. In the future,
+// we will have an open api and sdk dependencies with their given versions.
+const restVersion = fs.readFileSync(path.resolve(__dirname, '../../../version.txt'), 'UTF-8').trim();
+const sdkVersion = restVersion;
 
 describe('node routes', () => {
 	describe('get', () => {

rest/test/server/bootstrapper_spec.js

@@ -756,7 +756,7 @@ describe('server (bootstrapper)', () => {
 		// server.ws allows you to register any name and you can register multiple different routes
 		// the tests are using custom `/ws/block*` routes
 
-		const ports = { server: 1234, mq: 7902 };
+		const ports = { server: 1234, mq: 7912 };
 		const delays = { publish: 50 };
 
 		const createBlockBuffer = tag => Buffer.concat([