Merge pull request #104 from CudoVentures/automated-end-to-end-testin…

…g-v0.47.3

Automated end to end testing v0.47.3

Makefile

@@ -65,9 +65,15 @@ start-docker-test: stop-docker-test
 
 test-unit: start-docker-test
 	@echo "Executing unit tests..."
-	@go test -mod=readonly -v -coverprofile coverage.txt ./...
+	@go test -mod=readonly -v -coverprofile coverage.txt $(shell go list ./... | grep -v /integration_tests/)
 .PHONY: test-unit
 
+test-integration:
+	@echo "Running integration/e2e tests..."
+	@chmod +x ./integration_tests/run_tests_in_docker.sh
+	@./integration_tests/run_tests_in_docker.sh
+.PHONY: test-integration
+
 ###############################################################################
 ###                                Linting                                  ###
 ###############################################################################

README.md

@@ -17,7 +17,8 @@ created using [Hasura](https://hasura.io/).
 To know how to setup and run BDJuno, please refer to
 the [docs website](https://docs.bigdipper.live/cosmos-based/parser/overview/).
 
-## Testing
+# Testing
+## UNIT
 If you want to test the code, you can do so by running
 
 ```shell
@@ -30,6 +31,23 @@ This will:
 1. Create a Docker container running a PostgreSQL database.
 2. Run all the tests using that database as support.
 
+## INTEGRATION/END-TO-END
+If you want to test the code, you can do so by running
+
+```shell
+$ make test-integration
+```
+
+**Note**: Requires [Docker](https://docker.com).
+
+This will:
+
+1. Pull and run locally a root node instance
+2. Create a Docker container running a PostgreSQL database.
+3. Pull and run locally a BDJuno instance
+4. Execute actual TXs against the node and test BDJuno for correctly parsing them into the DB.
+5. Clean up after tests being completed
+
 # Cudos Fork
 
 The fork is based on branch cosmos/v0.47.x. It is at 67a1737418672d5357e72731b5e99a4c460c19df, which is 4 commit ahead of official version 4.0.0. These 4 commits (actually the last one) adds support to cosmos-sdk v0.47 proposals.

database/types/gov.go

@@ -112,6 +112,15 @@ func (w TallyResultRow) Equals(v TallyResultRow) bool {
 		w.Height == v.Height
 }
 
+// WeightedVoteRow represents a single row inside the proposal_vote_weighted table
+type WeightedVoteRow struct {
+	ProposalID int64  `db:"proposal_id"`
+	Voter      string `db:"voter_address"`
+	Option     string `db:"option"`
+	Weight     string `db:"weight"`
+	Height     int64  `db:"height"`
+}
+
 // VoteRow represents a single row inside the vote table
 type VoteRow struct {
 	ProposalID int64     `db:"proposal_id"`

integration_tests/Dockerfile

@@ -0,0 +1,11 @@
+FROM golang:1.20-buster AS builder
+USER root
+
+# Install jq and git
+RUN apt-get update && apt-get install -y jq git && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp/integration_tests
+
+COPY . ./
+RUN chmod +x ./set_and_run_tests.sh
+CMD ["./set_and_run_tests.sh"]

integration_tests/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3.6'
+services:
+  postgres:
+    image: postgres:14
+    restart: always
+    ports:
+      - "6666:5432"
+    environment:
+      POSTGRES_DB: bdjuno_test_db
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: 12345
+  tests:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - postgres

integration_tests/run_tests_in_docker.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+SETUP_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$SETUP_DIR"
+docker-compose -p integration_e2e down -v
+docker-compose -p integration_e2e up --build -d --force-recreate
+docker-compose -p integration_e2e logs --follow tests
+
+while [ "$(docker ps -q -f name=tests)" ]; do
+    sleep 5
+done
+
+echo "Tests finished"
+docker-compose -p integration_e2e down -v

integration_tests/set_and_run_tests.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+SETUP_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$SETUP_DIR/set_up"
+chmod +x ./start_node.sh
+echo "Starting the node..."
+source ./start_node.sh
+
+cd "$SETUP_DIR/set_up"
+chmod +x ./start_bdjuno.sh
+echo "Starting BDJuno..."
+source ./start_bdjuno.sh 

integration_tests/set_up/set_up.go

@@ -0,0 +1,248 @@
+package set_up
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/forbole/bdjuno/v4/integration_tests/types"
+
+	// Use effects
+	_ "github.com/lib/pq"
+)
+
+const (
+	dbHost     = "postgres"
+	dbPort     = 5432
+	dbUser     = "postgres"
+	dbPassword = "12345"
+	dbName     = "bdjuno_test_db"
+)
+
+var (
+	db                *sql.DB
+	averageBlockTime  = 7
+	cudosHome         = "/tmp/cudos-test-data"
+	cmd               = "cudos-noded"
+	Denom             = "acudos"
+	txCmd             = "tx"
+	queryCmd          = "q"
+	from              = "from"
+	authModule        = "auth"
+	homeFlag          = fmt.Sprintf("--home=%s", cudosHome)
+	keyringFlag       = "--keyring-backend=test"
+	gasAdjustmentFlag = "--gas-adjustment=1.3"
+	gasFlag           = "--gas=8000000"
+	feesFlag          = fmt.Sprintf("--fees=1000000000000000000%s", Denom)
+	gasPricesFlag     = fmt.Sprintf("--gas-prices=5000000000000%s", Denom)
+
+	GetFlag = func(key string, value string) string {
+		return fmt.Sprintf("--%s=%s", key, value)
+	}
+)
+
+func waitForAvgBlockTime() {
+	time.Sleep(time.Duration(averageBlockTime) * time.Second)
+}
+
+func getAddress(user string) (string, error) {
+	args := []string{
+		"keys", "show", user, "-a", keyringFlag,
+	}
+	return executeCommand(args...)
+}
+
+func executeCommand(args ...string) (string, error) {
+	args = append(args, []string{homeFlag}...)
+	output, err := exec.Command(cmd, args...).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("command failed: %v; output: %s", err, output)
+	}
+	return strings.TrimSpace(string(output)), nil
+}
+
+func ExecuteTxCommandWithFees(signerAddress string, args ...string) (string, error) {
+	additionalFlags := []string{GetFlag(from, signerAddress), keyringFlag, feesFlag, "-y"}
+	args = append(args, additionalFlags...)
+	args = append([]string{txCmd}, args...)
+	return executeCommand(args...)
+}
+
+func ExecuteTxCommandWithGas(signerAddress string, args ...string) (string, error) {
+	additionalFlags := []string{GetFlag(from, signerAddress), keyringFlag, gasAdjustmentFlag, gasFlag, gasPricesFlag, "-y"}
+	args = append(args, additionalFlags...)
+	args = append([]string{txCmd}, args...)
+	return executeCommand(args...)
+}
+
+func IsTxSuccess(result string) (string, uint64, error) {
+	txResult := extractTxResult(result)
+
+	if txResult.Code != 0 || txResult.TxHash == "" {
+		return "", 0, fmt.Errorf("invalid tx")
+	}
+
+	// Waiting for the successful Tx to be included in a block
+	waitForAvgBlockTime()
+
+	// Requerying to confirm it is included in a block
+	result, err := executeCommand([]string{
+		queryCmd, "tx", txResult.TxHash,
+	}...)
+	if err != nil {
+		return "", 0, err
+	}
+
+	txResult = extractTxResult(result)
+	if txResult.Height > 0 && txResult.Code == 0 {
+		return txResult.TxHash, txResult.Height, nil
+	}
+
+	return "", 0, fmt.Errorf(txResult.RawLog)
+}
+
+func GetTestUserAddress(userNumber int) string {
+	result, err := getAddress(fmt.Sprintf("testUser%s", strconv.Itoa(userNumber)))
+	if err != nil {
+		return ""
+	}
+	return result
+}
+
+func GetTestAdminAddress() string {
+	result, err := getAddress("test-admin")
+	if err != nil {
+		return ""
+	}
+	return result
+}
+
+func SaveToTempFile(v interface{}) (string, error) {
+	data, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	tmpFile, err := ioutil.TempFile("", "prefix-*.json")
+	if err != nil {
+		return "", err
+	}
+
+	_, err = tmpFile.Write(data)
+	if err != nil {
+		tmpFile.Close() // ensure closure of the file in case of error
+		return "", err
+	}
+
+	return tmpFile.Name(), tmpFile.Close()
+}
+
+func extractAddressFromQueryResult(queryResultString string) string {
+	var address string
+	re := regexp.MustCompile(`address: (\w+)`)
+	match := re.FindStringSubmatch(queryResultString)
+	if len(match) > 1 {
+		address = match[1]
+	}
+	return address
+}
+
+func extractTxResult(result string) types.TxResult {
+	var txRes types.TxResult
+
+	// Extract code
+	codeRegex := regexp.MustCompile(`code: (\d+)`)
+	codeMatches := codeRegex.FindStringSubmatch(result)
+	if len(codeMatches) > 1 {
+		code, _ := strconv.Atoi(codeMatches[1])
+		txRes.Code = code
+	}
+
+	// Extract height
+	heightRegex := regexp.MustCompile(`height: "(\d+)"`)
+	heightMatches := heightRegex.FindStringSubmatch(result)
+	if len(heightMatches) > 1 {
+		height, _ := strconv.ParseUint(heightMatches[1], 10, 64)
+		txRes.Height = height
+	}
+
+	// Extract txhash
+	txhashRegex := regexp.MustCompile(`txhash: (\w+)`)
+	txhashMatches := txhashRegex.FindStringSubmatch(result)
+	if len(txhashMatches) > 1 {
+		txRes.TxHash = txhashMatches[1]
+	}
+
+	// Extract logs
+	rawLogRegex := regexp.MustCompile(`(?s)raw_log: '(.*?)'`)
+	rawLogMatches := rawLogRegex.FindStringSubmatch(result)
+	if len(rawLogMatches) > 1 {
+		txRes.RawLog = rawLogMatches[1]
+	}
+
+	return txRes
+}
+
+func setupDB() {
+	connStr := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", dbHost, dbPort, dbUser, dbPassword, dbName)
+	db, _ = sql.Open("postgres", connStr)
+}
+
+func QueryDatabase(query string, args ...interface{}) *sql.Row {
+	if db == nil {
+		setupDB()
+	}
+	return db.QueryRow(query, args...)
+}
+
+func QueryDatabaseMultiRows(query string, args ...interface{}) (*sql.Rows, error) {
+	if db == nil {
+		setupDB()
+	}
+	return db.Query(query, args...)
+}
+
+func IsParsedToTheDb(txHash string, blockHeight uint64) bool {
+	var exists bool
+
+	// Waiting for the successful Tx to be parsed to the DB
+	waitForAvgBlockTime()
+
+	// Check if the block is parsed to the DB
+	err := QueryDatabase(`SELECT EXISTS(SELECT 1 FROM block WHERE height = $1)`, blockHeight).Scan(&exists)
+	if err != nil || !exists {
+		return false
+	}
+
+	// Check if the tx is parsed to the DB
+	err = QueryDatabase(`SELECT EXISTS(SELECT 1 FROM transaction WHERE hash = $1)`, txHash).Scan(&exists)
+	if err != nil || !exists {
+		return false
+	}
+
+	return exists
+}
+
+func GetModuleAccountAddress(moduleName string) (string, error) {
+	var address string
+
+	args := []string{
+		queryCmd,
+		authModule,
+		"module-account",
+		moduleName,
+	}
+
+	result, err := executeCommand(args...)
+	if err != nil {
+		return address, err
+	}
+	address = extractAddressFromQueryResult(result)
+	return address, nil
+}

integration_tests/set_up/start_bdjuno.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -e
+
+TEST_BRANCH="automated-end-to-end-testing-v0.47.3"
+export BDJUNO_INSTALL_PATH="/tmp/cudos-test-bdjuno"
+export BDJUNO_CONFIG_PATH="/sample_configs/integration-tests-config/bdjuno"
+TESTS_PATH="$BDJUNO_INSTALL_PATH/integration_tests/tests"
+HOME="$BDJUNO_INSTALL_PATH$BDJUNO_CONFIG_PATH"
+
+# cleanup from previous runs
+pkill bdjuno || true
+rm -rf $BDJUNO_INSTALL_PATH
+
+# clone bdjuno repo and install binary
+git clone -b $TEST_BRANCH https://github.com/CudoVentures/cudos-bdjuno.git $BDJUNO_INSTALL_PATH
+cd $BDJUNO_INSTALL_PATH
+make install
+
+bdjuno database migrate --home $HOME
+bdjuno start --home $HOME &> /dev/null &
+
+cd $TESTS_PATH
+go test -v -p 1 ./...