Merge branch 'tt-1725-improve-go-doc-pipeline' into test_doc_generation

.github/workflows/framework-golden-tests.yml

@@ -10,9 +10,6 @@ jobs:
     env:
       LOKI_TENANT_ID: promtail
       LOKI_URL: http://localhost:3030/loki/api/v1/push
-      # this is not the best practice, and it must be fixed, run your tests WITHOUT IT!
-      # however, on current latest image we must use this flag
-      CTF_IGNORE_CRITICAL_LOGS: true
     runs-on: ubuntu-latest
     permissions:
       id-token: write

.github/workflows/framework.yml

@@ -9,6 +9,8 @@ jobs:
     defaults:
       run:
         working-directory: framework
+    env:
+      CTF_JD_IMAGE: ${{ secrets.CTF_JD_IMAGE }}
     runs-on: ubuntu-latest
     permissions:
       id-token: write

.github/workflows/generate-go-docs.yaml

@@ -64,8 +64,6 @@ jobs:
         env:
           FILTERS: ${{ steps.changes.outputs.changes }}        
         run: |
-          # remove after test
-          exit 1
           PATH=$PATH:$(go env GOPATH)/bin
           export PATH                  
 
@@ -97,9 +95,7 @@ jobs:
           cat filtered_folders.json | jq -c '.[]' | while read -r item; do
             folder=$(echo "$item" | jq -r '.folder')
             echo "Processing folder: $folder"
-            generate-go-function-docs diff -b ${{ github.event.pull_request.base.sha }} -c ${{ github.event.pull_request.head.sha }} --saveCosts --generator chatgpt --folder "$folder"
-            cd "$folder"
-            cd -
+            generate-go-function-docs diff -b "${{ github.event.pull_request.base.sha }}" -c "${{ github.event.pull_request.head.sha }}" --saveCosts --generator chatgpt --generatorSubType "${{ vars.GO_DOC_GEN_CHATGPT_MODEL }}" --folder "$folder"
           done
           rm filtered_folders.json
 
@@ -111,7 +107,7 @@ jobs:
 
       - name: Remove costs before committing
         shell: bash
-        run: rm -rf costs 
+        run: rm -rf costs
 
       - name: Setup GitHub Token for creating a new PR
         id: setup-github-token-write
@@ -123,6 +119,7 @@ jobs:
 
       - name: Create a new PR targeting current PR
         uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
+        id: create-pr
         with:
           token: ${{ steps.setup-github-token-write.outputs.access-token }}
           branch: ${{ github.head_ref }}-docs
@@ -131,42 +128,61 @@ jobs:
           body: "This PR contains automatically generated go documentation for the PR#${{ github.event.pull_request.number }}. Please review the changes."
           commit-message: "[Bot] Add automatically generated go documentation"
 
-      - name: Send Slack notification
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        if: failure() 
-        id: slack
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      - name: Find comment with PR link
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Go doc generation 
+
+      - name: Create comment in the original PR
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        if: ${{ steps.fc.outputs.comment-id }} == ''
         with:
-          channel-id: 'C049X3353K2'
-          payload: |
-            {
-              "attachments": [
-                {
-                  "color": "#C62828",
-                  "blocks": [
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "Go doc generation - Failed :x:"
-                      }
-                    },
-                    {
-                        "type": "section",
-                        "text": {
-                          "type": "mrkdwn",
-                          "text": "<@U060CGGPY8H>, please have a look."
-                        }
-                    },                    
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}|PR#${{ github.event.pull_request.number }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
+          token: ${{ steps.setup-github-token-write.outputs.access-token }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## Go doc generation
+            Hey @${{ github.actor }}, you can check generated Go function generation [here](${{ steps.create-pr.outputs.pull-request-url }}).
+            Please review them and merge to this PR once you're satisfied with them.
+
+      # - name: Send Slack notification
+      #   uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+      #   if: failure() 
+      #   id: slack
+      #   env:
+      #     SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      #   with:
+      #     channel-id: 'C049X3353K2'
+      #     payload: |
+      #       {
+      #         "attachments": [
+      #           {
+      #             "color": "#C62828",
+      #             "blocks": [
+      #               {
+      #                 "type": "section",
+      #                 "text": {
+      #                   "type": "mrkdwn",
+      #                   "text": "Go doc generation - Failed :x:"
+      #                 }
+      #               },
+      #               {
+      #                   "type": "section",
+      #                   "text": {
+      #                     "type": "mrkdwn",
+      #                     "text": "<@U060CGGPY8H>, please have a look."
+      #                   }
+      #               },                    
+      #               {
+      #                 "type": "section",
+      #                 "text": {
+      #                   "type": "mrkdwn",
+      #                   "text": "<${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}|PR#${{ github.event.pull_request.number }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+      #                 }
+      #               }
+      #             ]
+      #           }
+      #         ]
+      #       }

book/src/framework/components/blockchains/evm.md

@@ -72,5 +72,17 @@ Public: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266
 Private: ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
 ```
 
-Test keys for `Besu` can be found [here](https://besu.hyperledger.org/23.4.1/private-networks/reference/accounts-for-testing)
+For `Besu` keys are
+```
+Public: 0xfe3b557e8fb62b89f4916b721be55ceb828dbd73
+Private: 0x8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63
+
+Public: 0x627306090abaB3A6e1400e9345bC60c78a8BEf57
+Private: 0xc87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3
+
+Public: 0xf17f52151EbEF6C7334FAD080c5704D77216b732
+Private: 0xae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f
+```
+
+More docs for `Besu` can be found [here](https://besu.hyperledger.org/private-networks/reference/accounts-for-testing)
 

book/src/framework/components/state.md

@@ -40,7 +40,7 @@ One node set is enough for any kind of testing, if you need more nodes consider
 
 ## Custom ports
 
-You can also define a custom set of ports for any node
+You can also define a custom set of ports for any node.
 ```toml
 [nodeset]
   nodes = 5
@@ -53,6 +53,7 @@ You can also define a custom set of ports for any node
 
     [nodeset.node_specs.node]
       # here we defined 2 new ports to listen and mapped them to our host machine
-      custom_ports = [14000, 14001]
+      # syntax is "host:docker", if you provide only host port then we map 1-to-1
+      custom_ports = ["14000:15000", "20000"]
       image = "public.ecr.aws/chainlink/chainlink:v2.16.0"
 ```

book/src/framework/configuration.md

@@ -7,9 +7,10 @@
 |         CTF_CONFIGS          | Path(s) to test config files. <br/>Can be more than one, ex.: smoke.toml,smoke_1.toml,smoke_2.toml.<br/>First filepath will hold all the merged values | Any valid TOML file path |                          -                          |     ✅     |
 |        CTF_LOG_LEVEL         |                                                                   Harness log level                                                                    | `info`, `debug`, `trace` |                       `info`                        |    🚫     |
 |      CTF_PROMTAIL_DEBUG      |                                    Set `true` if you are integrating with remote `Loki` push API to debug Promtail                                     |          `true`, `false` |                       `false`                       |    🚫     |
+|   CTF_IGNORE_CRITICAL_LOGS   |                                      Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!)                                       |          `true`, `false` |                       `false`                       |    🚫     |
+|     CTF_CHAINLINK_IMAGE      |                                           Flag to override Chainlink Docker image in format $repository:$tag                                           |         $repository:$tag |                          -                          |    🚫     |
+|         CTF_JD_IMAGE         |                                                Job distributor service image in format $repository:$tag                                                |         $repository:$tag |                          -                          |    🚫     |
 |           LOKI_URL           |                                            URL to `Loki` push api, should be like`${host}/loki/api/v1/push`                                            |                      URL | `http://host.docker.internal:3030/loki/api/v1/push` |    🚫     |
 |        LOKI_TENANT_ID        |                                                Streams all components logs to `Loki`, see params below                                                 |                 `string` |                     `promtail`                      |    🚫     |
 |       LOKI_BASIC_AUTH        |                                                          Basic auth in format $user:$password                                                          |        `$user:$password` |                          -                          |    🚫     |
 |         RESTY_DEBUG          |                                                            Log all Resty client HTTP calls                                                             |          `true`, `false` |                       `false`                       |    🚫     |
-|   CTF_IGNORE_CRITICAL_LOGS   |                                      Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!)                                       |          `true`, `false` |                       `false`                       |    🚫     |
-|     CTF_CHAINLINK_IMAGE      |                                           Flag to override Chainlink Docker image in format $repository:$tag                                           |         $repository:$tag |                          -                          |    🚫     |

book/src/lib.md

@@ -252,38 +252,13 @@ Builder will read the location of chain configuration from env var named `PRIVAT
 
 `net` is an instance of `blockchain.EVMNetwork`, which contains characteristics of the network and can be used to connect to it using an EVM client. `rpc` variable contains arrays of public and private RPC endpoints, where "private" means URL that's accessible from the same Docker network as the chain is running in.
 
-# Using LogStream
+## Logs
+By default, we will save logs of all Docker containers running on the host machine, when the test ends (regardless whether it failed or succeeded). They will be available in the `./logs/<test-name><date>` directory. Same goes for dumping the databases of PostgresDBs
+used by the Chainlink nodes. These will be saves in the `./db_dumps/<test-name><date>` directory.
 
-LogStream is a package that allows to connect to a Docker container and then flush logs to configured targets. Currently 3 targets are supported:
+## Loki and Grafana
 
-- `file` - saves logs to a file in `./logs` folder
-- `loki` - sends logs to Loki
-- `in-memory` - stores logs in memory
-
-It can be configured to use multiple targets at once. If no target is specified, it becomes a no-op.
-
-LogStream has to be configured by passing an instance of `LoggingConfig` to the constructor.
-
-When you connect a container LogStream will create a new consumer and start a detached goroutine that listens to logs emitted by that container and which reconnects and re-requests logs if listening fails for whatever reason. Retry limit and timeout can both be configured using functional options. In most cases one container should have one consumer, but it's possible to have multiple consumers for one container.
-
-LogStream stores all logs in gob temporary file. To actually send/save them, you need to flush them. When you do it, LogStream will decode the file and send logs to configured targets. If log handling results in an error it won't be retried and processing of logs for given consumer will stop (if you think we should add a retry mechanism please let us know).
-
-_Important:_ Flushing and accepting logs is blocking operation. That's because they both share the same cursor to temporary file and otherwise it's position would be racey and could result in mixed up logs.
-
-## Configuration
-
-Basic `LogStream` TOML configuration is following:
-
-```toml
-[LogStream]
-log_targets=["file"]
-log_producer_timeout="10s"
-log_producer_retry_limit=10
-```
-
-You can find it here: [logging_default.toml](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/tomls/logging_default.toml)
-
-When using `in-memory` or `file` target no other configuration variables are required. When using `loki` target, following ones must be set:
+If you need to pass Loki or Grafana configuration to your tests you can do that by providing the following config:
 
 ```toml
 [Logging.Loki]
@@ -295,87 +270,13 @@ bearer_token_secret="bearer-token"
 
 Also, do remember that different URL should be used when running in CI and everywhere else. In CI it should be a public endpoint, while in local environment it should be a private one.
 
-If your test has a Grafana dashboard in order for the url to be correctly printed you should provide the following config:
+If your test has a Grafana dashboard you should provide the following config:
 
 ```toml
 [Logging.Grafana]
-url="http://grafana.somwhere.com/my_dashboard"
-```
-
-## Initialisation
-
-First you need to create a new instance:
-
-```golang
-// t - instance of *testing.T (can be nil)
-// testConfig.Logging - pointer to logging part of TestConfig
-ls := logstream.NewLogStream(t, testConfig.Logging)
-```
-
-## Listening to logs
-
-If using `testcontainers-go` Docker containers it is recommended to use life cycle hooks for connecting and disconnecting LogStream from the container. You can do that when creating `ContainerRequest` in the following way:
-
-```golang
-containerRequest := &tc.ContainerRequest{
-		LifecycleHooks: []tc.ContainerLifecycleHooks{
-			{PostStarts: []tc.ContainerHook{
-				func(ctx context.Context, c tc.Container) error {
-					if ls != nil {
-						return n.ls.ConnectContainer(ctx, c, "custom-container-prefix-can-be-empty")
-					}
-					return nil
-				},
-			},
-				PostStops: []tc.ContainerHook{
-					func(ctx context.Context, c tc.Container) error {
-						if ls != nil {
-							return n.ls.DisconnectContainer(c)
-						}
-						return nil
-					},
-				}},
-		},
-	}
-```
-
-You can print log location for each target using this function: `(m *LogStream) PrintLogTargetsLocations()`. For `file` target it will print relative folder path, for `loki` it will print URL of a Grafana Dashboard scoped to current execution and container ids. For `in-memory` target it's no-op.
-
-It is recommended to shutdown LogStream at the end of your tests. Here's an example:
-
-```golang
-t.Cleanup(func() {
-    l.Warn().Msg("Shutting down Log Stream")
-
-    if t.Failed() || os.Getenv("TEST_LOG_COLLECT") == "true" {
-        // we can't do much if this fails, so we just log the error
-        _ = logStream.FlushLogsToTargets()
-        // this will log log locations for each target (for file it will be a folder, for Loki Grafana dashboard -- remember to provide it's url in config!)
-        logStream.PrintLogTargetsLocations()
-        // this will save log locations in test summary, so that they can be easily accessed in GH's step summary
-        logStream.SaveLogLocationInTestSummary()
-    }
-
-    // we can't do much if this fails, so we just log the error
-    _ = logStream.Shutdown(testcontext.Get(b.t))
-    })
-```
-
-or in a bit shorter way:
-
-```golang
-t.Cleanup(func() {
-    l.Warn().Msg("Shutting down Log Stream")
-
-    if t.Failed() || os.Getenv("TEST_LOG_COLLECT") == "true" {
-        // this will log log locations for each target (for file it will be a folder, for Loki Grafana dashboard -- remember to provide it's url in config!)
-        logStream.PrintLogTargetsLocations()
-        // this will save log locations in test summary, so that they can be easily accessed in GH's step summary
-    }
-
-    // we can't do much if this fails
-    _ = logStream.FlushAndShutdown()
-    })
+base_url="https://your-grafana-url"
+dashboard_url="/my-dashboard"
+dashboard_uid="my-dashboard-uid" # optional
 ```
 
 ## Grouping test execution

book/src/lib/config/config.md

@@ -278,7 +278,6 @@ convert_to_toml_array() {
 }
 
 selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
-log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
 
 if [ -n "$PYROSCOPE_SERVER" ]; then
 	pyroscope_enabled=true
@@ -292,12 +291,6 @@ else
 	execution_layer="geth"
 fi
 
-if [ -n "$TEST_LOG_COLLECT" ]; then
-	test_log_collect=true
-else
-	test_log_collect=false
-fi
-
 cat << EOF > config.toml
 [Network]
 selected_networks=$selected_networks
@@ -312,13 +305,6 @@ server_url="$PYROSCOPE_SERVER"
 environment="$PYROSCOPE_ENVIRONMENT"
 key_secret="$PYROSCOPE_KEY"
 
-[Logging]
-test_log_collect=$test_log_collect
-run_id="$RUN_ID"
-
-[Logging.LogStream]
-log_targets=$log_targets
-
 [Logging.Loki]
 tenant_id="$LOKI_TENANT_ID"
 url="$LOKI_URL"

framework/.changeset/v0.3.2.md

@@ -0,0 +1,5 @@
+- Allow exposing CL node ports in host:docker format
+- Expose default test private keys as framework constants
+- Rename CHAINLINK_IMAGE to CTF_CHAINLINK_IMAGE to avoid CI collisions
+- Add CTF_JD_IMAGE env var
+- Add JobDistributor component

framework/.changeset/v0.3.3.md

@@ -0,0 +1,3 @@
+- Upgrade Besu to 24.9.1
+- Expose default test private keys as framework constants
+- Collect all the logs even between restarts

framework/.changeset/v0.3.4.md

@@ -0,0 +1 @@
+- Bump go-ethereum as in core, fix darwin amd/arm binary release