Skip to content

Commit

Permalink
Merge branch 'tt-1725-improve-go-doc-pipeline' into test_doc_generation
Browse files Browse the repository at this point in the history
  • Loading branch information
Tofel committed Dec 9, 2024
2 parents 02c7efc + 3e0c763 commit 9e69e13
Show file tree
Hide file tree
Showing 82 changed files with 1,751 additions and 4,527 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/framework-golden-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,6 @@ jobs:
env:
LOKI_TENANT_ID: promtail
LOKI_URL: http://localhost:3030/loki/api/v1/push
# this is not the best practice, and it must be fixed, run your tests WITHOUT IT!
# however, on current latest image we must use this flag
CTF_IGNORE_CRITICAL_LOGS: true
runs-on: ubuntu-latest
permissions:
id-token: write
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/framework.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ jobs:
defaults:
run:
working-directory: framework
env:
CTF_JD_IMAGE: ${{ secrets.CTF_JD_IMAGE }}
runs-on: ubuntu-latest
permissions:
id-token: write
Expand Down
104 changes: 60 additions & 44 deletions .github/workflows/generate-go-docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,6 @@ jobs:
env:
FILTERS: ${{ steps.changes.outputs.changes }}
run: |
# remove after test
exit 1
PATH=$PATH:$(go env GOPATH)/bin
export PATH
Expand Down Expand Up @@ -97,9 +95,7 @@ jobs:
cat filtered_folders.json | jq -c '.[]' | while read -r item; do
folder=$(echo "$item" | jq -r '.folder')
echo "Processing folder: $folder"
generate-go-function-docs diff -b ${{ github.event.pull_request.base.sha }} -c ${{ github.event.pull_request.head.sha }} --saveCosts --generator chatgpt --folder "$folder"
cd "$folder"
cd -
generate-go-function-docs diff -b "${{ github.event.pull_request.base.sha }}" -c "${{ github.event.pull_request.head.sha }}" --saveCosts --generator chatgpt --generatorSubType "${{ vars.GO_DOC_GEN_CHATGPT_MODEL }}" --folder "$folder"
done
rm filtered_folders.json
Expand All @@ -111,7 +107,7 @@ jobs:

- name: Remove costs before committing
shell: bash
run: rm -rf costs
run: rm -rf costs

- name: Setup GitHub Token for creating a new PR
id: setup-github-token-write
Expand All @@ -123,6 +119,7 @@ jobs:

- name: Create a new PR targeting current PR
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
id: create-pr
with:
token: ${{ steps.setup-github-token-write.outputs.access-token }}
branch: ${{ github.head_ref }}-docs
Expand All @@ -131,42 +128,61 @@ jobs:
body: "This PR contains automatically generated go documentation for the PR#${{ github.event.pull_request.number }}. Please review the changes."
commit-message: "[Bot] Add automatically generated go documentation"

- name: Send Slack notification
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
if: failure()
id: slack
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
- name: Find comment with PR link
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Go doc generation

- name: Create comment in the original PR
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
if: ${{ steps.fc.outputs.comment-id }} == ''

Check failure on line 141 in .github/workflows/generate-go-docs.yaml

View workflow job for this annotation

GitHub Actions / actionlint

[actionlint] .github/workflows/generate-go-docs.yaml#L141

if: condition "${{ steps.fc.outputs.comment-id }} == ''" is always evaluated to true because extra characters are around ${{ }} [if-cond]
Raw output
.github/workflows/generate-go-docs.yaml:141:13: if: condition "${{ steps.fc.outputs.comment-id }} == ''" is always evaluated to true because extra characters are around ${{ }} [if-cond]
with:
channel-id: 'C049X3353K2'
payload: |
{
"attachments": [
{
"color": "#C62828",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Go doc generation - Failed :x:"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<@U060CGGPY8H>, please have a look."
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}|PR#${{ github.event.pull_request.number }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
}
}
]
}
]
}
token: ${{ steps.setup-github-token-write.outputs.access-token }}
issue-number: ${{ github.event.pull_request.number }}
body: |
## Go doc generation
Hey @${{ github.actor }}, you can check generated Go function generation [here](${{ steps.create-pr.outputs.pull-request-url }}).
Please review them and merge to this PR once you're satisfied with them.
# - name: Send Slack notification
# uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
# if: failure()
# id: slack
# env:
# SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
# with:
# channel-id: 'C049X3353K2'
# payload: |
# {
# "attachments": [
# {
# "color": "#C62828",
# "blocks": [
# {
# "type": "section",
# "text": {
# "type": "mrkdwn",
# "text": "Go doc generation - Failed :x:"
# }
# },
# {
# "type": "section",
# "text": {
# "type": "mrkdwn",
# "text": "<@U060CGGPY8H>, please have a look."
# }
# },
# {
# "type": "section",
# "text": {
# "type": "mrkdwn",
# "text": "<${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}|PR#${{ github.event.pull_request.number }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
# }
# }
# ]
# }
# ]
# }
14 changes: 13 additions & 1 deletion book/src/framework/components/blockchains/evm.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,5 +72,17 @@ Public: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266
Private: ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
```

Test keys for `Besu` can be found [here](https://besu.hyperledger.org/23.4.1/private-networks/reference/accounts-for-testing)
For `Besu` keys are
```
Public: 0xfe3b557e8fb62b89f4916b721be55ceb828dbd73
Private: 0x8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63
Public: 0x627306090abaB3A6e1400e9345bC60c78a8BEf57
Private: 0xc87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3
Public: 0xf17f52151EbEF6C7334FAD080c5704D77216b732
Private: 0xae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f
```

More docs for `Besu` can be found [here](https://besu.hyperledger.org/private-networks/reference/accounts-for-testing)

5 changes: 3 additions & 2 deletions book/src/framework/components/state.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ One node set is enough for any kind of testing, if you need more nodes consider

## Custom ports

You can also define a custom set of ports for any node
You can also define a custom set of ports for any node.
```toml
[nodeset]
nodes = 5
Expand All @@ -53,6 +53,7 @@ You can also define a custom set of ports for any node

[nodeset.node_specs.node]
# here we defined 2 new ports to listen and mapped them to our host machine
custom_ports = [14000, 14001]
# syntax is "host:docker", if you provide only host port then we map 1-to-1
custom_ports = ["14000:15000", "20000"]
image = "public.ecr.aws/chainlink/chainlink:v2.16.0"
```
5 changes: 3 additions & 2 deletions book/src/framework/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@
| CTF_CONFIGS | Path(s) to test config files. <br/>Can be more than one, ex.: smoke.toml,smoke_1.toml,smoke_2.toml.<br/>First filepath will hold all the merged values | Any valid TOML file path | - ||
| CTF_LOG_LEVEL | Harness log level | `info`, `debug`, `trace` | `info` | 🚫 |
| CTF_PROMTAIL_DEBUG | Set `true` if you are integrating with remote `Loki` push API to debug Promtail | `true`, `false` | `false` | 🚫 |
| CTF_IGNORE_CRITICAL_LOGS | Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!) | `true`, `false` | `false` | 🚫 |
| CTF_CHAINLINK_IMAGE | Flag to override Chainlink Docker image in format $repository:$tag | $repository:$tag | - | 🚫 |
| CTF_JD_IMAGE | Job distributor service image in format $repository:$tag | $repository:$tag | - | 🚫 |
| LOKI_URL | URL to `Loki` push api, should be like`${host}/loki/api/v1/push` | URL | `http://host.docker.internal:3030/loki/api/v1/push` | 🚫 |
| LOKI_TENANT_ID | Streams all components logs to `Loki`, see params below | `string` | `promtail` | 🚫 |
| LOKI_BASIC_AUTH | Basic auth in format $user:$password | `$user:$password` | - | 🚫 |
| RESTY_DEBUG | Log all Resty client HTTP calls | `true`, `false` | `false` | 🚫 |
| CTF_IGNORE_CRITICAL_LOGS | Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!) | `true`, `false` | `false` | 🚫 |
| CTF_CHAINLINK_IMAGE | Flag to override Chainlink Docker image in format $repository:$tag | $repository:$tag | - | 🚫 |
117 changes: 9 additions & 108 deletions book/src/lib.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,38 +252,13 @@ Builder will read the location of chain configuration from env var named `PRIVAT

`net` is an instance of `blockchain.EVMNetwork`, which contains characteristics of the network and can be used to connect to it using an EVM client. `rpc` variable contains arrays of public and private RPC endpoints, where "private" means URL that's accessible from the same Docker network as the chain is running in.

# Using LogStream
## Logs
By default, we will save logs of all Docker containers running on the host machine, when the test ends (regardless whether it failed or succeeded). They will be available in the `./logs/<test-name><date>` directory. Same goes for dumping the databases of PostgresDBs
used by the Chainlink nodes. These will be saves in the `./db_dumps/<test-name><date>` directory.

LogStream is a package that allows to connect to a Docker container and then flush logs to configured targets. Currently 3 targets are supported:
## Loki and Grafana

- `file` - saves logs to a file in `./logs` folder
- `loki` - sends logs to Loki
- `in-memory` - stores logs in memory

It can be configured to use multiple targets at once. If no target is specified, it becomes a no-op.

LogStream has to be configured by passing an instance of `LoggingConfig` to the constructor.

When you connect a container LogStream will create a new consumer and start a detached goroutine that listens to logs emitted by that container and which reconnects and re-requests logs if listening fails for whatever reason. Retry limit and timeout can both be configured using functional options. In most cases one container should have one consumer, but it's possible to have multiple consumers for one container.

LogStream stores all logs in gob temporary file. To actually send/save them, you need to flush them. When you do it, LogStream will decode the file and send logs to configured targets. If log handling results in an error it won't be retried and processing of logs for given consumer will stop (if you think we should add a retry mechanism please let us know).

_Important:_ Flushing and accepting logs is blocking operation. That's because they both share the same cursor to temporary file and otherwise it's position would be racey and could result in mixed up logs.

## Configuration

Basic `LogStream` TOML configuration is following:

```toml
[LogStream]
log_targets=["file"]
log_producer_timeout="10s"
log_producer_retry_limit=10
```

You can find it here: [logging_default.toml](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/tomls/logging_default.toml)

When using `in-memory` or `file` target no other configuration variables are required. When using `loki` target, following ones must be set:
If you need to pass Loki or Grafana configuration to your tests you can do that by providing the following config:

```toml
[Logging.Loki]
Expand All @@ -295,87 +270,13 @@ bearer_token_secret="bearer-token"

Also, do remember that different URL should be used when running in CI and everywhere else. In CI it should be a public endpoint, while in local environment it should be a private one.

If your test has a Grafana dashboard in order for the url to be correctly printed you should provide the following config:
If your test has a Grafana dashboard you should provide the following config:

```toml
[Logging.Grafana]
url="http://grafana.somwhere.com/my_dashboard"
```

## Initialisation

First you need to create a new instance:

```golang
// t - instance of *testing.T (can be nil)
// testConfig.Logging - pointer to logging part of TestConfig
ls := logstream.NewLogStream(t, testConfig.Logging)
```

## Listening to logs

If using `testcontainers-go` Docker containers it is recommended to use life cycle hooks for connecting and disconnecting LogStream from the container. You can do that when creating `ContainerRequest` in the following way:

```golang
containerRequest := &tc.ContainerRequest{
LifecycleHooks: []tc.ContainerLifecycleHooks{
{PostStarts: []tc.ContainerHook{
func(ctx context.Context, c tc.Container) error {
if ls != nil {
return n.ls.ConnectContainer(ctx, c, "custom-container-prefix-can-be-empty")
}
return nil
},
},
PostStops: []tc.ContainerHook{
func(ctx context.Context, c tc.Container) error {
if ls != nil {
return n.ls.DisconnectContainer(c)
}
return nil
},
}},
},
}
```

You can print log location for each target using this function: `(m *LogStream) PrintLogTargetsLocations()`. For `file` target it will print relative folder path, for `loki` it will print URL of a Grafana Dashboard scoped to current execution and container ids. For `in-memory` target it's no-op.

It is recommended to shutdown LogStream at the end of your tests. Here's an example:

```golang
t.Cleanup(func() {
l.Warn().Msg("Shutting down Log Stream")

if t.Failed() || os.Getenv("TEST_LOG_COLLECT") == "true" {
// we can't do much if this fails, so we just log the error
_ = logStream.FlushLogsToTargets()
// this will log log locations for each target (for file it will be a folder, for Loki Grafana dashboard -- remember to provide it's url in config!)
logStream.PrintLogTargetsLocations()
// this will save log locations in test summary, so that they can be easily accessed in GH's step summary
logStream.SaveLogLocationInTestSummary()
}

// we can't do much if this fails, so we just log the error
_ = logStream.Shutdown(testcontext.Get(b.t))
})
```

or in a bit shorter way:

```golang
t.Cleanup(func() {
l.Warn().Msg("Shutting down Log Stream")

if t.Failed() || os.Getenv("TEST_LOG_COLLECT") == "true" {
// this will log log locations for each target (for file it will be a folder, for Loki Grafana dashboard -- remember to provide it's url in config!)
logStream.PrintLogTargetsLocations()
// this will save log locations in test summary, so that they can be easily accessed in GH's step summary
}

// we can't do much if this fails
_ = logStream.FlushAndShutdown()
})
base_url="https://your-grafana-url"
dashboard_url="/my-dashboard"
dashboard_uid="my-dashboard-uid" # optional
```

## Grouping test execution
Expand Down
14 changes: 0 additions & 14 deletions book/src/lib/config/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,6 @@ convert_to_toml_array() {
}
selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
if [ -n "$PYROSCOPE_SERVER" ]; then
pyroscope_enabled=true
Expand All @@ -292,12 +291,6 @@ else
execution_layer="geth"
fi
if [ -n "$TEST_LOG_COLLECT" ]; then
test_log_collect=true
else
test_log_collect=false
fi
cat << EOF > config.toml
[Network]
selected_networks=$selected_networks
Expand All @@ -312,13 +305,6 @@ server_url="$PYROSCOPE_SERVER"
environment="$PYROSCOPE_ENVIRONMENT"
key_secret="$PYROSCOPE_KEY"
[Logging]
test_log_collect=$test_log_collect
run_id="$RUN_ID"
[Logging.LogStream]
log_targets=$log_targets
[Logging.Loki]
tenant_id="$LOKI_TENANT_ID"
url="$LOKI_URL"
Expand Down
5 changes: 5 additions & 0 deletions framework/.changeset/v0.3.2.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- Allow exposing CL node ports in host:docker format
- Expose default test private keys as framework constants
- Rename CHAINLINK_IMAGE to CTF_CHAINLINK_IMAGE to avoid CI collisions
- Add CTF_JD_IMAGE env var
- Add JobDistributor component
3 changes: 3 additions & 0 deletions framework/.changeset/v0.3.3.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- Upgrade Besu to 24.9.1
- Expose default test private keys as framework constants
- Collect all the logs even between restarts
1 change: 1 addition & 0 deletions framework/.changeset/v0.3.4.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Bump go-ethereum as in core, fix darwin amd/arm binary release
Loading

0 comments on commit 9e69e13

Please sign in to comment.