Skip to content

Commit

Permalink
e2e test workflow added
Browse files Browse the repository at this point in the history
  • Loading branch information
ocdbytes committed Aug 29, 2024
1 parent bf4350e commit e4cc75c
Show file tree
Hide file tree
Showing 22 changed files with 253 additions and 3,782 deletions.
56 changes: 28 additions & 28 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,50 +1,50 @@
##### ORCHESTRATOR #####

HOST=
PORT=
DATABASE_URL=
MADARA_RPC_URL=
DA_LAYER=
SETTLEMENT_LAYER=

# Ethereum
ETHEREUM_PRIVATE_KEY=
ETHEREUM_RPC_URL=
MEMORY_PAGES_CONTRACT_ADDRESS=
STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
##### AWS CONFIG #####

AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
AWS_ENDPOINT_URL=

# Starknet
STARKNET_PUBLIC_KEY=
STARNET_PRIVATE_KEY=
STARKNET_RPC_URL=
STARKNET_CAIRO_CORE_CONTRACT_ADDRESS=
##### STORAGE #####

# MongoDB connection string
MONGODB_CONNECTION_STRING=
DATA_STORAGE=
AWS_S3_BUCKET_NAME=

# AWS
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_DEFAULT_REGION=
##### QUEUE #####

# SQS
QUEUE_PROVIDER=
SQS_JOB_PROCESSING_QUEUE_URL=
SQS_JOB_VERIFICATION_QUEUE_URL=
SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
SQS_WORKER_TRIGGER_QUEUE_URL=

# S3
AWS_S3_BUCKET_NAME=
AWS_S3_BUCKET_REGION=
##### DATABASE #####

DATABASE=
MONGODB_CONNECTION_STRING=

# Ethereum Settlement
DEFAULT_SETTLEMENT_CLIENT_RPC=
DEFAULT_L1_CORE_CONTRACT_ADDRESS=
##### PROVER #####

# Sharp Services
PROVER_SERVICE=
SHARP_CUSTOMER_ID=
SHARP_URL=
SHARP_USER_CRT=
SHARP_USER_KEY=
SHARP_SERVER_CRT=
SHARP_PROOF_LAYOUT=

##### ON CHAIN CONFIG #####

DA_LAYER=
SETTLEMENT_LAYER=
ETHEREUM_RPC_URL=
MADARA_RPC_URL=
MEMORY_PAGES_CONTRACT_ADDRESS=
PRIVATE_KEY=
ETHEREUM_PRIVATE_KEY=
STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
71 changes: 42 additions & 29 deletions .env.test
Original file line number Diff line number Diff line change
@@ -1,42 +1,40 @@
##### AWS config #####
##### ORCHESTRATOR #####

HOST=127.0.0.1
PORT=3000

##### AWS CONFIG #####

AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID"
AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"
AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
AWS_S3_BUCKET_REGION="us-east-1"
AWS_REGION="us-east-1"
AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"
SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue"
SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue"
SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
SQS_WORKER_TRIGGER_QUEUE_URL=
AWS_DEFAULT_REGION="localhost"

##### On chain config #####
##### STORAGE #####

MADARA_RPC_URL="http://localhost:3000"
ETHEREUM_RPC_URL="http://localhost:3001"
ETHEREUM_MAINNET_RPC_URL="https://mainnet.infura.io/v3/bf9e41563a6a45e28eb60382d85ef3c9" # for forking during tests
MEMORY_PAGES_CONTRACT_ADDRESS="0x47312450B3Ac8b5b8e247a6bB6d523e7605bDb60"
PRIVATE_KEY="0xdead"
# Private key of Test wallet provided by Anvil
ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"
DATA_STORAGE="s3"
AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"

##### Config URLs #####
##### QUEUE #####

DA_LAYER="ethereum"
PROVER_SERVICE="sharp"
SETTLEMENT_LAYER="ethereum"
DATA_STORAGE="s3"
QUEUE_PROVIDER="sqs"
SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue"
SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue"
SQS_JOB_HANDLE_FAILURE_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_handle_failure_queue"
SQS_WORKER_TRIGGER_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_worker_trigger_queue"

##### SNS #####
ALERTS="sns"

##### DATABASE #####

DATABASE="mongodb"
MONGODB_CONNECTION_STRING="mongodb://localhost:27017"
DEFAULT_SETTLEMENT_CLIENT_RPC="http://localhost:3000"

# Ethereum Settlement
DEFAULT_L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
SHOULD_IMPERSONATE_ACCOUNT="true"
TEST_DUMMY_CONTRACT_ADDRESS="0xE5b6F5e695BA6E4aeD92B68c4CC8Df1160D69A81"
##### PROVER #####

# Sharp Services
PROVER_SERVICE="sharp"
SHARP_CUSTOMER_ID="sharp_consumer_id"
SHARP_URL="http://127.0.0.1:5000"
# [IMP!!!] These are test certificates (they don't work)
Expand All @@ -45,6 +43,21 @@ SHARP_USER_KEY="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lH
SHARP_SERVER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQWxPZ0F3SUJBZ0lVRUR0Rjd0YVNTUnVQQTJ6Uk1aNWNzY2JCRm5jd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBNE1UTXhNekEzTVROYUZ3MHlOVEE0Ck1UTXhNekEzTVROYU1FVXhDekFKQmdOVkJBWVRBa2xPTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFRHcEEwNEZ1QlNFaE5PNVYvMGxTaDkvSEgxeVRZT2dRVFdoOG43eDlRCnZGMHpvZFZueVFIdjE5elU5eVdia2xvOEkvOXFBVm9lRzdXTnpUVFg2Q295ZlNjb1YvazN0Q2UwVnVWMlFJTVQKdW82SzJSU05CVHB1TlNqNTlzUiszVTQ2OFRBQnY0YVpsYjU4TU5CRXM3MVRieVpLRHBGRVRkMkg3T0ZKajg4QQpNRi9MaXJkeDZPOFdZL0tDeisxd1ZXL1JRdytYYjRJSWx4bXJFOC9UZ3FNSEo4dFUxYkZiOWJNcTEvOTN5YWtJClU1V2J2NVhXKzFwZFVyTUFNcTFFaC9vZThMN2pFaFdvZXZrNzgyU0kwUk0xeG5MaEtrUUVBYXd6Zkg2ODZiR2YKUHQ3RkFIQ1pGaWJ4KzZzSkg0R1M3S25iK0x5bk9ud3phMWZPUXZEZmcvRm5BZ01CQUFHalV6QlJNQjBHQTFVZApEZ1FXQkJUYlFUdmlUTW1xNXlNK2ZJRVI4VjdTZk1pK3B6QWZCZ05WSFNNRUdEQVdnQlRiUVR2aVRNbXE1eU0rCmZJRVI4VjdTZk1pK3B6QVBCZ05WSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRREYKTllyRnpBa2RIVkhjUkd5SUNsTi9IVGswaldOcTVSdTB1RUpDQ21Dbm9ZY1pRSTlDQlcwTkl3dGpZUkpTKzR1UwordWh4VWpSYTA5YXdOWDhvYmU0dDZjK25HRnhZMGZqamk0cGZnbU1kMWNJeGdsM3E3Nlp0ZkllRGR6alRLRXN1CjRFUTVadnEwMnJvTEZ0ZjEvL3dRVG0xNkNKdFpGWnhNZ1phYnNxc2JRc3M2dWdMUGtTTmdBWjI1L2VhcWhnQ20KTjFUV2FxL0xJMVBLSkxPK085NFlMa2FsNVpyOTJCOXk4Q0VKVUVuSTA1R1N1MmJUOFM2a0ZBMEpadEszTW9SbwpqRWZWV1lQVHR5TFR4amNvRndCcDlHaXZYSDdSdHBxMDlmSmFhU1pNekxmNGlyNHpBdXprbExBNWZvampPNXlKCllnYlVaQUU2aS81N1NFWjR3VmxTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
SHARP_PROOF_LAYOUT="small"

# Test vars
# used when setting up the environment for e2e test
##### ON CHAIN CONFIG #####

DA_LAYER="ethereum"
SETTLEMENT_LAYER="ethereum_test"
SETTLEMENT_RPC_URL="http://localhost:3001"
MADARA_RPC_URL="http://localhost:3000"
L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
MEMORY_PAGES_CONTRACT_ADDRESS="0x47312450B3Ac8b5b8e247a6bB6d523e7605bDb60"
PRIVATE_KEY="0xdead"
ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
DEFAULT_L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
TEST_DUMMY_CONTRACT_ADDRESS="0xE5b6F5e695BA6E4aeD92B68c4CC8Df1160D69A81"
STARKNET_OPERATOR_ADDRESS="0x2C169DFe5fBbA12957Bdd0Ba47d9CEDbFE260CA7"

##### E2E test vars #####

L2_BLOCK_NUMBER_FOR_TEST=671070
71 changes: 71 additions & 0 deletions .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: Rust Test & Coverage

on:
pull_request_target:
branches:
- main
types: [opened, synchronize, reopened]
push:
branches-ignore:
- main
workflow_call:
workflow_dispatch:

jobs:
coverage:
runs-on: ubuntu-latest

services:
localstack:
image: localstack/localstack
env:
SERVICES: s3, sqs
DEFAULT_REGION: us-east-1
AWS_ACCESS_KEY_ID: "AWS_ACCESS_KEY_ID"
AWS_SECRET_ACCESS_KEY: "AWS_SECRET_ACCESS_KEY"
ports:
- 4566:4566
mongodb:
image: mongo:latest
ports:
- 27017:27017

steps:
- uses: actions/checkout@v3

# selecting a toolchain either by action or manual `rustup` calls should happen
# before the plugin, as the cache uses the current rustc version as its cache key
- run: rustup show

- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true

- name: Rust Cache
uses: Swatinem/rust-cache@v2

- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1

- name: Check Anvil Installation
run: |
if command -v anvil &> /dev/null
then
echo "Anvil is installed. Version information:"
anvil --version
else
echo "Anvil is not installed or not in PATH"
exit 1
fi
- name: Running anvil
run: |
# Forking block number : 20607627 on ethereum mainnet
anvil --fork-url ${{ secrets.ETHEREUM_BLAST_RPC_URL }}@20607627
- name: Run e2e test
run: |
RUST_LOG=trace cargo test test_orchestrator_workflow -- --ignored --nocapture
10 changes: 7 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,19 @@ authors = ["Apoorv Sadana <@apoorvsadana>"]
[workspace.dependencies]
num = { version = "0.4.1" }
async-trait = { version = "0.1.77" }
alloy = { version = "0.2.1", features = ["full",
alloy = { version = "0.2.1", features = [
"full",
"node-bindings",
"rpc-types-debug",
"rpc-types-trace",
"json-rpc",
"rpc-client",] }
"rpc-client",
] }
aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
aws-sdk-s3 = { version = "1.38.0", features = ["behavior-version-latest"] }
aws-sdk-eventbridge = { version = "1.41.0", features = ["behavior-version-latest"] }
aws-sdk-eventbridge = { version = "1.41.0", features = [
"behavior-version-latest",
] }
aws-sdk-sqs = "1.36.0"
axum = { version = "0.7.4" }
axum-macros = "0.4.1"
Expand Down
3 changes: 1 addition & 2 deletions crates/orchestrator/src/jobs/da_job/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ use std::ops::{Add, Mul, Rem};
use std::str::FromStr;

use async_trait::async_trait;
use bytes::Buf;
use color_eyre::eyre::WrapErr;
use lazy_static::lazy_static;
use num_bigint::{BigUint, ToBigUint};
Expand Down Expand Up @@ -206,7 +205,7 @@ pub fn convert_to_biguint(elements: Vec<FieldElement>) -> Vec<BigUint> {
pub fn data_to_blobs(blob_size: u64, block_data: Vec<BigUint>) -> Result<Vec<Vec<u8>>, JobError> {
// Validate blob size
if blob_size < 32 {
return Err(DaError::InsufficientBlobSize { blob_size })?;
Err(DaError::InsufficientBlobSize { blob_size })?
}

let mut blobs: Vec<Vec<u8>> = Vec::new();
Expand Down
5 changes: 2 additions & 3 deletions crates/orchestrator/src/jobs/state_update_job/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -146,16 +146,15 @@ impl Job for StateUpdateJob {
/// 1. the last settlement tx hash is successful,
/// 2. the expected last settled block from our configuration is indeed the one found in the provider.
async fn verify_job(&self, config: &Config, job: &mut JobItem) -> Result<JobVerificationStatus, JobError> {
log::info!("job item : {:?}", job);

let attempt_no = job
.metadata
.get(JOB_PROCESS_ATTEMPT_METADATA_KEY)
.ok_or_else(|| StateUpdateError::AttemptNumberNotFound)?;

// TODO : fix this after test
let metadata_tx_hashes = job
.metadata
.get(&format!("{}{}", JOB_METADATA_STATE_UPDATE_ATTEMPT_PREFIX, attempt_no))
.get(&format!("{}{}", JOB_METADATA_STATE_UPDATE_ATTEMPT_PREFIX, attempt_no.parse::<u32>().unwrap() - 1))
.ok_or_else(|| StateUpdateError::TxnHashMetadataNotFound)?
.clone()
.replace(' ', "");
Expand Down
Loading

0 comments on commit e4cc75c

Please sign in to comment.