diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42b4067..1dffbcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# This file was autogenerated by cargo-dist: https://opensource.axo.dev/cargo-dist/ +# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/ # # Copyright 2022-2024, axodotdev # SPDX-License-Identifier: MIT or Apache-2.0 @@ -6,7 +6,7 @@ # CI that: # # * checks for a Git Tag that looks like a release -# * builds artifacts with cargo-dist (archives, installers, hashes) +# * builds artifacts with dist (archives, installers, hashes) # * uploads those artifacts to temporary workflow zip # * on success, uploads the artifacts to a GitHub Release # @@ -24,10 +24,10 @@ permissions: # must be a Cargo-style SemVer Version (must have at least major.minor.patch). # # If PACKAGE_NAME is specified, then the announcement will be for that -# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# package (erroring out if it doesn't have the given version or isn't dist-able). # # If PACKAGE_NAME isn't specified, then the announcement will be for all -# (cargo-dist-able) packages in the workspace with that version (this mode is +# (dist-able) packages in the workspace with that version (this mode is # intended for workspaces with only one dist-able package, or with all dist-able # packages versioned/released in lockstep). # @@ -45,7 +45,7 @@ on: - '**[0-9]+.[0-9]+.[0-9]+*' jobs: - # Run 'cargo dist plan' (or host) to determine what tasks we need to do + # Run 'dist plan' (or host) to determine what tasks we need to do plan: runs-on: "ubuntu-20.04" outputs: @@ -59,16 +59,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cargo-dist + - name: Install dist # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.23.0/cargo-dist-installer.sh | sh" - - name: Cache cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.27.0/cargo-dist-installer.sh | sh" + - name: Cache dist uses: actions/upload-artifact@v4 with: name: cargo-dist-cache - path: ~/.cargo/bin/cargo-dist + path: ~/.cargo/bin/dist # sure would be cool if github gave us proper conditionals... # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible # functionality based on whether this is a pull_request, and whether it's from a fork. @@ -76,8 +76,8 @@ jobs: # but also really annoying to build CI around when it needs secrets to work right.) - id: plan run: | - cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json - echo "cargo dist ran successfully" + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" cat plan-dist-manifest.json echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" - name: "Upload dist-manifest.json" @@ -95,18 +95,19 @@ jobs: if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} strategy: fail-fast: false - # Target platforms/runners are computed by cargo-dist in create-release. + # Target platforms/runners are computed by dist in create-release. # Each member of the matrix has the following arguments: # # - runner: the github runner - # - dist-args: cli flags to pass to cargo dist - # - install-dist: expression to run to install cargo-dist on the runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner # # Typically there will be: # - 1 "global" task that builds universal installers # - N "local" tasks that build each platform's binaries and platform-specific installers matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json @@ -117,8 +118,15 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cargo-dist - run: ${{ matrix.install_dist }} + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install dist + run: ${{ matrix.install_dist.run }} # Get the dist-manifest - name: Fetch local artifacts uses: actions/download-artifact@v4 @@ -132,8 +140,8 @@ jobs: - name: Build artifacts run: | # Actually do builds and make zips and whatnot - cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json - echo "cargo dist ran successfully" + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" - id: cargo-dist name: Post-build # We force bash here just because github makes it really hard to get values up @@ -143,7 +151,7 @@ jobs: run: | # Parse out what we just built and upload it to scratch storage echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" cp dist-manifest.json "$BUILD_MANIFEST_NAME" @@ -168,12 +176,12 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cached cargo-dist + - name: Install cached dist uses: actions/download-artifact@v4 with: name: cargo-dist-cache path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/cargo-dist + - run: chmod +x ~/.cargo/bin/dist # Get all the local artifacts for the global tasks to use (for e.g. checksums) - name: Fetch local artifacts uses: actions/download-artifact@v4 @@ -184,8 +192,8 @@ jobs: - id: cargo-dist shell: bash run: | - cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json - echo "cargo dist ran successfully" + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" # Parse out what we just built and upload it to scratch storage echo "paths<> "$GITHUB_OUTPUT" @@ -217,12 +225,12 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cached cargo-dist + - name: Install cached dist uses: actions/download-artifact@v4 with: name: cargo-dist-cache path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/cargo-dist + - run: chmod +x ~/.cargo/bin/dist # Fetch artifacts from scratch-storage - name: Fetch artifacts uses: actions/download-artifact@v4 @@ -233,7 +241,7 @@ jobs: - id: host shell: bash run: | - cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json echo "artifacts uploaded and released successfully" cat dist-manifest.json echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" diff --git a/Cargo.lock b/Cargo.lock index 51d749e..0b54433 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,7 +403,7 @@ dependencies = [ [[package]] name = "doppler" -version = "0.3.7" +version = "0.4.0" dependencies = [ "anyhow", "base64", diff --git a/dist-workspace.toml b/dist-workspace.toml index 87c0ced..2af8a56 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -1,10 +1,10 @@ [workspace] members = ["npm:doppler_ui", "cargo:doppler"] -# Config for 'cargo dist' +# Config for 'dist' [dist] -# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.23.0" +# The preferred dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.27.0" # CI backends to support ci = "github" # Target platforms to build apps for (Rust target-triple syntax) diff --git a/doppler/Cargo.toml b/doppler/Cargo.toml index 5d2e080..cf35675 100644 --- a/doppler/Cargo.toml +++ b/doppler/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "doppler" -version = "0.3.7" +version = "0.4.0" repository = "https://github.com/tee8z/doppler.git" [package.metadata.dist] diff --git a/doppler/config/regtest/bitcoin.conf b/doppler/config/regtest/bitcoin.conf index ad8dce6..974514f 100755 --- a/doppler/config/regtest/bitcoin.conf +++ b/doppler/config/regtest/bitcoin.conf @@ -14,4 +14,4 @@ fallbackfee=0.00001 blockfilterindex=1 peerblockfilters=1 listen=1 -rpcthreads=25 \ No newline at end of file +rpcthreads=25 diff --git a/doppler/config/signet/bitcoin.conf b/doppler/config/signet/bitcoin.conf index 2a55581..ec0f72d 100644 --- a/doppler/config/signet/bitcoin.conf +++ b/doppler/config/signet/bitcoin.conf @@ -17,4 +17,4 @@ fallbackfee=0.00001 blockfilterindex=1 peerblockfilters=1 listen=1 -rpcthreads=256 \ No newline at end of file +rpcthreads=256 diff --git a/doppler/src/bitcoind.rs b/doppler/src/bitcoind.rs index ebf3048..f7d5666 100644 --- a/doppler/src/bitcoind.rs +++ b/doppler/src/bitcoind.rs @@ -3,7 +3,9 @@ use crate::{ }; use anyhow::{anyhow, Error, Result}; use conf_parser::processer::{FileConf, Section}; -use docker_compose_types::{EnvFile, Networks, Ports, Service, Volumes}; +use docker_compose_types::{ + EnvFile, Healthcheck, HealthcheckTest, Networks, Ports, Service, Volumes, +}; use log::{error, info}; use std::{ fs::{File, OpenOptions}, @@ -24,6 +26,9 @@ pub struct Bitcoind { pub zmqpubhashblock: String, pub zmqpubrawtx: String, pub path_vol: String, + // Only added when initially creating the docker compose + pub public_p2p: Option, + pub public_rpc: Option, } pub enum L1Enum { @@ -113,25 +118,48 @@ pub fn build_bitcoind( image: &ImageInfo, is_miner: bool, ) -> Result<()> { - let bitcoind_conf = get_config(options, name, is_miner).unwrap(); + let mut bitcoind_conf = get_config(options, name, is_miner).unwrap(); + let public_p2p = options.new_port(); + let public_rpc = options.new_port(); + let bitcoind = Service { image: Some(image.get_image()), container_name: Some(bitcoind_conf.container_name.clone()), ports: Ports::Short(vec![ - format!("{}:{}", options.new_port(), bitcoind_conf.p2pport), - format!("{}:{}", options.new_port(), bitcoind_conf.rpcport), + format!("{}:{}", public_p2p, bitcoind_conf.p2pport), + format!("{}:{}", public_rpc, bitcoind_conf.rpcport), ]), volumes: Volumes::Simple(vec![format!( "{}:/home/bitcoin/.bitcoin:rw", bitcoind_conf.path_vol )]), env_file: Some(EnvFile::Simple(".env".to_owned())), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single( + [ + "bitcoin-cli".to_string(), + format!("-rpcuser={}", bitcoind_conf.user), + format!("-rpcpassword={}", bitcoind_conf.password), + format!("-rpcport={}", bitcoind_conf.rpcport), + "getblockchaininfo".to_string(), + ] + .join(" "), + )), + interval: Some("30s".to_string()), + timeout: Some("10s".to_string()), + retries: 3, + disable: false, + start_period: Some("40s".to_string()), + }), networks: Networks::Simple(vec![NETWORK.to_owned()]), ..Default::default() }; options .services .insert(bitcoind_conf.container_name.clone(), Some(bitcoind)); + + bitcoind_conf.public_p2p = Some(public_p2p); + bitcoind_conf.public_rpc = Some(public_rpc); options.bitcoinds.push(bitcoind_conf); Ok(()) } @@ -180,6 +208,8 @@ fn load_config(name: &str, container_name: &str, network: &str) -> Result Result { - if conf.sections.get(&options.network).is_none() { + if !conf.sections.contains_key(&options.network) { conf.sections .insert(options.network.clone(), Section::new()); } let bitcoin = conf.sections.get_mut(&options.network).unwrap(); - let port = options.new_port(); - let rpc_port = options.new_port(); + let port = match options.network.as_ref() { + "signet" => "38333", + _ => "18444", + }; + let rpc_port = match options.network.as_ref() { + "signet" => "38332", + _ => "18443", + }; + bitcoin.set_property("bind", "0.0.0.0"); - bitcoin.set_property("port", &port.to_string()); - bitcoin.set_property("rpcport", &rpc_port.to_string()); - bitcoin.set_property("rpcuser", "admin"); - bitcoin.set_property("rpcpassword", "1234"); - bitcoin.set_property( - "zmqpubrawblock", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); - bitcoin.set_property( - "zmqpubrawtx", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); - bitcoin.set_property( - "zmqpubhashblock", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); + bitcoin.set_property("port", port); + bitcoin.set_property("rpcport", rpc_port); + bitcoin.set_property("rpcuser", "bitcoin"); + bitcoin.set_property("rpcpassword", "bitcoin"); + bitcoin.set_property("zmqpubrawblock", "tcp://0.0.0.0:28332"); + bitcoin.set_property("zmqpubrawtx", "tcp://0.0.0.0:28333"); + bitcoin.set_property("zmqpubhashtx", "tcp://0.0.0.0:28332"); + bitcoin.set_property("zmqpubhashblock", "tcp://0.0.0.0:28332"); let network_section = get_network_section(conf, &options.network)?; Ok(network_section) } diff --git a/doppler/src/cln.rs b/doppler/src/cln.rs index dfe5e98..9ec2339 100644 --- a/doppler/src/cln.rs +++ b/doppler/src/cln.rs @@ -50,7 +50,7 @@ impl Cln { ) -> Result { get_peers_short_channel_id(self, options, node_command, "source") } - + pub fn add_rune(&mut self, options: &Options) -> Result<(), Error> { let rune = get_rune(self, options)?; self.rune = Some(rune); diff --git a/doppler/src/conf_handler.rs b/doppler/src/conf_handler.rs index 117b133..3e1131c 100644 --- a/doppler/src/conf_handler.rs +++ b/doppler/src/conf_handler.rs @@ -19,8 +19,9 @@ use std::{ use crate::{ add_bitcoinds, add_coreln_nodes, add_eclair_nodes, add_external_lnd_nodes, add_lnd_nodes, - get_latest_polar_images, get_polar_images, new, update_bash_alias_external, Bitcoind, Cln, - CloneableHashMap, Eclair, ImageInfo, L1Node, L2Node, Lnd, NodeKind, Tag, Tags, NETWORK, + get_latest_polar_images, get_polar_images, get_supported_tool_images, new, + update_bash_alias_external, Bitcoind, Cln, CloneableHashMap, Eclair, Esplora, ImageInfo, + L1Node, L2Node, Lnd, NodeKind, SupportedTool, Tag, Tags, ToolImageInfo, NETWORK, }; #[derive(Subcommand)] @@ -61,9 +62,11 @@ impl std::fmt::Display for ShellType { #[derive(Clone)] pub struct Options { default_images: CloneableHashMap, + default_tool_images: CloneableHashMap, known_polar_images: CloneableHashMap>, pub images: Vec, pub bitcoinds: Vec, + pub esplora: Vec, pub lnd_nodes: Vec, pub eclair_nodes: Vec, pub cln_nodes: Vec, @@ -152,14 +155,17 @@ impl Options { if external_nodes_path.is_some() { rest = true; } + let default_tool_images = get_supported_tool_images(); Self { default_images: latest_polar_images, + default_tool_images, known_polar_images: all_polar_images, images: vec::Vec::new(), bitcoinds: vec::Vec::new(), lnd_nodes: vec::Vec::new(), eclair_nodes: vec::Vec::new(), cln_nodes: vec::Vec::new(), + esplora: vec::Vec::new(), ports: starting_port, compose_path: None, services: indexmap::IndexMap::new(), @@ -172,19 +178,21 @@ impl Options { loop_count: Arc::new(AtomicI64::new(0)), read_end_of_doppler_file: Arc::new(AtomicBool::new(true)), tags: Arc::new(Mutex::new(new(connection))), - rest: rest, - external_nodes_path: external_nodes_path, + rest, + external_nodes_path, external_nodes: None, ui_config_path, network, } } + pub fn get_image(&self, name: &str) -> Option { self.images .iter() .find(|image| image.is_image(name)) .map(|image| image.clone()) } + pub fn get_default_image(&self, node_kind: NodeKind) -> ImageInfo { match self.default_images.get(node_kind) { Some(image) => image, @@ -192,6 +200,15 @@ impl Options { } .clone() } + + pub fn get_default_tool_image(&self, tool: SupportedTool) -> ToolImageInfo { + match self.default_tool_images.get(tool) { + Some(image) => image, + None => panic!("error no default images found!"), + } + .clone() + } + pub fn add_thread(&self, thread_handler: Thread) { self.thread_handlers.lock().unwrap().push(thread_handler); } diff --git a/doppler/src/docker.rs b/doppler/src/docker.rs index 5b5a5df..a850b61 100644 --- a/doppler/src/docker.rs +++ b/doppler/src/docker.rs @@ -1,4 +1,3 @@ -extern crate ini; use crate::{ create_ui_config_files, get_absolute_path, pair_bitcoinds, L1Node, L2Node, NodeCommand, Options, }; @@ -26,7 +25,7 @@ pub fn load_options_from_external_nodes( debug!("loaded lnds"); let network = options.external_nodes.clone().unwrap()[0].network.clone(); - create_ui_config_files(&options, &network)?; + create_ui_config_files(options, &network)?; Ok(()) } diff --git a/doppler/src/lib.rs b/doppler/src/lib.rs index 6c8af0f..4ea1181 100644 --- a/doppler/src/lib.rs +++ b/doppler/src/lib.rs @@ -10,6 +10,7 @@ mod node_kind; mod parser; mod polar_default_images; mod simple_storage; +mod tools; mod visualizer; mod workflow; @@ -25,5 +26,6 @@ pub use node_kind::*; pub use parser::*; pub use polar_default_images::*; pub use simple_storage::*; +pub use tools::*; pub use visualizer::*; pub use workflow::*; diff --git a/doppler/src/node.rs b/doppler/src/node.rs index 134b063..c6e5ccb 100644 --- a/doppler/src/node.rs +++ b/doppler/src/node.rs @@ -196,6 +196,7 @@ impl ImageInfo { pub fn is_image(&self, name: &str) -> bool { self.name == name } + pub fn get_name(&self) -> String { self.name.clone() } diff --git a/doppler/src/node_kind.rs b/doppler/src/node_kind.rs index 27642fc..b81e508 100644 --- a/doppler/src/node_kind.rs +++ b/doppler/src/node_kind.rs @@ -13,7 +13,6 @@ pub enum NodeKind { Coreln, Eclair, } - impl<'a> TryFrom> for NodeKind { type Error = anyhow::Error; @@ -31,3 +30,61 @@ impl<'a> TryFrom> for NodeKind { } } } + +impl From for NodeKind { + fn from(value: LnNodeKind) -> NodeKind { + match value { + LnNodeKind::Lnd => NodeKind::Lnd, + LnNodeKind::Coreln => NodeKind::Coreln, + LnNodeKind::Eclair => NodeKind::Eclair, + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum BtcNodeKind { + Bitcoind, + #[default] + BitcoindMiner, +} + +impl<'a> TryFrom> for BtcNodeKind { + type Error = anyhow::Error; + + fn try_from(value: Pair) -> Result { + match value.as_rule() { + Rule::node_kind => match value.as_str() { + "BITCOIND" => Ok(BtcNodeKind::Bitcoind), + "BITCOIND_MINER" => Ok(BtcNodeKind::BitcoindMiner), + _ => bail!("invalid btc_node_kind"), + }, + _ => bail!("pair should be a btc_node_kind"), + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum LnNodeKind { + #[default] + Lnd, + Coreln, + Eclair, +} + +impl<'a> TryFrom> for LnNodeKind { + type Error = anyhow::Error; // or whatever error type you're using + + fn try_from(pair: Pair) -> Result { + match pair.as_rule() { + Rule::ln_node_kind => match pair.as_str() { + "LND" => Ok(LnNodeKind::Lnd), + "ECLAIR" => Ok(LnNodeKind::Eclair), + "CORELN" => Ok(LnNodeKind::Coreln), + _ => Err(anyhow::anyhow!("Unknown ln node kind")), + }, + _ => Err(anyhow::anyhow!( + "invalid ln node kind: pair should be a ln_node_kind" + )), + } + } +} diff --git a/doppler/src/parser/grammar.pest b/doppler/src/parser/grammar.pest index c19f155..dba0693 100644 --- a/doppler/src/parser/grammar.pest +++ b/doppler/src/parser/grammar.pest @@ -9,19 +9,30 @@ num = @{ ASCII_DIGIT+ } time_digits = { "s" | "m" | "h" } -node_kind = { "BITCOIND_MINER" | "BITCOIND" | "LND" | "CORELN" | "ECLAIR" } + +btc_node_kind = { "BITCOIND_MINER" | "BITCOIND" } +ln_node_kind = { "LND" | "CORELN" | "ECLAIR" } +node_kind = { btc_node_kind | ln_node_kind } + +tool_kind = { "TOOL" } image_name = @{ ASCII_ALPHANUMERIC+ } image_version = @{ (ASCII_ALPHANUMERIC | PUNCTUATION)+ } +supported_tool = { "ESPLORA" } + +tool_def = { "TOOL" ~ supported_tool ~ ident ~ "FOR" ~ ident } + node_def = { (node_kind ~ ident ~ image_name) | (node_kind ~ ident ) } node_image = { node_kind ~ "IMAGE" ~ image_name ~ image_version } -node_pair = { node_kind ~ (( ident ~ "PAIR") | ( ident ~ image_name ~ "PAIR")) ~ (ident ~ num | ident) } +node_pair = { ln_node_kind ~ (( ident ~ "PAIR") | ( ident ~ image_name ~ "PAIR")) ~ (ident ~ num | ident) } skip_conf = { "SKIP_CONF" } -conf = { node_pair | node_image | node_def } + +conf = { node_pair | node_image | node_def | tool_def } + every = { "EVERY" } loop = { "LOOP" } tag = { "TAG" ~ ident } @@ -47,4 +58,3 @@ btc_node_action_type = { "MINE_BLOCKS" | "STOP_BTC" | "START_BTC" | "SEND_COINS" btc_node_action = { (image_name ~ btc_node_action_type ~ image_name ~ "AMT" ~ (num ~ sub_command | num )) | (image_name ~ btc_node_action_type ~ ( num ~ sub_command | num)) | (image_name ~ btc_node_action_type) } page = { SOI ~ ( EMPTY_LINE | COMMENT | ( (EMPTY_LINE | (skip_conf ~ NEWLINE) | (EMPTY_LINE | conf ~ NEWLINE)* ~ (up ~ NEWLINE) ) ~ ( EMPTY_LINE | (loop_content* ~ NEWLINE ) | (ln_node_action ~ NEWLINE ) | (btc_node_action ~ NEWLINE ) )*) ) ~ EOI } - diff --git a/doppler/src/tools/esplora.rs b/doppler/src/tools/esplora.rs new file mode 100644 index 0000000..839fbb7 --- /dev/null +++ b/doppler/src/tools/esplora.rs @@ -0,0 +1,314 @@ +use crate::{create_folder, get_absolute_path, Bitcoind, Options, NETWORK}; +use anyhow::{anyhow, Result}; +use docker_compose_types::{ + DependsCondition, DependsOnOptions, Entrypoint, Environment, Networks, Ports, Service, Volumes, +}; +use indexmap::IndexMap; +use std::{path::Path, process::Command}; + +use super::ToolImageInfo; + +#[derive(Clone)] +pub struct Esplora { + pub name: String, + pub http_connection: String, + pub electrum_port: String, +} + +pub fn build_esplora( + options: &mut Options, + name: &str, + image: &ToolImageInfo, + target_node: &str, +) -> Result<()> { + if options.bitcoinds.is_empty() { + return Err(anyhow!( + "bitcoind nodes need to be defined before esplora nodes can be setup" + )); + } + let electrum_port = options.new_port(); + let esplora_web_port = options.new_port(); + + let bitcoind: &Bitcoind = match options.get_bitcoind_by_name(target_node) { + Ok(bitcoind) => bitcoind, + Err(err) => return Err(err), + }; + let esplora_container_name = format!("doppler-{}-{}", name, bitcoind.name); + let mut conditional = IndexMap::new(); + conditional.insert( + bitcoind.container_name.to_owned(), + DependsCondition { + condition: String::from("service_healthy"), + }, + ); + let volume = &format!("data/{}/logs", name); + create_folder(volume)?; + let log_paths = [ + format!("{}/electrs/debug.log", volume), + format!("{}/nginx/access.log", volume), + format!("{}/nginx/error.log", volume), + format!("{}/nginx/current", volume), + format!("{}/prerenderer/current", volume), + format!("{}/websocket/current", volume), + ]; + for log_path in &log_paths { + if let Some(parent) = Path::new(log_path).parent() { + create_folder(parent.to_str().unwrap())?; + } + std::fs::File::create(log_path)?; + } + let full_path = get_absolute_path(volume)?.to_str().unwrap().to_string(); + + let (uid, gid) = get_user_ids(); + + let esplora = Service { + image: Some(image.get_tag()), + container_name: Some(esplora_container_name.clone()), + depends_on: DependsOnOptions::Conditional(conditional), + ports: Ports::Short(vec![ + format!("{}:50001", electrum_port), // Electrum RPC + format!("{}:80", esplora_web_port), // Esplora Web Interface And API Server Port + ]), + volumes: Volumes::Simple(vec![ + format!( + "{}/electrs/debug.log:/data/logs/electrs/debug.log:rw", + full_path + ), + format!("{}/nginx/error.log:/var/log/nginx/error.log:rw", full_path), + format!( + "{}/nginx/access.log:/var/log/nginx/access.log:rw", + full_path + ), + format!("{}/nginx/current:/data/logs/nginx/current:rw", full_path), + format!( + "{}/prerenderer/current:/data/logs/prerenderer/current:rw", + full_path + ), + format!( + "{}/websocket/current:/data/logs/websocket/current:rw", + full_path + ), + ]), + environment: Environment::List(vec![ + format!("FLAVOR=bitcoin-{}", options.network), + String::from("MODE=explorer"), + String::from("DEBUG=verbose"), + format!("FLAVOR=bitcoin-{}", options.network), + format!("NETWORK={}", options.network), + format!("DAEMON_RPC_ADDR={}", bitcoind.container_name), + format!("DAEMON_RPC_PORT={}", bitcoind.rpcport), + format!("RPC_USER={}", bitcoind.user), + format!("RPC_PASS={}", bitcoind.password), + format!("USER_ID={}", uid), + format!("GROUP_ID={}", gid), + String::from("STATIC_ROOT=http://localhost:5000/"), + ]), + networks: Networks::Simple(vec![NETWORK.to_owned()]), + entrypoint: Some(Entrypoint::List(vec![ + "bash".to_owned(), + "-c".to_owned(), + format!( + r#"chown $$USER_ID:$$GROUP_ID /data && \ +cat > /srv/explorer/custom_run.sh << 'EOL' +{} +EOL +chmod +x /srv/explorer/custom_run.sh && \ +exec /srv/explorer/custom_run.sh "$@""#, + CUSTOM_RUN_SCRIPT + ), + ])), + ..Default::default() + }; + + options + .services + .insert(esplora_container_name.clone(), Some(esplora)); + + options.esplora.push(Esplora { + name: name.to_owned(), + http_connection: format!("http://localhost:{}", esplora_web_port), + electrum_port: format!("localhost:{}", electrum_port), + }); + + Ok(()) +} + +fn get_user_ids() -> (String, String) { + #[cfg(target_os = "macos")] + { + // On macOS staff group is typically 20 + let default_gid = "20".to_string(); + let uid = Command::new("id") + .arg("-u") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("501".to_string()) // macOS typically starts user IDs at 501 + .trim() + .to_string(); + + (uid, default_gid) + } + + #[cfg(target_os = "linux")] + { + let uid = Command::new("id") + .arg("-u") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("1000".to_string()) + .trim() + .to_string(); + + let gid = Command::new("id") + .arg("-g") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("1000".to_string()) + .trim() + .to_string(); + + (uid, gid) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + ("1000".to_string(), "1000".to_string()) + } +} + +// This custom script allows us to point esplora at an existing bitcoind instance instead of having it create one while it starts up \ +// (which fixes many locking issues of two bitcoind instance trying to access the same data) +const CUSTOM_RUN_SCRIPT: &str = r#"#!/bin/bash +set -eo pipefail + +echo "FLAVOR: $$FLAVOR" +echo "MODE: $$MODE" +echo "DEBUG: $$DEBUG" + +# Validate required environment variables +if [ -z "$$NETWORK" ] || [ -z "$$DAEMON_RPC_ADDR" ] || [ -z "$$DAEMON_RPC_PORT" ] || [ -z "$$RPC_USER" ] || [ -z "$$RPC_PASS" ]; then + echo "Required environment variables are not set" + echo "NETWORK: $$NETWORK" + echo "DAEMON_RPC_ADDR: $$DAEMON_RPC_ADDR" + echo "DAEMON_RPC_PORT: $$DAEMON_RPC_PORT" + echo "RPC_USER: $$RPC_USER" + echo "RPC_PASS: $$RPC_PASS" + exit 1 +fi + +# Validate flavor is regtest or signet +if [ "$$NETWORK" != "regtest" ] && [ "$$NETWORK" != "signet" ]; then + echo "Only regtest and signet are supported" + echo "For example: run.sh bitcoin-regtest explorer" + exit 1 +fi + +STATIC_DIR=/srv/explorer/static/bitcoin-$${NETWORK} +if [ ! -d "$$STATIC_DIR" ]; then + echo "Static directory $$STATIC_DIR not found" + exit 1 +fi + +echo "Enabled mode $${MODE} for bitcoin-$${NETWORK}" + +# Set up all required directories +mkdir -p /data/logs/{electrs,nginx,prerenderer,websocket} +mkdir -p /data/electrs_db/$$NETWORK +mkdir -p /etc/service/{nginx,prerenderer,websocket}/log +mkdir -p /etc/run_once +mkdir -p /var/run/electrs +mkdir -p /etc/service/socat + +cp /srv/explorer/source/contrib/runits/socat.runit /etc/service/socat/run + +# Configure nginx +NGINX_PATH="$${NETWORK}/" +NGINX_NOSLASH_PATH="$${NETWORK}" +NGINX_REWRITE="rewrite ^/$${NETWORK}(/.*)$$ \$$1 break;" +NGINX_REWRITE_NOJS="rewrite ^/$${NETWORK}(/.*)$$ \"/$${NETWORK}/nojs\$$1?\" permanent" +NGINX_CSP="default-src 'self'; script-src 'self' 'unsafe-eval'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; font-src 'self' data:; object-src 'none'" +NGINX_LOGGING="access_log /var/log/nginx/access.log" + +# Configure electrs +ELECTRS_DB_DIR="/data/electrs_db/$$NETWORK" +ELECTRS_LOG_FILE="/data/logs/electrs/debug.log" + +# Start electrs in the background +if [ "$${DEBUG}" == "verbose" ]; then + RUST_BACKTRACE=full /srv/explorer/electrs_bitcoin/bin/electrs \ + --timestamp \ + --http-addr 127.0.0.1:3000 \ + --network $$NETWORK \ + --daemon-rpc-addr "$${DAEMON_RPC_ADDR}:$${DAEMON_RPC_PORT}" \ + --cookie="$${RPC_USER}:$${RPC_PASS}" \ + --monitoring-addr 0.0.0.0:4224 \ + --electrum-rpc-addr 0.0.0.0:50001 \ + --db-dir "$$ELECTRS_DB_DIR" \ + --http-socket-file /var/electrs-rest.sock \ + --jsonrpc-import \ + --address-search \ + -vvvv > "$$ELECTRS_LOG_FILE" 2>&1 & +else + /srv/explorer/electrs_bitcoin/bin/electrs \ + --timestamp \ + --http-addr 127.0.0.1:3000 \ + --network $$NETWORK \ + --daemon-rpc-addr "$${DAEMON_RPC_ADDR}:$${DAEMON_RPC_PORT}" \ + --cookie="$${RPC_USER}:$${RPC_PASS}" \ + --monitoring-addr 0.0.0.0:4224 \ + --electrum-rpc-addr 0.0.0.0:50001 \ + --db-dir "$$ELECTRS_DB_DIR" \ + --http-socket-file /var/electrs-rest.sock \ + --address-search \ + -vvvv > "$$ELECTRS_LOG_FILE" 2>&1 & +fi + +ELECTRS_PID=$! + +# Configure services +cp /srv/explorer/source/contrib/runits/nginx.runit /etc/service/nginx/run +cp /srv/explorer/source/contrib/runits/nginx-log.runit /etc/service/nginx/log/run +cp /srv/explorer/source/contrib/runits/nginx-log-config.runit /data/logs/nginx/config + +cp /srv/explorer/source/contrib/runits/prerenderer.runit /etc/service/prerenderer/run +cp /srv/explorer/source/contrib/runits/prerenderer-log.runit /etc/service/prerenderer/log/run +cp /srv/explorer/source/contrib/runits/prerenderer-log-config.runit /data/logs/prerenderer/config + +cp /srv/explorer/source/contrib/runits/websocket.runit /etc/service/websocket/run +cp /srv/explorer/source/contrib/runits/websocket-log.runit /etc/service/websocket/log/run +cp /srv/explorer/source/contrib/runits/websocket-log-config.runit /data/logs/websocket/config + +chmod +x /etc/service/*/run + +# Process nginx configuration +function preprocess(){ + in_file=$$1 + out_file=$$2 + cat $$in_file | \ + sed -e "s|{DAEMON}|bitcoin|g" \ + -e "s|{DAEMON_DIR}|$$DAEMON_DIR|g" \ + -e "s|{NETWORK}|$$NETWORK|g" \ + -e "s|{STATIC_DIR}|$$STATIC_DIR|g" \ + -e "s#{ELECTRS_ARGS}#$$ELECTRS_ARGS#g" \ + -e "s|{ELECTRS_BACKTRACE}|$$ELECTRS_BACKTRACE|g" \ + -e "s|{NGINX_LOGGING}|$$NGINX_LOGGING|g" \ + -e "s|{NGINX_PATH}|$$NGINX_PATH|g" \ + -e "s|{NGINX_CSP}|$$NGINX_CSP|g" \ + -e "s|{NGINX_REWRITE}|$$NGINX_REWRITE|g" \ + -e "s|{NGINX_REWRITE_NOJS}|$$NGINX_REWRITE_NOJS|g" \ + -e "s|{FLAVOR}|bitcoin-$$NETWORK|g" \ + -e "s|{NGINX_NOSLASH_PATH}|$$NGINX_NOSLASH_PATH|g" \ + >$$out_file +} + +preprocess /srv/explorer/source/contrib/nginx.conf.in /etc/nginx/sites-enabled/default +sed -i 's/user www-data;/user root;/' /etc/nginx/nginx.conf + +echo "Testing nginx configuration..." +nginx -t + +# Start runit services +exec /srv/explorer/source/contrib/runit_boot.sh"#; diff --git a/doppler/src/tools/mod.rs b/doppler/src/tools/mod.rs new file mode 100644 index 0000000..ccad07f --- /dev/null +++ b/doppler/src/tools/mod.rs @@ -0,0 +1,5 @@ +mod esplora; +mod supported_tools; + +pub use esplora::*; +pub use supported_tools::*; diff --git a/doppler/src/tools/supported_tools.rs b/doppler/src/tools/supported_tools.rs new file mode 100644 index 0000000..6842cce --- /dev/null +++ b/doppler/src/tools/supported_tools.rs @@ -0,0 +1,77 @@ +use crate::{CloneableHashMap, Rule}; +use pest::iterators::Pair; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum SupportedTool { + #[default] + Esplora, +} + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct ToolImageInfo { + tag: String, + name: String, + is_custom: bool, + tool_type: SupportedTool, +} + +impl ToolImageInfo { + pub fn new( + tag: String, + name: String, + is_custom: bool, + tool_type: SupportedTool, + ) -> ToolImageInfo { + ToolImageInfo { + tag, + name, + is_custom, + tool_type, + } + } + pub fn get_image(&self) -> String { + if self.is_custom { + self.tag.clone() + } else { + match self.tool_type { + SupportedTool::Esplora => String::from("blockstream/esplora:latest"), + } + } + } + pub fn is_image(&self, name: &str) -> bool { + self.name == name + } + + pub fn get_name(&self) -> String { + self.name.clone() + } + pub fn get_tag(&self) -> String { + self.tag.clone() + } +} + +impl TryFrom> for SupportedTool { + type Error = String; + + fn try_from(pair: Pair) -> Result { + match pair.as_str() { + "ESPLORA" => Ok(SupportedTool::Esplora), + _ => Err(format!("Unknown tool type: {}", pair.as_str())), + } + } +} + +pub fn get_supported_tool_images() -> CloneableHashMap { + let mut hash_map = CloneableHashMap::new(); + // NOTE: safe to use * as name since the grammar of the parse wont allow for special characters for the image name, only for the image tag + hash_map.insert( + SupportedTool::Esplora, + ToolImageInfo::new( + String::from("blockstream/esplora:latest"), + String::from("*5"), + false, + SupportedTool::Esplora, + ), + ); + hash_map +} diff --git a/doppler/src/visualizer.rs b/doppler/src/visualizer.rs index aa306ac..1d3786a 100644 --- a/doppler/src/visualizer.rs +++ b/doppler/src/visualizer.rs @@ -55,6 +55,36 @@ pub fn create_ui_config_files(options: &Options, network: &str) -> Result<(), Er node_config.write_all(network.as_bytes())?; } + for node in &options.bitcoinds { + let header = format!("[{}] \n", node.name); + let password = format!("PASSWORD={} \n", node.password); + let user = format!("USER={} \n", node.user); + let network = format!("NETWORK={} \n", network); + let node_type = format!("TYPE={} \n", "bitcoind"); + let public_p2p = format!("P2P=localhost:{} \n", node.public_p2p.unwrap()); + let public_rpc = format!("RPC=localhost:{} \n", node.public_rpc.unwrap()); + node_config.write_all(header.as_bytes())?; + node_config.write_all(node_type.as_bytes())?; + node_config.write_all(password.as_bytes())?; + node_config.write_all(user.as_bytes())?; + node_config.write_all(public_p2p.as_bytes())?; + node_config.write_all(public_rpc.as_bytes())?; + node_config.write_all(network.as_bytes())?; + } + + for node in &options.esplora { + let header = format!("[{}] \n", node.name); + let api_endpoint = format!("API_ENDPOINT={} \n", node.http_connection); + let electrum_port = format!("ELECTRUM_PORT={} \n", node.electrum_port); + let network = format!("NETWORK={} \n", network); + let node_type = format!("TYPE={} \n", "esplora"); + node_config.write_all(header.as_bytes())?; + node_config.write_all(node_type.as_bytes())?; + node_config.write_all(api_endpoint.as_bytes())?; + node_config.write_all(electrum_port.as_bytes())?; + node_config.write_all(network.as_bytes())?; + } + node_config.flush()?; Ok(()) diff --git a/doppler/src/workflow.rs b/doppler/src/workflow.rs index f27f1e3..9ae0960 100644 --- a/doppler/src/workflow.rs +++ b/doppler/src/workflow.rs @@ -1,7 +1,7 @@ use crate::{ - build_bitcoind, build_cln, build_eclair, build_lnd, load_options_from_compose, - load_options_from_external_nodes, run_cluster, DopplerParser, ImageInfo, L1Node, MinerTime, - NodeCommand, NodeKind, Options, Rule, Tag, + build_bitcoind, build_cln, build_eclair, build_esplora, build_lnd, load_options_from_compose, + load_options_from_external_nodes, run_cluster, DopplerParser, ImageInfo, L1Node, LnNodeKind, + MinerTime, NodeCommand, NodeKind, Options, Rule, SupportedTool, Tag, ToolImageInfo, }; use anyhow::{Error, Result}; use log::{debug, error, info}; @@ -99,7 +99,7 @@ pub fn run_workflow_until_stop( Ok(()) } -#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +#[derive(PartialEq, Default, Eq, PartialOrd, Ord, Hash, Clone)] struct LoopOptions { name: String, iterations: Option, @@ -107,17 +107,6 @@ struct LoopOptions { sleep_time_amt: Option, } -impl Default for LoopOptions { - fn default() -> Self { - Self { - name: String::from(""), - iterations: None, - sleep_time_interval_type: None, - sleep_time_amt: None, - } - } -} - fn process_start_loop(line: Pair) -> LoopOptions { let mut line_inner = line.into_inner(); @@ -301,7 +290,7 @@ fn run_loop( } fn get_image(options: &mut Options, node_kind: NodeKind, possible_name: &str) -> ImageInfo { - let image_info = if !possible_name.is_empty() { + if !possible_name.is_empty() { if let Some(image) = options.get_image(possible_name) { image } else { @@ -309,8 +298,7 @@ fn get_image(options: &mut Options, node_kind: NodeKind, possible_name: &str) -> } } else { options.get_default_image(node_kind) - }; - image_info + } } fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { @@ -344,26 +332,26 @@ fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { let node_name = inner.next().expect("node name").as_str(); let image: ImageInfo = match inner.next() { Some(image) => get_image(options, kind.clone(), image.as_str()), - None => options.get_default_image(kind.clone()), + _ => options.get_default_image(kind.clone()), }; handle_build_command(options, node_name, kind, &image, None)?; } Rule::node_pair => { - let kind: NodeKind = inner + let kind: LnNodeKind = inner .next() - .expect("node") + .expect("ln node kind") .try_into() - .expect("invalid node kind"); - if options.external_nodes.is_some() && kind != NodeKind::Lnd { + .expect("invalid ln node kind"); + if options.external_nodes.is_some() && kind != LnNodeKind::Lnd { unimplemented!("can only support LND nodes at the moment for remote nodes"); } let name = inner.next().expect("ident").as_str(); let image = match inner.peek().unwrap().as_rule() { Rule::image_name => { let image_name = inner.next().expect("image name").as_str(); - get_image(options, kind.clone(), image_name) + get_image(options, kind.clone().into(), image_name) } - _ => options.get_default_image(kind.clone()), + _ => options.get_default_image(kind.clone().into()), }; let to_pair = inner.next().expect("invalid layer 1 node name").as_str(); let amount = match inner.peek().is_some() { @@ -378,11 +366,26 @@ fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { handle_build_command( options, name, - kind, + kind.into(), &image, BuildDetails::new_pair(to_pair.to_owned(), amount), )?; } + Rule::tool_def => { + let tool_type: SupportedTool = inner + .next() + .expect("supported tool") + .try_into() + .expect("invalid tool type"); + + let tool_name = inner.next().expect("tool name").as_str(); + + let image: ToolImageInfo = options.get_default_tool_image(tool_type.clone()); + + let target_node = inner.next().expect("target node name").as_str(); + + handle_tool_command(options, tool_name, tool_type, &image, target_node)?; + } _ => (), } @@ -453,6 +456,18 @@ fn handle_build_command( } } +fn handle_tool_command( + options: &mut Options, + tool_name: &str, + tool_type: SupportedTool, + image: &ToolImageInfo, + target_node: &str, +) -> Result<()> { + match tool_type { + SupportedTool::Esplora => build_esplora(options, tool_name, image, target_node), + } +} + fn handle_up(options: &mut Options) -> Result<(), Error> { run_cluster(options, COMPOSE_PATH).map_err(|e| { error!("Failed to start cluster from generated compose file: {}", e); @@ -539,7 +554,7 @@ fn process_ln_action(line: Pair) -> NodeCommand { // convert to seconds match time_type { 'h' => time_num = time_num * 60 * 60, - 'm' => time_num = time_num * 60, + 'm' => time_num *= 60, _ => (), } node_command.timeout = Some(time_num) @@ -573,7 +588,7 @@ fn process_btc_action(line: Pair) -> NodeCommand { let mut line_inner = line_inner.clone().peekable(); let btc_node = line_inner.next().expect("invalid input").as_str(); let command_name = line_inner.next().expect("invalid input").as_str(); - if let None = line_inner.peek() { + if line_inner.peek().is_none() { return NodeCommand { name: command_name.to_owned(), from: btc_node.to_owned(), diff --git a/doppler_ui/package-lock.json b/doppler_ui/package-lock.json index 11bc322..9aef370 100644 --- a/doppler_ui/package-lock.json +++ b/doppler_ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "doppler", - "version": "0.3.4", + "version": "0.3.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "doppler", - "version": "0.3.4", + "version": "0.3.7", "dependencies": { "bun": "^1.0.0", "chokidar": "^4.0.1", diff --git a/doppler_ui/package.json b/doppler_ui/package.json index d3fbac5..aa0ae5f 100644 --- a/doppler_ui/package.json +++ b/doppler_ui/package.json @@ -1,6 +1,6 @@ { "name": "doppler", - "version": "0.3.7", + "version": "0.4.0", "repository": "github:tee8z/doppler", "bin": { "doppler_ui": "build/index.js" diff --git a/doppler_ui/src/components/Visualizer.svelte b/doppler_ui/src/components/Visualizer.svelte index b2361e0..1705019 100644 --- a/doppler_ui/src/components/Visualizer.svelte +++ b/doppler_ui/src/components/Visualizer.svelte @@ -120,7 +120,11 @@ let balance = await requests.fetchBalance(); let info = await requests.fetchInfo(); if (!response['error'] && info && info.identity_pubkey && key) { - nodeConnections.push({ pubkey: info.identity_pubkey, alias: key, connection: requests }); + nodeConnections.push({ + pubkey: info.identity_pubkey, + alias: key, + connection: requests + }); } if (channels && balance && info && key) { return { @@ -153,7 +157,10 @@ }; } } else if (connectionConfig.type === 'eclair') { - const requests = new EclairRequests(connectionConfig.host, connectionConfig.password); + const requests = new EclairRequests( + connectionConfig.host, + connectionConfig.password + ); const channels = await requests.fetchChannels(); const balance = await requests.fetchBalance(); const info = await requests.fetchInfo(); @@ -235,7 +242,9 @@ currentData = data; if (data.known) { - const connection = nodeConnections.find((connection) => connection.pubkey === data.known); + const connection = nodeConnections.find( + (connection) => connection.pubkey === data.known + ); if (connection) { try { const nodeInfo = await connection.connection.fetchInfo(); @@ -246,7 +255,9 @@ } } } else if (data.id !== data.known) { - const connection = nodeConnections.find((connection) => connection.pubkey === data.known); + const connection = nodeConnections.find( + (connection) => connection.pubkey === data.known + ); if (connection) { try { const nodeInfo = await connection.connection.fetchSpecificNodeInfo(data.id); @@ -293,7 +304,10 @@ return true; } } catch (error) { - console.error(`Error fetching node info using connection ${connection.alias}:`, error); + console.error( + `Error fetching node info using connection ${connection.alias}:`, + error + ); } } return false; @@ -358,14 +372,17 @@ function prettyPrintConnections(connections: Connections): string { const filteredConnections = Object.entries(connections).reduce( (acc, [key, config]) => { - const filteredConfig = Object.entries(config).reduce((configAcc, [propKey, propValue]) => { - if (propValue != null && propValue !== '') { - if (isKeyOfConnectionConfig(propKey)) { - configAcc[propKey] = propValue; + const filteredConfig = Object.entries(config).reduce( + (configAcc, [propKey, propValue]) => { + if (propValue != null && propValue !== '') { + if (isKeyOfConnectionConfig(propKey)) { + configAcc[propKey] = propValue; + } } - } - return configAcc; - }, {} as Partial); + return configAcc; + }, + {} as Partial + ); acc[key] = filteredConfig; return acc; @@ -377,7 +394,16 @@ } function isKeyOfConnectionConfig(key: string): key is keyof ConnectionConfig { - return ['macaroon', 'password', 'rune', 'host', 'type'].includes(key); + return [ + 'macaroon', + 'password', + 'user', + 'rune', + 'host', + 'type', + 'rpc_port', + 'p2p_port' + ].includes(key); } function stop() { diff --git a/doppler_ui/src/lib/connections.ts b/doppler_ui/src/lib/connections.ts index a182797..86d7cbe 100644 --- a/doppler_ui/src/lib/connections.ts +++ b/doppler_ui/src/lib/connections.ts @@ -4,6 +4,8 @@ export interface ConnectionConfig { rune: string; host: string; type: string; + rpc_port: string; + p2p_port: string; } export interface Connections { diff --git a/doppler_ui/src/routes/api/connections/+server.ts b/doppler_ui/src/routes/api/connections/+server.ts index e5e0a2e..ca03280 100644 --- a/doppler_ui/src/routes/api/connections/+server.ts +++ b/doppler_ui/src/routes/api/connections/+server.ts @@ -3,13 +3,17 @@ import fs from 'fs'; import { parse } from 'ini'; import { resolve } from 'path'; import * as path from 'path'; +import { UI_CONFIG_PATH } from '$env/static/private'; export interface ConnectionConfig { macaroon: string; rune: string; + user: String; password: string; type: string; host: string; + rpc_port: string; + p2p_port: string; } export interface Connections { @@ -29,7 +33,7 @@ function safeReadFileSync(path: string): Buffer | null { } } -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); //TODO: have the info.conf be change based on the run script export const GET: RequestHandler = async function () { @@ -57,7 +61,10 @@ export const GET: RequestHandler = async function () { host: sectionConfig.API_ENDPOINT, type: sectionConfig.TYPE, rune: '', - password: '' + password: '', + user: '', + rpc_port: '', + p2p_port: '' }; } else if (sectionConfig.TYPE === 'coreln') { connections[section] = { @@ -65,7 +72,10 @@ export const GET: RequestHandler = async function () { rune: sectionConfig.RUNE, host: sectionConfig.API_ENDPOINT, type: sectionConfig.TYPE, - password: '' + password: '', + user: '', + rpc_port: '', + p2p_port: '' }; } else if (sectionConfig.TYPE === 'eclair') { connections[section] = { @@ -73,7 +83,32 @@ export const GET: RequestHandler = async function () { rune: '', host: sectionConfig.API_ENDPOINT, password: sectionConfig.API_PASSWORD, - type: sectionConfig.TYPE + type: sectionConfig.TYPE, + user: '', + rpc_port: '', + p2p_port: '' + }; + } else if (sectionConfig.TYPE === 'bitcoind') { + connections[section] = { + macaroon: '', + rune: '', + host: '', + password: sectionConfig.PASSWORD, + type: sectionConfig.TYPE, + user: sectionConfig.USER, + rpc_port: sectionConfig.RPC, + p2p_port: sectionConfig.P2P + }; + } else if (sectionConfig.TYPE === 'esplora') { + connections[section] = { + macaroon: '', + rune: '', + host: sectionConfig.API_ENDPOINT, + password: '', + type: sectionConfig.TYPE, + user: '', + rpc_port: sectionConfig.ELECTRUM_PORT, + p2p_port: '' }; } else { throw Error(`node type ${sectionConfig.TYPE} not supported yet!`); diff --git a/doppler_ui/src/routes/api/download/+server.ts b/doppler_ui/src/routes/api/download/+server.ts index 76cf08f..44a980d 100644 --- a/doppler_ui/src/routes/api/download/+server.ts +++ b/doppler_ui/src/routes/api/download/+server.ts @@ -3,8 +3,9 @@ import type { RequestHandler } from './$types'; import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/logs/+server.ts b/doppler_ui/src/routes/api/logs/+server.ts index 86ef8e2..827d3e7 100644 --- a/doppler_ui/src/routes/api/logs/+server.ts +++ b/doppler_ui/src/routes/api/logs/+server.ts @@ -4,8 +4,9 @@ import * as fs from 'fs'; import * as path from 'path'; import chokidar from 'chokidar'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const LOGS_FOLDER = config.paths.logsFolder; diff --git a/doppler_ui/src/routes/api/reset/+server.ts b/doppler_ui/src/routes/api/reset/+server.ts index ae9c16c..2026bfd 100644 --- a/doppler_ui/src/routes/api/reset/+server.ts +++ b/doppler_ui/src/routes/api/reset/+server.ts @@ -6,9 +6,10 @@ import { parse } from 'ini'; import { resolve } from 'path'; import { v7 } from 'uuid'; import { logStreamManager } from '$lib/log_stream_manager'; +import { UI_CONFIG_PATH } from '$env/static/private'; // Read and parse the INI config file -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const LOGS_FOLDER = config.paths.logsFolder; diff --git a/doppler_ui/src/routes/api/run/+server.ts b/doppler_ui/src/routes/api/run/+server.ts index 593d7bc..a2bef38 100644 --- a/doppler_ui/src/routes/api/run/+server.ts +++ b/doppler_ui/src/routes/api/run/+server.ts @@ -5,8 +5,9 @@ import { spawn } from 'child_process'; import { parse } from 'ini'; import { createLogParser } from '$lib/log_transformers'; import { logStreamManager } from '$lib/log_stream_manager'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/save/+server.ts b/doppler_ui/src/routes/api/save/+server.ts index 465ae82..ee6aa90 100644 --- a/doppler_ui/src/routes/api/save/+server.ts +++ b/doppler_ui/src/routes/api/save/+server.ts @@ -2,8 +2,9 @@ import { json, type RequestHandler } from '@sveltejs/kit'; import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/scripts/+server.ts b/doppler_ui/src/routes/api/scripts/+server.ts index a8a03e0..d0fa08e 100644 --- a/doppler_ui/src/routes/api/scripts/+server.ts +++ b/doppler_ui/src/routes/api/scripts/+server.ts @@ -3,8 +3,9 @@ import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; import { getDirectoryTree } from '$lib/file_accessor'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/examples/doppler_files/6_tools_example/README.md b/examples/doppler_files/6_tools_example/README.md new file mode 100644 index 0000000..c3af44b --- /dev/null +++ b/examples/doppler_files/6_tools_example/README.md @@ -0,0 +1,22 @@ +### Esplora on bitcoind in cluster + +Once the simulation has finished coming up after you've hit run, you will be able to access the esplora instance by going directly to the host url shown by clicking the "show connections" button on the left side of the screen. It will look something like: +``` +"esp": { + "host": "http://localhost:9102", + "type": "esplora", + "rpc_port": "localhost:9101" +} +``` + +Additionally, the hose url will also be the base at which the esplora API is hosted at, docs for it can be found here: https://github.com/Blockstream/esplora/blob/master/API.md + +Example of calling out to it from curl: +request +``` + curl http://localhost:9102/regtest/api/blocks/tip/hash +``` +response +``` + 3f4f3ffae9ba83afb207198e6c05dde877afb7677e90439f7bd351f24634dfc5 +``` diff --git a/examples/doppler_files/6_tools_example/setup.doppler b/examples/doppler_files/6_tools_example/setup.doppler new file mode 100644 index 0000000..318063f --- /dev/null +++ b/examples/doppler_files/6_tools_example/setup.doppler @@ -0,0 +1,9 @@ +BITCOIND_MINER bd1 + +LND lnd1 PAIR bd1 +LND lnd2 PAIR bd1 +LND lnd3 PAIR bd1 + +TOOL ESPLORA esp FOR bd1 + +UP