From e2d4c8e61aacb8ce8aad15c0bfba36222d3af269 Mon Sep 17 00:00:00 2001 From: tee8z Date: Sat, 4 Jan 2025 20:11:27 -0500 Subject: [PATCH 1/5] adds esplora instance to docker compose --- doppler/config/regtest/bitcoin.conf | 5 +- doppler/config/signet/bitcoin.conf | 5 +- doppler/src/cln.rs | 2 +- doppler/src/conf_handler.rs | 23 +++++- doppler/src/docker.rs | 1 - doppler/src/lib.rs | 2 + doppler/src/node.rs | 1 + doppler/src/node_kind.rs | 59 +++++++++++++- doppler/src/parser/grammar.pest | 18 ++++- doppler/src/tools/esplora.rs | 61 +++++++++++++++ doppler/src/tools/mod.rs | 5 ++ doppler/src/tools/supported_tools.rs | 77 +++++++++++++++++++ doppler/src/workflow.rs | 49 +++++++++--- .../doppler_files/tools_example/setup.doppler | 9 +++ 14 files changed, 293 insertions(+), 24 deletions(-) create mode 100644 doppler/src/tools/esplora.rs create mode 100644 doppler/src/tools/mod.rs create mode 100644 doppler/src/tools/supported_tools.rs create mode 100644 examples/doppler_files/tools_example/setup.doppler diff --git a/doppler/config/regtest/bitcoin.conf b/doppler/config/regtest/bitcoin.conf index ad8dce6..c49c126 100755 --- a/doppler/config/regtest/bitcoin.conf +++ b/doppler/config/regtest/bitcoin.conf @@ -14,4 +14,7 @@ fallbackfee=0.00001 blockfilterindex=1 peerblockfilters=1 listen=1 -rpcthreads=25 \ No newline at end of file +rpcthreads=25 +mempoolfullrbf=1 +blocknotify=pkill -USR1 electrs +maxmempool=1000 diff --git a/doppler/config/signet/bitcoin.conf b/doppler/config/signet/bitcoin.conf index 2a55581..7c3c5ad 100644 --- a/doppler/config/signet/bitcoin.conf +++ b/doppler/config/signet/bitcoin.conf @@ -17,4 +17,7 @@ fallbackfee=0.00001 blockfilterindex=1 peerblockfilters=1 listen=1 -rpcthreads=256 \ No newline at end of file +rpcthreads=256 +mempoolfullrbf=1 +blocknotify=pkill -USR1 electrs +maxmempool=1000 diff --git a/doppler/src/cln.rs b/doppler/src/cln.rs index dfe5e98..9ec2339 100644 --- a/doppler/src/cln.rs +++ b/doppler/src/cln.rs @@ -50,7 +50,7 @@ impl Cln { ) -> Result { get_peers_short_channel_id(self, options, node_command, "source") } - + pub fn add_rune(&mut self, options: &Options) -> Result<(), Error> { let rune = get_rune(self, options)?; self.rune = Some(rune); diff --git a/doppler/src/conf_handler.rs b/doppler/src/conf_handler.rs index 117b133..b70d18c 100644 --- a/doppler/src/conf_handler.rs +++ b/doppler/src/conf_handler.rs @@ -19,8 +19,9 @@ use std::{ use crate::{ add_bitcoinds, add_coreln_nodes, add_eclair_nodes, add_external_lnd_nodes, add_lnd_nodes, - get_latest_polar_images, get_polar_images, new, update_bash_alias_external, Bitcoind, Cln, - CloneableHashMap, Eclair, ImageInfo, L1Node, L2Node, Lnd, NodeKind, Tag, Tags, NETWORK, + get_latest_polar_images, get_polar_images, get_supported_tool_images, new, + update_bash_alias_external, Bitcoind, Cln, CloneableHashMap, Eclair, ImageInfo, L1Node, L2Node, + Lnd, NodeKind, SupportedTool, Tag, Tags, ToolImageInfo, NETWORK, }; #[derive(Subcommand)] @@ -61,6 +62,7 @@ impl std::fmt::Display for ShellType { #[derive(Clone)] pub struct Options { default_images: CloneableHashMap, + default_tool_images: CloneableHashMap, known_polar_images: CloneableHashMap>, pub images: Vec, pub bitcoinds: Vec, @@ -152,8 +154,10 @@ impl Options { if external_nodes_path.is_some() { rest = true; } + let default_tool_images = get_supported_tool_images(); Self { default_images: latest_polar_images, + default_tool_images, known_polar_images: all_polar_images, images: vec::Vec::new(), bitcoinds: vec::Vec::new(), @@ -172,19 +176,21 @@ impl Options { loop_count: Arc::new(AtomicI64::new(0)), read_end_of_doppler_file: Arc::new(AtomicBool::new(true)), tags: Arc::new(Mutex::new(new(connection))), - rest: rest, - external_nodes_path: external_nodes_path, + rest, + external_nodes_path, external_nodes: None, ui_config_path, network, } } + pub fn get_image(&self, name: &str) -> Option { self.images .iter() .find(|image| image.is_image(name)) .map(|image| image.clone()) } + pub fn get_default_image(&self, node_kind: NodeKind) -> ImageInfo { match self.default_images.get(node_kind) { Some(image) => image, @@ -192,6 +198,15 @@ impl Options { } .clone() } + + pub fn get_default_tool_image(&self, tool: SupportedTool) -> ToolImageInfo { + match self.default_tool_images.get(tool) { + Some(image) => image, + None => panic!("error no default images found!"), + } + .clone() + } + pub fn add_thread(&self, thread_handler: Thread) { self.thread_handlers.lock().unwrap().push(thread_handler); } diff --git a/doppler/src/docker.rs b/doppler/src/docker.rs index 5b5a5df..edede52 100644 --- a/doppler/src/docker.rs +++ b/doppler/src/docker.rs @@ -1,4 +1,3 @@ -extern crate ini; use crate::{ create_ui_config_files, get_absolute_path, pair_bitcoinds, L1Node, L2Node, NodeCommand, Options, }; diff --git a/doppler/src/lib.rs b/doppler/src/lib.rs index 6c8af0f..4ea1181 100644 --- a/doppler/src/lib.rs +++ b/doppler/src/lib.rs @@ -10,6 +10,7 @@ mod node_kind; mod parser; mod polar_default_images; mod simple_storage; +mod tools; mod visualizer; mod workflow; @@ -25,5 +26,6 @@ pub use node_kind::*; pub use parser::*; pub use polar_default_images::*; pub use simple_storage::*; +pub use tools::*; pub use visualizer::*; pub use workflow::*; diff --git a/doppler/src/node.rs b/doppler/src/node.rs index 134b063..c6e5ccb 100644 --- a/doppler/src/node.rs +++ b/doppler/src/node.rs @@ -196,6 +196,7 @@ impl ImageInfo { pub fn is_image(&self, name: &str) -> bool { self.name == name } + pub fn get_name(&self) -> String { self.name.clone() } diff --git a/doppler/src/node_kind.rs b/doppler/src/node_kind.rs index 27642fc..b81e508 100644 --- a/doppler/src/node_kind.rs +++ b/doppler/src/node_kind.rs @@ -13,7 +13,6 @@ pub enum NodeKind { Coreln, Eclair, } - impl<'a> TryFrom> for NodeKind { type Error = anyhow::Error; @@ -31,3 +30,61 @@ impl<'a> TryFrom> for NodeKind { } } } + +impl From for NodeKind { + fn from(value: LnNodeKind) -> NodeKind { + match value { + LnNodeKind::Lnd => NodeKind::Lnd, + LnNodeKind::Coreln => NodeKind::Coreln, + LnNodeKind::Eclair => NodeKind::Eclair, + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum BtcNodeKind { + Bitcoind, + #[default] + BitcoindMiner, +} + +impl<'a> TryFrom> for BtcNodeKind { + type Error = anyhow::Error; + + fn try_from(value: Pair) -> Result { + match value.as_rule() { + Rule::node_kind => match value.as_str() { + "BITCOIND" => Ok(BtcNodeKind::Bitcoind), + "BITCOIND_MINER" => Ok(BtcNodeKind::BitcoindMiner), + _ => bail!("invalid btc_node_kind"), + }, + _ => bail!("pair should be a btc_node_kind"), + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum LnNodeKind { + #[default] + Lnd, + Coreln, + Eclair, +} + +impl<'a> TryFrom> for LnNodeKind { + type Error = anyhow::Error; // or whatever error type you're using + + fn try_from(pair: Pair) -> Result { + match pair.as_rule() { + Rule::ln_node_kind => match pair.as_str() { + "LND" => Ok(LnNodeKind::Lnd), + "ECLAIR" => Ok(LnNodeKind::Eclair), + "CORELN" => Ok(LnNodeKind::Coreln), + _ => Err(anyhow::anyhow!("Unknown ln node kind")), + }, + _ => Err(anyhow::anyhow!( + "invalid ln node kind: pair should be a ln_node_kind" + )), + } + } +} diff --git a/doppler/src/parser/grammar.pest b/doppler/src/parser/grammar.pest index c19f155..dba0693 100644 --- a/doppler/src/parser/grammar.pest +++ b/doppler/src/parser/grammar.pest @@ -9,19 +9,30 @@ num = @{ ASCII_DIGIT+ } time_digits = { "s" | "m" | "h" } -node_kind = { "BITCOIND_MINER" | "BITCOIND" | "LND" | "CORELN" | "ECLAIR" } + +btc_node_kind = { "BITCOIND_MINER" | "BITCOIND" } +ln_node_kind = { "LND" | "CORELN" | "ECLAIR" } +node_kind = { btc_node_kind | ln_node_kind } + +tool_kind = { "TOOL" } image_name = @{ ASCII_ALPHANUMERIC+ } image_version = @{ (ASCII_ALPHANUMERIC | PUNCTUATION)+ } +supported_tool = { "ESPLORA" } + +tool_def = { "TOOL" ~ supported_tool ~ ident ~ "FOR" ~ ident } + node_def = { (node_kind ~ ident ~ image_name) | (node_kind ~ ident ) } node_image = { node_kind ~ "IMAGE" ~ image_name ~ image_version } -node_pair = { node_kind ~ (( ident ~ "PAIR") | ( ident ~ image_name ~ "PAIR")) ~ (ident ~ num | ident) } +node_pair = { ln_node_kind ~ (( ident ~ "PAIR") | ( ident ~ image_name ~ "PAIR")) ~ (ident ~ num | ident) } skip_conf = { "SKIP_CONF" } -conf = { node_pair | node_image | node_def } + +conf = { node_pair | node_image | node_def | tool_def } + every = { "EVERY" } loop = { "LOOP" } tag = { "TAG" ~ ident } @@ -47,4 +58,3 @@ btc_node_action_type = { "MINE_BLOCKS" | "STOP_BTC" | "START_BTC" | "SEND_COINS" btc_node_action = { (image_name ~ btc_node_action_type ~ image_name ~ "AMT" ~ (num ~ sub_command | num )) | (image_name ~ btc_node_action_type ~ ( num ~ sub_command | num)) | (image_name ~ btc_node_action_type) } page = { SOI ~ ( EMPTY_LINE | COMMENT | ( (EMPTY_LINE | (skip_conf ~ NEWLINE) | (EMPTY_LINE | conf ~ NEWLINE)* ~ (up ~ NEWLINE) ) ~ ( EMPTY_LINE | (loop_content* ~ NEWLINE ) | (ln_node_action ~ NEWLINE ) | (btc_node_action ~ NEWLINE ) )*) ) ~ EOI } - diff --git a/doppler/src/tools/esplora.rs b/doppler/src/tools/esplora.rs new file mode 100644 index 0000000..ae2f9a4 --- /dev/null +++ b/doppler/src/tools/esplora.rs @@ -0,0 +1,61 @@ +use crate::{Bitcoind, Options, NETWORK}; +use anyhow::{anyhow, Result}; +use docker_compose_types::{ + Command, DependsOnOptions, Environment, Networks, Ports, Service, Volumes, +}; + +use super::ToolImageInfo; + +pub fn build_esplora( + options: &mut Options, + name: &str, + image: &ToolImageInfo, + target_node: &str, +) -> Result<()> { + if options.bitcoinds.is_empty() { + return Err(anyhow!( + "bitcoind nodes need to be defined before esplora nodes can be setup" + )); + } + let electrum_port = options.new_port(); + let esplora_web_port = options.new_port(); + + let bitcoind: &Bitcoind = match options.get_bitcoind_by_name(target_node) { + Ok(bitcoind) => bitcoind, + Err(err) => return Err(err), + }; + let esplora_container_name = format!("doppler-{}-{}", name, bitcoind.name); + let mut env_vars = vec![ + String::from("DEBUG=verbose"), + format!("BITCOIN_NODE_HOST={}", bitcoind.container_name), + format!("BITCOIN_NODE_PORT={}", bitcoind.rpcport), + ]; + if options.network == "regtest" { + env_vars.push(String::from("NO_REGTEST_MINING=1")) + } + + let esplora = Service { + image: Some(image.get_tag()), + container_name: Some(esplora_container_name.clone()), + depends_on: DependsOnOptions::Simple(vec![bitcoind.container_name.clone()]), + ports: Ports::Short(vec![ + format!("{}:50001", electrum_port), // Electrum RPC + format!("{}:80", esplora_web_port), // Esplora Web Interface And API Server Port + ]), + volumes: Volumes::Simple(vec![format!("{}:/data:rw", bitcoind.path_vol)]), + command: Some(Command::Args(vec![ + "/srv/explorer/run.sh".to_owned(), + format!("bitcoin-{}", options.network), + "explorer".to_owned(), + ])), + networks: Networks::Simple(vec![NETWORK.to_owned()]), + environment: Environment::List(env_vars.into()), + ..Default::default() + }; + + options + .services + .insert(esplora_container_name.clone(), Some(esplora)); + + Ok(()) +} diff --git a/doppler/src/tools/mod.rs b/doppler/src/tools/mod.rs new file mode 100644 index 0000000..ccad07f --- /dev/null +++ b/doppler/src/tools/mod.rs @@ -0,0 +1,5 @@ +mod esplora; +mod supported_tools; + +pub use esplora::*; +pub use supported_tools::*; diff --git a/doppler/src/tools/supported_tools.rs b/doppler/src/tools/supported_tools.rs new file mode 100644 index 0000000..6842cce --- /dev/null +++ b/doppler/src/tools/supported_tools.rs @@ -0,0 +1,77 @@ +use crate::{CloneableHashMap, Rule}; +use pest::iterators::Pair; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum SupportedTool { + #[default] + Esplora, +} + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct ToolImageInfo { + tag: String, + name: String, + is_custom: bool, + tool_type: SupportedTool, +} + +impl ToolImageInfo { + pub fn new( + tag: String, + name: String, + is_custom: bool, + tool_type: SupportedTool, + ) -> ToolImageInfo { + ToolImageInfo { + tag, + name, + is_custom, + tool_type, + } + } + pub fn get_image(&self) -> String { + if self.is_custom { + self.tag.clone() + } else { + match self.tool_type { + SupportedTool::Esplora => String::from("blockstream/esplora:latest"), + } + } + } + pub fn is_image(&self, name: &str) -> bool { + self.name == name + } + + pub fn get_name(&self) -> String { + self.name.clone() + } + pub fn get_tag(&self) -> String { + self.tag.clone() + } +} + +impl TryFrom> for SupportedTool { + type Error = String; + + fn try_from(pair: Pair) -> Result { + match pair.as_str() { + "ESPLORA" => Ok(SupportedTool::Esplora), + _ => Err(format!("Unknown tool type: {}", pair.as_str())), + } + } +} + +pub fn get_supported_tool_images() -> CloneableHashMap { + let mut hash_map = CloneableHashMap::new(); + // NOTE: safe to use * as name since the grammar of the parse wont allow for special characters for the image name, only for the image tag + hash_map.insert( + SupportedTool::Esplora, + ToolImageInfo::new( + String::from("blockstream/esplora:latest"), + String::from("*5"), + false, + SupportedTool::Esplora, + ), + ); + hash_map +} diff --git a/doppler/src/workflow.rs b/doppler/src/workflow.rs index f27f1e3..9bb2006 100644 --- a/doppler/src/workflow.rs +++ b/doppler/src/workflow.rs @@ -1,7 +1,7 @@ use crate::{ - build_bitcoind, build_cln, build_eclair, build_lnd, load_options_from_compose, - load_options_from_external_nodes, run_cluster, DopplerParser, ImageInfo, L1Node, MinerTime, - NodeCommand, NodeKind, Options, Rule, Tag, + build_bitcoind, build_cln, build_eclair, build_esplora, build_lnd, load_options_from_compose, + load_options_from_external_nodes, run_cluster, DopplerParser, ImageInfo, L1Node, LnNodeKind, + MinerTime, NodeCommand, NodeKind, Options, Rule, SupportedTool, Tag, ToolImageInfo, }; use anyhow::{Error, Result}; use log::{debug, error, info}; @@ -344,26 +344,26 @@ fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { let node_name = inner.next().expect("node name").as_str(); let image: ImageInfo = match inner.next() { Some(image) => get_image(options, kind.clone(), image.as_str()), - None => options.get_default_image(kind.clone()), + _ => options.get_default_image(kind.clone()), }; handle_build_command(options, node_name, kind, &image, None)?; } Rule::node_pair => { - let kind: NodeKind = inner + let kind: LnNodeKind = inner .next() - .expect("node") + .expect("ln node kind") .try_into() - .expect("invalid node kind"); - if options.external_nodes.is_some() && kind != NodeKind::Lnd { + .expect("invalid ln node kind"); + if options.external_nodes.is_some() && kind != LnNodeKind::Lnd { unimplemented!("can only support LND nodes at the moment for remote nodes"); } let name = inner.next().expect("ident").as_str(); let image = match inner.peek().unwrap().as_rule() { Rule::image_name => { let image_name = inner.next().expect("image name").as_str(); - get_image(options, kind.clone(), image_name) + get_image(options, kind.clone().into(), image_name) } - _ => options.get_default_image(kind.clone()), + _ => options.get_default_image(kind.clone().into()), }; let to_pair = inner.next().expect("invalid layer 1 node name").as_str(); let amount = match inner.peek().is_some() { @@ -378,11 +378,26 @@ fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { handle_build_command( options, name, - kind, + kind.into(), &image, BuildDetails::new_pair(to_pair.to_owned(), amount), )?; } + Rule::tool_def => { + let tool_type: SupportedTool = inner + .next() + .expect("supported tool") + .try_into() + .expect("invalid tool type"); + + let tool_name = inner.next().expect("tool name").as_str(); + + let image: ToolImageInfo = options.get_default_tool_image(tool_type.clone()); + + let target_node = inner.next().expect("target node name").as_str(); + + handle_tool_command(options, tool_name, tool_type, &image, target_node)?; + } _ => (), } @@ -453,6 +468,18 @@ fn handle_build_command( } } +fn handle_tool_command( + options: &mut Options, + tool_name: &str, + tool_type: SupportedTool, + image: &ToolImageInfo, + target_node: &str, +) -> Result<()> { + match tool_type { + SupportedTool::Esplora => build_esplora(options, tool_name, image, target_node), + } +} + fn handle_up(options: &mut Options) -> Result<(), Error> { run_cluster(options, COMPOSE_PATH).map_err(|e| { error!("Failed to start cluster from generated compose file: {}", e); diff --git a/examples/doppler_files/tools_example/setup.doppler b/examples/doppler_files/tools_example/setup.doppler new file mode 100644 index 0000000..318063f --- /dev/null +++ b/examples/doppler_files/tools_example/setup.doppler @@ -0,0 +1,9 @@ +BITCOIND_MINER bd1 + +LND lnd1 PAIR bd1 +LND lnd2 PAIR bd1 +LND lnd3 PAIR bd1 + +TOOL ESPLORA esp FOR bd1 + +UP From c9ef75d35059c5aa6bab7e0ab5b5698297d46e89 Mon Sep 17 00:00:00 2001 From: tee8z Date: Sun, 5 Jan 2025 01:13:24 -0500 Subject: [PATCH 2/5] getting close, just having issue with the UI now --- .github/workflows/release.yml | 60 ++--- dist-workspace.toml | 6 +- doppler/config/regtest/bitcoin.conf | 3 - doppler/config/signet/bitcoin.conf | 3 - doppler/src/bitcoind.rs | 75 +++++-- doppler/src/conf_handler.rs | 6 +- doppler/src/tools/esplora.rs | 210 ++++++++++++++++-- doppler/src/visualizer.rs | 30 +++ doppler_ui/package-lock.json | 4 +- doppler_ui/src/components/Visualizer.svelte | 52 +++-- doppler_ui/src/lib/connections.ts | 2 + .../src/routes/api/connections/+server.ts | 43 +++- doppler_ui/src/routes/api/download/+server.ts | 3 +- doppler_ui/src/routes/api/logs/+server.ts | 3 +- doppler_ui/src/routes/api/reset/+server.ts | 3 +- doppler_ui/src/routes/api/run/+server.ts | 3 +- doppler_ui/src/routes/api/save/+server.ts | 3 +- doppler_ui/src/routes/api/scripts/+server.ts | 3 +- 18 files changed, 414 insertions(+), 98 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42b4067..1dffbcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# This file was autogenerated by cargo-dist: https://opensource.axo.dev/cargo-dist/ +# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/ # # Copyright 2022-2024, axodotdev # SPDX-License-Identifier: MIT or Apache-2.0 @@ -6,7 +6,7 @@ # CI that: # # * checks for a Git Tag that looks like a release -# * builds artifacts with cargo-dist (archives, installers, hashes) +# * builds artifacts with dist (archives, installers, hashes) # * uploads those artifacts to temporary workflow zip # * on success, uploads the artifacts to a GitHub Release # @@ -24,10 +24,10 @@ permissions: # must be a Cargo-style SemVer Version (must have at least major.minor.patch). # # If PACKAGE_NAME is specified, then the announcement will be for that -# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# package (erroring out if it doesn't have the given version or isn't dist-able). # # If PACKAGE_NAME isn't specified, then the announcement will be for all -# (cargo-dist-able) packages in the workspace with that version (this mode is +# (dist-able) packages in the workspace with that version (this mode is # intended for workspaces with only one dist-able package, or with all dist-able # packages versioned/released in lockstep). # @@ -45,7 +45,7 @@ on: - '**[0-9]+.[0-9]+.[0-9]+*' jobs: - # Run 'cargo dist plan' (or host) to determine what tasks we need to do + # Run 'dist plan' (or host) to determine what tasks we need to do plan: runs-on: "ubuntu-20.04" outputs: @@ -59,16 +59,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cargo-dist + - name: Install dist # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.23.0/cargo-dist-installer.sh | sh" - - name: Cache cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.27.0/cargo-dist-installer.sh | sh" + - name: Cache dist uses: actions/upload-artifact@v4 with: name: cargo-dist-cache - path: ~/.cargo/bin/cargo-dist + path: ~/.cargo/bin/dist # sure would be cool if github gave us proper conditionals... # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible # functionality based on whether this is a pull_request, and whether it's from a fork. @@ -76,8 +76,8 @@ jobs: # but also really annoying to build CI around when it needs secrets to work right.) - id: plan run: | - cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json - echo "cargo dist ran successfully" + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" cat plan-dist-manifest.json echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" - name: "Upload dist-manifest.json" @@ -95,18 +95,19 @@ jobs: if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} strategy: fail-fast: false - # Target platforms/runners are computed by cargo-dist in create-release. + # Target platforms/runners are computed by dist in create-release. # Each member of the matrix has the following arguments: # # - runner: the github runner - # - dist-args: cli flags to pass to cargo dist - # - install-dist: expression to run to install cargo-dist on the runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner # # Typically there will be: # - 1 "global" task that builds universal installers # - N "local" tasks that build each platform's binaries and platform-specific installers matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json @@ -117,8 +118,15 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cargo-dist - run: ${{ matrix.install_dist }} + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install dist + run: ${{ matrix.install_dist.run }} # Get the dist-manifest - name: Fetch local artifacts uses: actions/download-artifact@v4 @@ -132,8 +140,8 @@ jobs: - name: Build artifacts run: | # Actually do builds and make zips and whatnot - cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json - echo "cargo dist ran successfully" + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" - id: cargo-dist name: Post-build # We force bash here just because github makes it really hard to get values up @@ -143,7 +151,7 @@ jobs: run: | # Parse out what we just built and upload it to scratch storage echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" cp dist-manifest.json "$BUILD_MANIFEST_NAME" @@ -168,12 +176,12 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cached cargo-dist + - name: Install cached dist uses: actions/download-artifact@v4 with: name: cargo-dist-cache path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/cargo-dist + - run: chmod +x ~/.cargo/bin/dist # Get all the local artifacts for the global tasks to use (for e.g. checksums) - name: Fetch local artifacts uses: actions/download-artifact@v4 @@ -184,8 +192,8 @@ jobs: - id: cargo-dist shell: bash run: | - cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json - echo "cargo dist ran successfully" + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" # Parse out what we just built and upload it to scratch storage echo "paths<> "$GITHUB_OUTPUT" @@ -217,12 +225,12 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install cached cargo-dist + - name: Install cached dist uses: actions/download-artifact@v4 with: name: cargo-dist-cache path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/cargo-dist + - run: chmod +x ~/.cargo/bin/dist # Fetch artifacts from scratch-storage - name: Fetch artifacts uses: actions/download-artifact@v4 @@ -233,7 +241,7 @@ jobs: - id: host shell: bash run: | - cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json echo "artifacts uploaded and released successfully" cat dist-manifest.json echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" diff --git a/dist-workspace.toml b/dist-workspace.toml index 87c0ced..2af8a56 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -1,10 +1,10 @@ [workspace] members = ["npm:doppler_ui", "cargo:doppler"] -# Config for 'cargo dist' +# Config for 'dist' [dist] -# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.23.0" +# The preferred dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.27.0" # CI backends to support ci = "github" # Target platforms to build apps for (Rust target-triple syntax) diff --git a/doppler/config/regtest/bitcoin.conf b/doppler/config/regtest/bitcoin.conf index c49c126..974514f 100755 --- a/doppler/config/regtest/bitcoin.conf +++ b/doppler/config/regtest/bitcoin.conf @@ -15,6 +15,3 @@ blockfilterindex=1 peerblockfilters=1 listen=1 rpcthreads=25 -mempoolfullrbf=1 -blocknotify=pkill -USR1 electrs -maxmempool=1000 diff --git a/doppler/config/signet/bitcoin.conf b/doppler/config/signet/bitcoin.conf index 7c3c5ad..ec0f72d 100644 --- a/doppler/config/signet/bitcoin.conf +++ b/doppler/config/signet/bitcoin.conf @@ -18,6 +18,3 @@ blockfilterindex=1 peerblockfilters=1 listen=1 rpcthreads=256 -mempoolfullrbf=1 -blocknotify=pkill -USR1 electrs -maxmempool=1000 diff --git a/doppler/src/bitcoind.rs b/doppler/src/bitcoind.rs index ebf3048..d2ff594 100644 --- a/doppler/src/bitcoind.rs +++ b/doppler/src/bitcoind.rs @@ -3,7 +3,9 @@ use crate::{ }; use anyhow::{anyhow, Error, Result}; use conf_parser::processer::{FileConf, Section}; -use docker_compose_types::{EnvFile, Networks, Ports, Service, Volumes}; +use docker_compose_types::{ + EnvFile, Healthcheck, HealthcheckTest, Networks, Ports, Service, Volumes, +}; use log::{error, info}; use std::{ fs::{File, OpenOptions}, @@ -24,6 +26,9 @@ pub struct Bitcoind { pub zmqpubhashblock: String, pub zmqpubrawtx: String, pub path_vol: String, + // Only added when initially creating the docker compose + pub public_p2p: Option, + pub public_rpc: Option, } pub enum L1Enum { @@ -113,25 +118,48 @@ pub fn build_bitcoind( image: &ImageInfo, is_miner: bool, ) -> Result<()> { - let bitcoind_conf = get_config(options, name, is_miner).unwrap(); + let mut bitcoind_conf = get_config(options, name, is_miner).unwrap(); + let public_p2p = options.new_port(); + let public_rpc = options.new_port(); + let bitcoind = Service { image: Some(image.get_image()), container_name: Some(bitcoind_conf.container_name.clone()), ports: Ports::Short(vec![ - format!("{}:{}", options.new_port(), bitcoind_conf.p2pport), - format!("{}:{}", options.new_port(), bitcoind_conf.rpcport), + format!("{}:{}", public_p2p, bitcoind_conf.p2pport), + format!("{}:{}", public_rpc, bitcoind_conf.rpcport), ]), volumes: Volumes::Simple(vec![format!( "{}:/home/bitcoin/.bitcoin:rw", bitcoind_conf.path_vol )]), env_file: Some(EnvFile::Simple(".env".to_owned())), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single( + vec![ + "bitcoin-cli".to_string(), + format!("-rpcuser={}", bitcoind_conf.user), + format!("-rpcpassword={}", bitcoind_conf.password), + format!("-rpcport={}", bitcoind_conf.rpcport), + "getblockchaininfo".to_string(), + ] + .join(" "), + )), + interval: Some("30s".to_string()), + timeout: Some("10s".to_string()), + retries: 3, + disable: false, + start_period: Some("40s".to_string()), + }), networks: Networks::Simple(vec![NETWORK.to_owned()]), ..Default::default() }; options .services .insert(bitcoind_conf.container_name.clone(), Some(bitcoind)); + + bitcoind_conf.public_p2p = Some(public_p2p); + bitcoind_conf.public_rpc = Some(public_rpc); options.bitcoinds.push(bitcoind_conf); Ok(()) } @@ -180,6 +208,8 @@ fn load_config(name: &str, container_name: &str, network: &str) -> Result Result "38333", + _ => "18444", + }; + let rpc_port = match options.network.as_ref() { + "signet" => "38332", + _ => "18443", + }; + bitcoin.set_property("bind", "0.0.0.0"); - bitcoin.set_property("port", &port.to_string()); - bitcoin.set_property("rpcport", &rpc_port.to_string()); - bitcoin.set_property("rpcuser", "admin"); - bitcoin.set_property("rpcpassword", "1234"); - bitcoin.set_property( - "zmqpubrawblock", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); - bitcoin.set_property( - "zmqpubrawtx", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); - bitcoin.set_property( - "zmqpubhashblock", - &format!("tcp://0.0.0.0:{}", options.new_port()), - ); + bitcoin.set_property("port", port); + bitcoin.set_property("rpcport", rpc_port); + bitcoin.set_property("rpcuser", "bitcoin"); + bitcoin.set_property("rpcpassword", "bitcoin"); + bitcoin.set_property("zmqpubrawblock", "tcp://0.0.0.0:28332"); + bitcoin.set_property("zmqpubrawtx", "tcp://0.0.0.0:28333"); + bitcoin.set_property("zmqpubhashtx", "tcp://0.0.0.0:28332"); + bitcoin.set_property("zmqpubhashblock", "tcp://0.0.0.0:28332"); let network_section = get_network_section(conf, &options.network)?; Ok(network_section) } diff --git a/doppler/src/conf_handler.rs b/doppler/src/conf_handler.rs index b70d18c..3e1131c 100644 --- a/doppler/src/conf_handler.rs +++ b/doppler/src/conf_handler.rs @@ -20,8 +20,8 @@ use std::{ use crate::{ add_bitcoinds, add_coreln_nodes, add_eclair_nodes, add_external_lnd_nodes, add_lnd_nodes, get_latest_polar_images, get_polar_images, get_supported_tool_images, new, - update_bash_alias_external, Bitcoind, Cln, CloneableHashMap, Eclair, ImageInfo, L1Node, L2Node, - Lnd, NodeKind, SupportedTool, Tag, Tags, ToolImageInfo, NETWORK, + update_bash_alias_external, Bitcoind, Cln, CloneableHashMap, Eclair, Esplora, ImageInfo, + L1Node, L2Node, Lnd, NodeKind, SupportedTool, Tag, Tags, ToolImageInfo, NETWORK, }; #[derive(Subcommand)] @@ -66,6 +66,7 @@ pub struct Options { known_polar_images: CloneableHashMap>, pub images: Vec, pub bitcoinds: Vec, + pub esplora: Vec, pub lnd_nodes: Vec, pub eclair_nodes: Vec, pub cln_nodes: Vec, @@ -164,6 +165,7 @@ impl Options { lnd_nodes: vec::Vec::new(), eclair_nodes: vec::Vec::new(), cln_nodes: vec::Vec::new(), + esplora: vec::Vec::new(), ports: starting_port, compose_path: None, services: indexmap::IndexMap::new(), diff --git a/doppler/src/tools/esplora.rs b/doppler/src/tools/esplora.rs index ae2f9a4..383210e 100644 --- a/doppler/src/tools/esplora.rs +++ b/doppler/src/tools/esplora.rs @@ -1,11 +1,20 @@ -use crate::{Bitcoind, Options, NETWORK}; +use crate::{create_folder, get_absolute_path, Bitcoind, Options, NETWORK}; use anyhow::{anyhow, Result}; use docker_compose_types::{ - Command, DependsOnOptions, Environment, Networks, Ports, Service, Volumes, + Command, DependsCondition, DependsOnOptions, Entrypoint, Environment, Networks, Ports, Service, + Volumes, }; +use indexmap::IndexMap; use super::ToolImageInfo; +#[derive(Clone)] +pub struct Esplora { + pub name: String, + pub http_connection: String, + pub electrum_port: String, +} + pub fn build_esplora( options: &mut Options, name: &str, @@ -25,31 +34,50 @@ pub fn build_esplora( Err(err) => return Err(err), }; let esplora_container_name = format!("doppler-{}-{}", name, bitcoind.name); - let mut env_vars = vec![ - String::from("DEBUG=verbose"), - format!("BITCOIN_NODE_HOST={}", bitcoind.container_name), - format!("BITCOIN_NODE_PORT={}", bitcoind.rpcport), - ]; - if options.network == "regtest" { - env_vars.push(String::from("NO_REGTEST_MINING=1")) - } + let mut conditional = IndexMap::new(); + conditional.insert( + bitcoind.container_name.to_owned(), + DependsCondition { + condition: String::from("service_healthy"), + }, + ); + let volume = &format!("data/{}", name); + create_folder(volume)?; + let full_path = get_absolute_path(volume)?.to_str().unwrap().to_string(); let esplora = Service { image: Some(image.get_tag()), container_name: Some(esplora_container_name.clone()), - depends_on: DependsOnOptions::Simple(vec![bitcoind.container_name.clone()]), + depends_on: DependsOnOptions::Conditional(conditional), ports: Ports::Short(vec![ format!("{}:50001", electrum_port), // Electrum RPC format!("{}:80", esplora_web_port), // Esplora Web Interface And API Server Port ]), - volumes: Volumes::Simple(vec![format!("{}:/data:rw", bitcoind.path_vol)]), + volumes: Volumes::Simple(vec![format!("{}:/data:rw", full_path)]), command: Some(Command::Args(vec![ - "/srv/explorer/run.sh".to_owned(), format!("bitcoin-{}", options.network), "explorer".to_owned(), + "verbose".to_owned(), ])), + environment: Environment::List(vec![ + format!("NETWORK={}", options.network), + format!("DAEMON_RPC_ADDR={}", bitcoind.container_name), + format!("DAEMON_RPC_PORT={}", bitcoind.rpcport), + format!("RPC_USER={}", bitcoind.user), + format!("RPC_PASS={}", bitcoind.password), + ]), networks: Networks::Simple(vec![NETWORK.to_owned()]), - environment: Environment::List(env_vars.into()), + entrypoint: Some(Entrypoint::List(vec![ + "bash".to_owned(), + "-c".to_owned(), + format!( + r#"cat > /srv/explorer/custom_run.sh << 'EOL' +{} +EOL +chmod +x /srv/explorer/custom_run.sh && exec /srv/explorer/custom_run.sh "$@""#, + CUSTOM_RUN_SCRIPT + ), + ])), ..Default::default() }; @@ -57,5 +85,159 @@ pub fn build_esplora( .services .insert(esplora_container_name.clone(), Some(esplora)); + options.esplora.push(Esplora { + name: name.to_owned(), + http_connection: format!("http://localhost:{}", esplora_web_port), + electrum_port: format!("localhost:{}", electrum_port), + }); + Ok(()) } + +// This custom script allows us to point esplora at an existing bitcoind instance instead of having it create one while it starts up \ +// (which fixes many locking issues of two bitcoind instance trying to access the same data) +const CUSTOM_RUN_SCRIPT: &str = r#"#!/bin/bash +set -eo pipefail + +# Initialize required variables +FLAVOR=$${0:-} +MODE=$${1:-} +DEBUG=$${2:-} + +# Debug - list content of relevant directories +echo "Checking binary locations:" +ls -l /srv/explorer/ +ls -l /srv/explorer/electrs*/bin/ || echo "No electrs binary found in expected location" + +# Validate required environment variables +if [ -z "$$NETWORK" ] || [ -z "$$DAEMON_RPC_ADDR" ] || [ -z "$$DAEMON_RPC_PORT" ] || [ -z "$$RPC_USER" ] || [ -z "$$RPC_PASS" ]; then + echo "Required environment variables are not set" + echo "NETWORK: $$NETWORK" + echo "DAEMON_RPC_ADDR: $$DAEMON_RPC_ADDR" + echo "DAEMON_RPC_PORT: $$DAEMON_RPC_PORT" + echo "RPC_USER: $$RPC_USER" + echo "RPC_PASS: $$RPC_PASS" + exit 1 +fi + +# Validate flavor is regtest or signet +if [ "$$NETWORK" != "regtest" ] && [ "$$NETWORK" != "signet" ]; then + echo "Only regtest and signet are supported" + echo "For example: run.sh bitcoin-regtest explorer" + exit 1 +fi + +STATIC_DIR=/srv/explorer/static/bitcoin-$${NETWORK} +if [ ! -d "$$STATIC_DIR" ]; then + echo "Static directory $$STATIC_DIR not found" + exit 1 +fi + +echo "Enabled mode $${MODE} for bitcoin-$${NETWORK}" + +# Set up directories +mkdir -p /data/logs/electrs +mkdir -p /data/electrs_db/$$NETWORK + +# Configure nginx +NGINX_PATH="$${NETWORK}/" +NGINX_NOSLASH_PATH="$${NETWORK}" +NGINX_REWRITE="rewrite ^/$${NETWORK}(/.*)$$ \$$1 break;" +NGINX_REWRITE_NOJS="rewrite ^/$${NETWORK}(/.*)$$ \"/$${NETWORK}/nojs\$$1?\" permanent" +NGINX_CSP="default-src 'self'; script-src 'self' 'unsafe-eval'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; font-src 'self' data:; object-src 'none'" +NGINX_LOGGING="access_log off" + +# Configure electrs +ELECTRS_DB_DIR="/data/electrs_db/$$NETWORK" +ELECTRS_LOG_FILE="/data/logs/electrs/debug.log" + +# Start electrs in the background +if [ "$${DEBUG}" == "verbose" ]; then + RUST_BACKTRACE=full /srv/explorer/electrs_bitcoin/bin/electrs \ + --timestamp \ + --http-addr 127.0.0.1:3000 \ + --network $$NETWORK \ + --daemon-rpc-addr "$${DAEMON_RPC_ADDR}:$${DAEMON_RPC_PORT}" \ + --cookie="$${RPC_USER}:$${RPC_PASS}" \ + --monitoring-addr 0.0.0.0:4224 \ + --electrum-rpc-addr 0.0.0.0:50001 \ + --db-dir "$$ELECTRS_DB_DIR" \ + --http-socket-file /var/electrs-rest.sock \ + --jsonrpc-import \ + --address-search \ + -vvvv > "$$ELECTRS_LOG_FILE" 2>&1 & +else + /srv/explorer/electrs_bitcoin/bin/electrs \ + --timestamp \ + --http-addr 127.0.0.1:3000 \ + --network $$NETWORK \ + --daemon-rpc-addr "$${DAEMON_RPC_ADDR}:$${DAEMON_RPC_PORT}" \ + --cookie="$${RPC_USER}:$${RPC_PASS}" \ + --monitoring-addr 0.0.0.0:4224 \ + --electrum-rpc-addr 0.0.0.0:50001 \ + --db-dir "$$ELECTRS_DB_DIR" \ + --http-socket-file /var/electrs-rest.sock \ + --address-search \ + -vvvv > "$$ELECTRS_LOG_FILE" 2>&1 & +fi + +ELECTRS_PID=$! + +# Set up minimal runit services for nginx and other required services +mkdir -p /etc/service/{nginx,prerenderer,websocket}/log +mkdir -p /data/logs/{nginx,prerenderer,websocket} + +# Configure nginx +cp /srv/explorer/source/contrib/runits/nginx.runit /etc/service/nginx/run +cp /srv/explorer/source/contrib/runits/nginx-log.runit /etc/service/nginx/log/run +cp /srv/explorer/source/contrib/runits/nginx-log-config.runit /data/logs/nginx/config + +# Set up prerenderer +cp /srv/explorer/source/contrib/runits/prerenderer.runit /etc/service/prerenderer/run +cp /srv/explorer/source/contrib/runits/prerenderer-log.runit /etc/service/prerenderer/log/run +cp /srv/explorer/source/contrib/runits/prerenderer-log-config.runit /data/logs/prerenderer/config + +# Set up websocket +cp /srv/explorer/source/contrib/runits/websocket.runit /etc/service/websocket/run +cp /srv/explorer/source/contrib/runits/websocket-log.runit /etc/service/websocket/log/run +cp /srv/explorer/source/contrib/runits/websocket-log-config.runit /data/logs/websocket/config + +# Make scripts executable +chmod +x /etc/service/*/run + +# Process nginx configuration +function preprocess(){ + in_file=$$1 + out_file=$$2 + cat $$in_file | \ + sed -e "s|{DAEMON}|bitcoin|g" \ + -e "s|{DAEMON_DIR}|$$DAEMON_DIR|g" \ + -e "s|{NETWORK}|$$NETWORK|g" \ + -e "s|{STATIC_DIR}|$$STATIC_DIR|g" \ + -e "s#{ELECTRS_ARGS}#$$ELECTRS_ARGS#g" \ + -e "s|{ELECTRS_BACKTRACE}|$$ELECTRS_BACKTRACE|g" \ + -e "s|{NGINX_LOGGING}|$$NGINX_LOGGING|g" \ + -e "s|{NGINX_PATH}|$$NGINX_PATH|g" \ + -e "s|{NGINX_CSP}|$$NGINX_CSP|g" \ + -e "s|{NGINX_REWRITE}|$$NGINX_REWRITE|g" \ + -e "s|{NGINX_REWRITE_NOJS}|$$NGINX_REWRITE_NOJS|g" \ + -e "s|{FLAVOR}|bitcoin-$$NETWORK|g" \ + -e "s|{NGINX_NOSLASH_PATH}|$$NGINX_NOSLASH_PATH|g" \ + >$$out_file +} + +preprocess /srv/explorer/source/contrib/nginx.conf.in /etc/nginx/sites-enabled/default +sed -i 's/user www-data;/user root;/' /etc/nginx/nginx.conf + +echo "Checking processed nginx config:" +cat /etc/nginx/sites-enabled/default + +# Test nginx configuration +echo "Testing nginx configuration..." +nginx -t + +# Create required directory for runit +mkdir -p /etc/run_once + +# Start runit services (nginx, prerenderer, websocket) +exec /srv/explorer/source/contrib/runit_boot.sh"#; diff --git a/doppler/src/visualizer.rs b/doppler/src/visualizer.rs index aa306ac..1d3786a 100644 --- a/doppler/src/visualizer.rs +++ b/doppler/src/visualizer.rs @@ -55,6 +55,36 @@ pub fn create_ui_config_files(options: &Options, network: &str) -> Result<(), Er node_config.write_all(network.as_bytes())?; } + for node in &options.bitcoinds { + let header = format!("[{}] \n", node.name); + let password = format!("PASSWORD={} \n", node.password); + let user = format!("USER={} \n", node.user); + let network = format!("NETWORK={} \n", network); + let node_type = format!("TYPE={} \n", "bitcoind"); + let public_p2p = format!("P2P=localhost:{} \n", node.public_p2p.unwrap()); + let public_rpc = format!("RPC=localhost:{} \n", node.public_rpc.unwrap()); + node_config.write_all(header.as_bytes())?; + node_config.write_all(node_type.as_bytes())?; + node_config.write_all(password.as_bytes())?; + node_config.write_all(user.as_bytes())?; + node_config.write_all(public_p2p.as_bytes())?; + node_config.write_all(public_rpc.as_bytes())?; + node_config.write_all(network.as_bytes())?; + } + + for node in &options.esplora { + let header = format!("[{}] \n", node.name); + let api_endpoint = format!("API_ENDPOINT={} \n", node.http_connection); + let electrum_port = format!("ELECTRUM_PORT={} \n", node.electrum_port); + let network = format!("NETWORK={} \n", network); + let node_type = format!("TYPE={} \n", "esplora"); + node_config.write_all(header.as_bytes())?; + node_config.write_all(node_type.as_bytes())?; + node_config.write_all(api_endpoint.as_bytes())?; + node_config.write_all(electrum_port.as_bytes())?; + node_config.write_all(network.as_bytes())?; + } + node_config.flush()?; Ok(()) diff --git a/doppler_ui/package-lock.json b/doppler_ui/package-lock.json index 11bc322..9aef370 100644 --- a/doppler_ui/package-lock.json +++ b/doppler_ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "doppler", - "version": "0.3.4", + "version": "0.3.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "doppler", - "version": "0.3.4", + "version": "0.3.7", "dependencies": { "bun": "^1.0.0", "chokidar": "^4.0.1", diff --git a/doppler_ui/src/components/Visualizer.svelte b/doppler_ui/src/components/Visualizer.svelte index b2361e0..1705019 100644 --- a/doppler_ui/src/components/Visualizer.svelte +++ b/doppler_ui/src/components/Visualizer.svelte @@ -120,7 +120,11 @@ let balance = await requests.fetchBalance(); let info = await requests.fetchInfo(); if (!response['error'] && info && info.identity_pubkey && key) { - nodeConnections.push({ pubkey: info.identity_pubkey, alias: key, connection: requests }); + nodeConnections.push({ + pubkey: info.identity_pubkey, + alias: key, + connection: requests + }); } if (channels && balance && info && key) { return { @@ -153,7 +157,10 @@ }; } } else if (connectionConfig.type === 'eclair') { - const requests = new EclairRequests(connectionConfig.host, connectionConfig.password); + const requests = new EclairRequests( + connectionConfig.host, + connectionConfig.password + ); const channels = await requests.fetchChannels(); const balance = await requests.fetchBalance(); const info = await requests.fetchInfo(); @@ -235,7 +242,9 @@ currentData = data; if (data.known) { - const connection = nodeConnections.find((connection) => connection.pubkey === data.known); + const connection = nodeConnections.find( + (connection) => connection.pubkey === data.known + ); if (connection) { try { const nodeInfo = await connection.connection.fetchInfo(); @@ -246,7 +255,9 @@ } } } else if (data.id !== data.known) { - const connection = nodeConnections.find((connection) => connection.pubkey === data.known); + const connection = nodeConnections.find( + (connection) => connection.pubkey === data.known + ); if (connection) { try { const nodeInfo = await connection.connection.fetchSpecificNodeInfo(data.id); @@ -293,7 +304,10 @@ return true; } } catch (error) { - console.error(`Error fetching node info using connection ${connection.alias}:`, error); + console.error( + `Error fetching node info using connection ${connection.alias}:`, + error + ); } } return false; @@ -358,14 +372,17 @@ function prettyPrintConnections(connections: Connections): string { const filteredConnections = Object.entries(connections).reduce( (acc, [key, config]) => { - const filteredConfig = Object.entries(config).reduce((configAcc, [propKey, propValue]) => { - if (propValue != null && propValue !== '') { - if (isKeyOfConnectionConfig(propKey)) { - configAcc[propKey] = propValue; + const filteredConfig = Object.entries(config).reduce( + (configAcc, [propKey, propValue]) => { + if (propValue != null && propValue !== '') { + if (isKeyOfConnectionConfig(propKey)) { + configAcc[propKey] = propValue; + } } - } - return configAcc; - }, {} as Partial); + return configAcc; + }, + {} as Partial + ); acc[key] = filteredConfig; return acc; @@ -377,7 +394,16 @@ } function isKeyOfConnectionConfig(key: string): key is keyof ConnectionConfig { - return ['macaroon', 'password', 'rune', 'host', 'type'].includes(key); + return [ + 'macaroon', + 'password', + 'user', + 'rune', + 'host', + 'type', + 'rpc_port', + 'p2p_port' + ].includes(key); } function stop() { diff --git a/doppler_ui/src/lib/connections.ts b/doppler_ui/src/lib/connections.ts index a182797..86d7cbe 100644 --- a/doppler_ui/src/lib/connections.ts +++ b/doppler_ui/src/lib/connections.ts @@ -4,6 +4,8 @@ export interface ConnectionConfig { rune: string; host: string; type: string; + rpc_port: string; + p2p_port: string; } export interface Connections { diff --git a/doppler_ui/src/routes/api/connections/+server.ts b/doppler_ui/src/routes/api/connections/+server.ts index e5e0a2e..ca03280 100644 --- a/doppler_ui/src/routes/api/connections/+server.ts +++ b/doppler_ui/src/routes/api/connections/+server.ts @@ -3,13 +3,17 @@ import fs from 'fs'; import { parse } from 'ini'; import { resolve } from 'path'; import * as path from 'path'; +import { UI_CONFIG_PATH } from '$env/static/private'; export interface ConnectionConfig { macaroon: string; rune: string; + user: String; password: string; type: string; host: string; + rpc_port: string; + p2p_port: string; } export interface Connections { @@ -29,7 +33,7 @@ function safeReadFileSync(path: string): Buffer | null { } } -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); //TODO: have the info.conf be change based on the run script export const GET: RequestHandler = async function () { @@ -57,7 +61,10 @@ export const GET: RequestHandler = async function () { host: sectionConfig.API_ENDPOINT, type: sectionConfig.TYPE, rune: '', - password: '' + password: '', + user: '', + rpc_port: '', + p2p_port: '' }; } else if (sectionConfig.TYPE === 'coreln') { connections[section] = { @@ -65,7 +72,10 @@ export const GET: RequestHandler = async function () { rune: sectionConfig.RUNE, host: sectionConfig.API_ENDPOINT, type: sectionConfig.TYPE, - password: '' + password: '', + user: '', + rpc_port: '', + p2p_port: '' }; } else if (sectionConfig.TYPE === 'eclair') { connections[section] = { @@ -73,7 +83,32 @@ export const GET: RequestHandler = async function () { rune: '', host: sectionConfig.API_ENDPOINT, password: sectionConfig.API_PASSWORD, - type: sectionConfig.TYPE + type: sectionConfig.TYPE, + user: '', + rpc_port: '', + p2p_port: '' + }; + } else if (sectionConfig.TYPE === 'bitcoind') { + connections[section] = { + macaroon: '', + rune: '', + host: '', + password: sectionConfig.PASSWORD, + type: sectionConfig.TYPE, + user: sectionConfig.USER, + rpc_port: sectionConfig.RPC, + p2p_port: sectionConfig.P2P + }; + } else if (sectionConfig.TYPE === 'esplora') { + connections[section] = { + macaroon: '', + rune: '', + host: sectionConfig.API_ENDPOINT, + password: '', + type: sectionConfig.TYPE, + user: '', + rpc_port: sectionConfig.ELECTRUM_PORT, + p2p_port: '' }; } else { throw Error(`node type ${sectionConfig.TYPE} not supported yet!`); diff --git a/doppler_ui/src/routes/api/download/+server.ts b/doppler_ui/src/routes/api/download/+server.ts index 76cf08f..44a980d 100644 --- a/doppler_ui/src/routes/api/download/+server.ts +++ b/doppler_ui/src/routes/api/download/+server.ts @@ -3,8 +3,9 @@ import type { RequestHandler } from './$types'; import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/logs/+server.ts b/doppler_ui/src/routes/api/logs/+server.ts index 86ef8e2..827d3e7 100644 --- a/doppler_ui/src/routes/api/logs/+server.ts +++ b/doppler_ui/src/routes/api/logs/+server.ts @@ -4,8 +4,9 @@ import * as fs from 'fs'; import * as path from 'path'; import chokidar from 'chokidar'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const LOGS_FOLDER = config.paths.logsFolder; diff --git a/doppler_ui/src/routes/api/reset/+server.ts b/doppler_ui/src/routes/api/reset/+server.ts index ae9c16c..2026bfd 100644 --- a/doppler_ui/src/routes/api/reset/+server.ts +++ b/doppler_ui/src/routes/api/reset/+server.ts @@ -6,9 +6,10 @@ import { parse } from 'ini'; import { resolve } from 'path'; import { v7 } from 'uuid'; import { logStreamManager } from '$lib/log_stream_manager'; +import { UI_CONFIG_PATH } from '$env/static/private'; // Read and parse the INI config file -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const LOGS_FOLDER = config.paths.logsFolder; diff --git a/doppler_ui/src/routes/api/run/+server.ts b/doppler_ui/src/routes/api/run/+server.ts index 593d7bc..a2bef38 100644 --- a/doppler_ui/src/routes/api/run/+server.ts +++ b/doppler_ui/src/routes/api/run/+server.ts @@ -5,8 +5,9 @@ import { spawn } from 'child_process'; import { parse } from 'ini'; import { createLogParser } from '$lib/log_transformers'; import { logStreamManager } from '$lib/log_stream_manager'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/save/+server.ts b/doppler_ui/src/routes/api/save/+server.ts index 465ae82..ee6aa90 100644 --- a/doppler_ui/src/routes/api/save/+server.ts +++ b/doppler_ui/src/routes/api/save/+server.ts @@ -2,8 +2,9 @@ import { json, type RequestHandler } from '@sveltejs/kit'; import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; diff --git a/doppler_ui/src/routes/api/scripts/+server.ts b/doppler_ui/src/routes/api/scripts/+server.ts index a8a03e0..d0fa08e 100644 --- a/doppler_ui/src/routes/api/scripts/+server.ts +++ b/doppler_ui/src/routes/api/scripts/+server.ts @@ -3,8 +3,9 @@ import fs from 'fs'; import path from 'path'; import { parse } from 'ini'; import { getDirectoryTree } from '$lib/file_accessor'; +import { UI_CONFIG_PATH } from '$env/static/private'; -const configPath = process.env.UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); +const configPath = UI_CONFIG_PATH || path.join(process.cwd(), '/build/ui_config'); const config = parse(fs.readFileSync(`${configPath}/server.conf.ini`, 'utf-8')); const DOPPLER_SCRIPTS_FOLDER = config.paths.dopplerScriptsFolder; From aa18bda8ec159de721e8cacc83147f7f54719806 Mon Sep 17 00:00:00 2001 From: tee8z Date: Sun, 5 Jan 2025 13:11:44 -0500 Subject: [PATCH 3/5] working esplora on existing bitcoin node --- doppler/src/bitcoind.rs | 4 +- doppler/src/docker.rs | 2 +- doppler/src/tools/esplora.rs | 151 ++++++++++++++++++++++++++--------- doppler/src/workflow.rs | 22 ++--- 4 files changed, 120 insertions(+), 59 deletions(-) diff --git a/doppler/src/bitcoind.rs b/doppler/src/bitcoind.rs index d2ff594..f7d5666 100644 --- a/doppler/src/bitcoind.rs +++ b/doppler/src/bitcoind.rs @@ -136,7 +136,7 @@ pub fn build_bitcoind( env_file: Some(EnvFile::Simple(".env".to_owned())), healthcheck: Some(Healthcheck { test: Some(HealthcheckTest::Single( - vec![ + [ "bitcoin-cli".to_string(), format!("-rpcuser={}", bitcoind_conf.user), format!("-rpcpassword={}", bitcoind_conf.password), @@ -290,7 +290,7 @@ fn get_bitcoind_config( } fn set_network_section(conf: &mut FileConf, options: &mut Options) -> Result { - if conf.sections.get(&options.network).is_none() { + if !conf.sections.contains_key(&options.network) { conf.sections .insert(options.network.clone(), Section::new()); } diff --git a/doppler/src/docker.rs b/doppler/src/docker.rs index edede52..a850b61 100644 --- a/doppler/src/docker.rs +++ b/doppler/src/docker.rs @@ -25,7 +25,7 @@ pub fn load_options_from_external_nodes( debug!("loaded lnds"); let network = options.external_nodes.clone().unwrap()[0].network.clone(); - create_ui_config_files(&options, &network)?; + create_ui_config_files(options, &network)?; Ok(()) } diff --git a/doppler/src/tools/esplora.rs b/doppler/src/tools/esplora.rs index 383210e..40fed61 100644 --- a/doppler/src/tools/esplora.rs +++ b/doppler/src/tools/esplora.rs @@ -1,10 +1,10 @@ use crate::{create_folder, get_absolute_path, Bitcoind, Options, NETWORK}; use anyhow::{anyhow, Result}; use docker_compose_types::{ - Command, DependsCondition, DependsOnOptions, Entrypoint, Environment, Networks, Ports, Service, - Volumes, + DependsCondition, DependsOnOptions, Entrypoint, Environment, Networks, Ports, Service, Volumes, }; use indexmap::IndexMap; +use std::{path::Path, process::Command}; use super::ToolImageInfo; @@ -28,6 +28,7 @@ pub fn build_esplora( } let electrum_port = options.new_port(); let esplora_web_port = options.new_port(); + let elects_port = options.new_port(); let bitcoind: &Bitcoind = match options.get_bitcoind_by_name(target_node) { Ok(bitcoind) => bitcoind, @@ -41,10 +42,26 @@ pub fn build_esplora( condition: String::from("service_healthy"), }, ); - let volume = &format!("data/{}", name); + let volume = &format!("data/{}/logs", name); create_folder(volume)?; + let log_paths = [ + format!("{}/electrs/debug.log", volume), + format!("{}/nginx/access.log", volume), + format!("{}/nginx/error.log", volume), + format!("{}/nginx/current", volume), + format!("{}/prerenderer/current", volume), + format!("{}/websocket/current", volume), + ]; + for log_path in &log_paths { + if let Some(parent) = Path::new(log_path).parent() { + create_folder(parent.to_str().unwrap())?; + } + std::fs::File::create(log_path)?; + } let full_path = get_absolute_path(volume)?.to_str().unwrap().to_string(); + let (uid, gid) = get_user_ids(); + let esplora = Service { image: Some(image.get_tag()), container_name: Some(esplora_container_name.clone()), @@ -52,29 +69,53 @@ pub fn build_esplora( ports: Ports::Short(vec![ format!("{}:50001", electrum_port), // Electrum RPC format!("{}:80", esplora_web_port), // Esplora Web Interface And API Server Port + format!("{}:3000", elects_port), // Esplora Web Interface And API Server Port + ]), + volumes: Volumes::Simple(vec![ + format!( + "{}/electrs/debug.log:/data/logs/electrs/debug.log:rw", + full_path + ), + format!("{}/nginx/error.log:/var/log/nginx/error.log:rw", full_path), + format!( + "{}/nginx/access.log:/var/log/nginx/access.log:rw", + full_path + ), + format!("{}/nginx/current:/data/logs/nginx/current:rw", full_path), + format!( + "{}/prerenderer/current:/data/logs/prerenderer/current:rw", + full_path + ), + format!( + "{}/websocket/current:/data/logs/websocket/current:rw", + full_path + ), ]), - volumes: Volumes::Simple(vec![format!("{}:/data:rw", full_path)]), - command: Some(Command::Args(vec![ - format!("bitcoin-{}", options.network), - "explorer".to_owned(), - "verbose".to_owned(), - ])), environment: Environment::List(vec![ + format!("FLAVOR=bitcoin-{}", options.network), + String::from("MODE=explorer"), + String::from("DEBUG=verbose"), + format!("FLAVOR=bitcoin-{}", options.network), format!("NETWORK={}", options.network), format!("DAEMON_RPC_ADDR={}", bitcoind.container_name), format!("DAEMON_RPC_PORT={}", bitcoind.rpcport), format!("RPC_USER={}", bitcoind.user), format!("RPC_PASS={}", bitcoind.password), + format!("USER_ID={}", uid), + format!("GROUP_ID={}", gid), + String::from("STATIC_ROOT=http://localhost:5000/"), ]), networks: Networks::Simple(vec![NETWORK.to_owned()]), entrypoint: Some(Entrypoint::List(vec![ "bash".to_owned(), "-c".to_owned(), format!( - r#"cat > /srv/explorer/custom_run.sh << 'EOL' + r#"chown $$USER_ID:$$GROUP_ID /data && \ +cat > /srv/explorer/custom_run.sh << 'EOL' {} EOL -chmod +x /srv/explorer/custom_run.sh && exec /srv/explorer/custom_run.sh "$@""#, +chmod +x /srv/explorer/custom_run.sh && \ +exec /srv/explorer/custom_run.sh "$@""#, CUSTOM_RUN_SCRIPT ), ])), @@ -94,20 +135,60 @@ chmod +x /srv/explorer/custom_run.sh && exec /srv/explorer/custom_run.sh "$@""#, Ok(()) } +fn get_user_ids() -> (String, String) { + #[cfg(target_os = "macos")] + { + // On macOS staff group is typically 20 + let default_gid = "20".to_string(); + let uid = Command::new("id") + .arg("-u") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("501".to_string()) // macOS typically starts user IDs at 501 + .trim() + .to_string(); + + (uid, default_gid) + } + + #[cfg(target_os = "linux")] + { + let uid = Command::new("id") + .arg("-u") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("1000".to_string()) + .trim() + .to_string(); + + let gid = Command::new("id") + .arg("-g") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .unwrap_or("1000".to_string()) + .trim() + .to_string(); + + (uid, gid) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + ("1000".to_string(), "1000".to_string()) + } +} + // This custom script allows us to point esplora at an existing bitcoind instance instead of having it create one while it starts up \ // (which fixes many locking issues of two bitcoind instance trying to access the same data) const CUSTOM_RUN_SCRIPT: &str = r#"#!/bin/bash set -eo pipefail -# Initialize required variables -FLAVOR=$${0:-} -MODE=$${1:-} -DEBUG=$${2:-} - -# Debug - list content of relevant directories -echo "Checking binary locations:" -ls -l /srv/explorer/ -ls -l /srv/explorer/electrs*/bin/ || echo "No electrs binary found in expected location" +echo "FLAVOR: $$FLAVOR" +echo "MODE: $$MODE" +echo "DEBUG: $$DEBUG" # Validate required environment variables if [ -z "$$NETWORK" ] || [ -z "$$DAEMON_RPC_ADDR" ] || [ -z "$$DAEMON_RPC_PORT" ] || [ -z "$$RPC_USER" ] || [ -z "$$RPC_PASS" ]; then @@ -135,9 +216,15 @@ fi echo "Enabled mode $${MODE} for bitcoin-$${NETWORK}" -# Set up directories -mkdir -p /data/logs/electrs +# Set up all required directories +mkdir -p /data/logs/{electrs,nginx,prerenderer,websocket} mkdir -p /data/electrs_db/$$NETWORK +mkdir -p /etc/service/{nginx,prerenderer,websocket}/log +mkdir -p /etc/run_once +mkdir -p /var/run/electrs +mkdir -p /etc/service/socat + +cp /srv/explorer/source/contrib/runits/socat.runit /etc/service/socat/run # Configure nginx NGINX_PATH="$${NETWORK}/" @@ -145,7 +232,7 @@ NGINX_NOSLASH_PATH="$${NETWORK}" NGINX_REWRITE="rewrite ^/$${NETWORK}(/.*)$$ \$$1 break;" NGINX_REWRITE_NOJS="rewrite ^/$${NETWORK}(/.*)$$ \"/$${NETWORK}/nojs\$$1?\" permanent" NGINX_CSP="default-src 'self'; script-src 'self' 'unsafe-eval'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; font-src 'self' data:; object-src 'none'" -NGINX_LOGGING="access_log off" +NGINX_LOGGING="access_log /var/log/nginx/access.log" # Configure electrs ELECTRS_DB_DIR="/data/electrs_db/$$NETWORK" @@ -183,26 +270,19 @@ fi ELECTRS_PID=$! -# Set up minimal runit services for nginx and other required services -mkdir -p /etc/service/{nginx,prerenderer,websocket}/log -mkdir -p /data/logs/{nginx,prerenderer,websocket} - -# Configure nginx +# Configure services cp /srv/explorer/source/contrib/runits/nginx.runit /etc/service/nginx/run cp /srv/explorer/source/contrib/runits/nginx-log.runit /etc/service/nginx/log/run cp /srv/explorer/source/contrib/runits/nginx-log-config.runit /data/logs/nginx/config -# Set up prerenderer cp /srv/explorer/source/contrib/runits/prerenderer.runit /etc/service/prerenderer/run cp /srv/explorer/source/contrib/runits/prerenderer-log.runit /etc/service/prerenderer/log/run cp /srv/explorer/source/contrib/runits/prerenderer-log-config.runit /data/logs/prerenderer/config -# Set up websocket cp /srv/explorer/source/contrib/runits/websocket.runit /etc/service/websocket/run cp /srv/explorer/source/contrib/runits/websocket-log.runit /etc/service/websocket/log/run cp /srv/explorer/source/contrib/runits/websocket-log-config.runit /data/logs/websocket/config -# Make scripts executable chmod +x /etc/service/*/run # Process nginx configuration @@ -229,15 +309,8 @@ function preprocess(){ preprocess /srv/explorer/source/contrib/nginx.conf.in /etc/nginx/sites-enabled/default sed -i 's/user www-data;/user root;/' /etc/nginx/nginx.conf -echo "Checking processed nginx config:" -cat /etc/nginx/sites-enabled/default - -# Test nginx configuration echo "Testing nginx configuration..." nginx -t -# Create required directory for runit -mkdir -p /etc/run_once - -# Start runit services (nginx, prerenderer, websocket) +# Start runit services exec /srv/explorer/source/contrib/runit_boot.sh"#; diff --git a/doppler/src/workflow.rs b/doppler/src/workflow.rs index 9bb2006..9ae0960 100644 --- a/doppler/src/workflow.rs +++ b/doppler/src/workflow.rs @@ -99,7 +99,7 @@ pub fn run_workflow_until_stop( Ok(()) } -#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +#[derive(PartialEq, Default, Eq, PartialOrd, Ord, Hash, Clone)] struct LoopOptions { name: String, iterations: Option, @@ -107,17 +107,6 @@ struct LoopOptions { sleep_time_amt: Option, } -impl Default for LoopOptions { - fn default() -> Self { - Self { - name: String::from(""), - iterations: None, - sleep_time_interval_type: None, - sleep_time_amt: None, - } - } -} - fn process_start_loop(line: Pair) -> LoopOptions { let mut line_inner = line.into_inner(); @@ -301,7 +290,7 @@ fn run_loop( } fn get_image(options: &mut Options, node_kind: NodeKind, possible_name: &str) -> ImageInfo { - let image_info = if !possible_name.is_empty() { + if !possible_name.is_empty() { if let Some(image) = options.get_image(possible_name) { image } else { @@ -309,8 +298,7 @@ fn get_image(options: &mut Options, node_kind: NodeKind, possible_name: &str) -> } } else { options.get_default_image(node_kind) - }; - image_info + } } fn handle_conf(options: &mut Options, line: Pair) -> Result<()> { @@ -566,7 +554,7 @@ fn process_ln_action(line: Pair) -> NodeCommand { // convert to seconds match time_type { 'h' => time_num = time_num * 60 * 60, - 'm' => time_num = time_num * 60, + 'm' => time_num *= 60, _ => (), } node_command.timeout = Some(time_num) @@ -600,7 +588,7 @@ fn process_btc_action(line: Pair) -> NodeCommand { let mut line_inner = line_inner.clone().peekable(); let btc_node = line_inner.next().expect("invalid input").as_str(); let command_name = line_inner.next().expect("invalid input").as_str(); - if let None = line_inner.peek() { + if line_inner.peek().is_none() { return NodeCommand { name: command_name.to_owned(), from: btc_node.to_owned(), From 6998fd341d7d66c699628fe7def600ac0e4e739c Mon Sep 17 00:00:00 2001 From: tee8z Date: Sun, 5 Jan 2025 14:08:37 -0500 Subject: [PATCH 4/5] add notes on how to use esplora tool --- doppler/src/tools/esplora.rs | 2 -- .../doppler_files/6_tools_example/README.md | 22 +++++++++++++++++++ .../setup.doppler | 0 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 examples/doppler_files/6_tools_example/README.md rename examples/doppler_files/{tools_example => 6_tools_example}/setup.doppler (100%) diff --git a/doppler/src/tools/esplora.rs b/doppler/src/tools/esplora.rs index 40fed61..839fbb7 100644 --- a/doppler/src/tools/esplora.rs +++ b/doppler/src/tools/esplora.rs @@ -28,7 +28,6 @@ pub fn build_esplora( } let electrum_port = options.new_port(); let esplora_web_port = options.new_port(); - let elects_port = options.new_port(); let bitcoind: &Bitcoind = match options.get_bitcoind_by_name(target_node) { Ok(bitcoind) => bitcoind, @@ -69,7 +68,6 @@ pub fn build_esplora( ports: Ports::Short(vec![ format!("{}:50001", electrum_port), // Electrum RPC format!("{}:80", esplora_web_port), // Esplora Web Interface And API Server Port - format!("{}:3000", elects_port), // Esplora Web Interface And API Server Port ]), volumes: Volumes::Simple(vec![ format!( diff --git a/examples/doppler_files/6_tools_example/README.md b/examples/doppler_files/6_tools_example/README.md new file mode 100644 index 0000000..c3af44b --- /dev/null +++ b/examples/doppler_files/6_tools_example/README.md @@ -0,0 +1,22 @@ +### Esplora on bitcoind in cluster + +Once the simulation has finished coming up after you've hit run, you will be able to access the esplora instance by going directly to the host url shown by clicking the "show connections" button on the left side of the screen. It will look something like: +``` +"esp": { + "host": "http://localhost:9102", + "type": "esplora", + "rpc_port": "localhost:9101" +} +``` + +Additionally, the hose url will also be the base at which the esplora API is hosted at, docs for it can be found here: https://github.com/Blockstream/esplora/blob/master/API.md + +Example of calling out to it from curl: +request +``` + curl http://localhost:9102/regtest/api/blocks/tip/hash +``` +response +``` + 3f4f3ffae9ba83afb207198e6c05dde877afb7677e90439f7bd351f24634dfc5 +``` diff --git a/examples/doppler_files/tools_example/setup.doppler b/examples/doppler_files/6_tools_example/setup.doppler similarity index 100% rename from examples/doppler_files/tools_example/setup.doppler rename to examples/doppler_files/6_tools_example/setup.doppler From 39a212676c5bc0d8b3ef108fc154bc450698989a Mon Sep 17 00:00:00 2001 From: tee8z Date: Sun, 5 Jan 2025 14:09:43 -0500 Subject: [PATCH 5/5] updating release --- Cargo.lock | 2 +- doppler/Cargo.toml | 2 +- doppler_ui/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51d749e..0b54433 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,7 +403,7 @@ dependencies = [ [[package]] name = "doppler" -version = "0.3.7" +version = "0.4.0" dependencies = [ "anyhow", "base64", diff --git a/doppler/Cargo.toml b/doppler/Cargo.toml index 5d2e080..cf35675 100644 --- a/doppler/Cargo.toml +++ b/doppler/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "doppler" -version = "0.3.7" +version = "0.4.0" repository = "https://github.com/tee8z/doppler.git" [package.metadata.dist] diff --git a/doppler_ui/package.json b/doppler_ui/package.json index d3fbac5..aa0ae5f 100644 --- a/doppler_ui/package.json +++ b/doppler_ui/package.json @@ -1,6 +1,6 @@ { "name": "doppler", - "version": "0.3.7", + "version": "0.4.0", "repository": "github:tee8z/doppler", "bin": { "doppler_ui": "build/index.js"