ord index update, better snapshot handling, remove delays due to snap…

…shot process
sadoprotocol · May 24, 2023 · 27f1521 · 27f1521
1 parent 25eb5c7
commit 27f1521
Show file tree

Hide file tree

Showing 3 changed files with 122 additions and 36 deletions.
diff --git a/ord-index-config b/ord-index-config
@@ -2,3 +2,4 @@ NETWORK="regtest" # OR "testnet3", "" for Mainnet
 DEFAULT_DATA_DIR="/home/bitcoin-regtest/.local/share/ord/"
 ALT_DATA_DIR="/home/bitcoin-regtest/ord-data/"
 ALT_DUP_DATA_DIR="/home/bitcoin-regtest/ord-data-dup/"
+SNAPSHOT_DATA_DIR="/home/bitcoin-regtest/ord-snapshot/"
diff --git a/ord-index.sh b/ord-index.sh
@@ -1,19 +1,18 @@
 #!/bin/bash
-# Note:
-# This index checking is only used for double database location
-# where the one who is indexing will keep the data in a separate location.
-# Once done with indexing, it should transfer to the original location
 
-if [ ! -f "./.ord-index-config" ]
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [ ! -f "$SCRIPT_DIR/.ord-index-config" ]
 then
   echo "No config file found.."
   exit
 fi
 
-source "./.ord-index-config"
+source "$SCRIPT_DIR/.ord-index-config"
+
+RUNNING=$(ps -fu $UID | grep "ord" | grep "data-dir" | grep -v "$SNAPSHOT_DATA_DIR" | grep "index-sats index")
 
-if ps -fu $UID | grep "ord" | grep "data-dir" | grep -q "index-sats index"
-then 
+if [ ! -z "${RUNNING}" ]; then 
   echo "Still running.."
   exit
 fi
@@ -23,14 +22,14 @@ if test -f "$ALT_DATA_DIR$NETWORK/lock"; then
   exit
 fi
 
+mkdir -p $ALT_DATA_DIR$NETWORK
+
 # LOCK
 touch "$ALT_DATA_DIR$NETWORK/lock"
 
 
 # == SETUP
 
-mkdir -p $ALT_DATA_DIR$NETWORK
-
 if [ ! -f $ALT_DATA_DIR$NETWORK"/index.redb" ]
 then
   rsync -a $DEFAULT_DATA_DIR$NETWORK"/index.redb" $ALT_DATA_DIR$NETWORK"/index.redb"
@@ -92,32 +91,6 @@ then
   rm $DEFAULT_DATA_DIR$NETWORK"/index.redb"
   ln $ALT_DUP_DATA_DIR$NETWORK"/index.redb" $DEFAULT_DATA_DIR$NETWORK"/index.redb"
 
-  # SNAPSHOT PROCESS
-  MOD=$(expr $HEIGHT % 30)
-
-  echo "MOD is $MOD"
-
-  if [ $MOD -eq 0 ];
-  then
-    STALE=$(expr $HEIGHT - 200)
-    STALEMAX=$(expr $HEIGHT - 1000)
-    echo "STALE is $STALE"
-    echo "STALEMAX is $STALEMAX"
-    echo "Creating snapshot"
-    rsync -a $ALT_DATA_DIR$NETWORK"/index.redb" $ALT_DATA_DIR$NETWORK"/index.redb.$HEIGHT"
-
-    while [ $STALE -ge $STALEMAX ]
-    do
-      echo "CURRENT STALE is $STALE"
-      if test -f "$ALT_DATA_DIR$NETWORK/index.redb.$STALE"; then
-        echo "Removing stale $STALE"
-        rm $ALT_DATA_DIR$NETWORK"/index.redb.$STALE"
-        echo "Removed stale"
-      fi
-      ((STALE=STALE-1))
-    done
-  fi
-
   # UNLOCK
   rm "$ALT_DATA_DIR$NETWORK/lock"
 

diff --git a/ord-snapshot.sh b/ord-snapshot.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [ ! -f "$SCRIPT_DIR/.ord-index-config" ]
+then
+  echo "No config file found.."
+  exit
+fi
+
+source "$SCRIPT_DIR/.ord-index-config"
+
+RUNNING=$(ps -fu $UID | grep "ord" | grep "data-dir" | grep "$SNAPSHOT_DATA_DIR" | grep "index-sats index")
+
+if [ ! -z "${RUNNING}" ]; then 
+  echo "Still running.."
+  exit
+fi
+
+if test -f "$SNAPSHOT_DATA_DIR$NETWORK/lock"; then
+  echo "Process not finished.."
+  exit
+fi
+
+mkdir -p $SNAPSHOT_DATA_DIR$NETWORK
+
+# LOCK
+touch "$SNAPSHOT_DATA_DIR$NETWORK/lock"
+
+
+# == SETUP
+
+if [ ! -f $SNAPSHOT_DATA_DIR$NETWORK"/index.redb" ]
+then
+  rsync -a $DEFAULT_DATA_DIR$NETWORK"/index.redb" $SNAPSHOT_DATA_DIR$NETWORK"/index.redb"
+fi
+
+# ==
+
+BITCOIN_CLI_OPTIONS=()
+ORD_CLI_OPTIONS=()
+
+if [ ! -z "${NETWORK}" ]; then
+  if [ $NETWORK = "regtest" ]; then
+    BITCOIN_CLI_OPTIONS=( --regtest )
+    ORD_CLI_OPTIONS=( -r )
+  elif [ $NETWORK = "testnet3" ]; then
+    BITCOIN_CLI_OPTIONS=( --testnet )
+    ORD_CLI_OPTIONS=( -t )
+  else
+    echo "Invalid network"
+    exit
+  fi
+fi
+
+# SETUP END ==
+
+
+HEIGHT=$(bitcoin-cli "${BITCOIN_CLI_OPTIONS[@]}" getblockcount)
+
+echo "HEIGHT is $HEIGHT"
+echo "Begin indexing.."
+
+REORG=$(ord@afwcxx "${ORD_CLI_OPTIONS[@]}" --data-dir=$SNAPSHOT_DATA_DIR --index-sats index 2>&1 | grep -P 'reorg')
+
+echo "REORG is $REORG"
+
+if [ -z "$REORG" ]
+then
+  # SNAPSHOT PROCESS
+  MOD=$(expr $HEIGHT % 30)
+
+  echo "MOD is $MOD"
+
+  if [ $MOD -eq 0 ];
+  then
+    STALE=$(expr $HEIGHT - 200)
+    STALEMAX=$(expr $HEIGHT - 1000)
+    echo "STALE is $STALE"
+    echo "STALEMAX is $STALEMAX"
+    echo "Creating snapshot"
+    rsync -a $SNAPSHOT_DATA_DIR$NETWORK"/index.redb" $SNAPSHOT_DATA_DIR$NETWORK"/index.redb.$HEIGHT"
+
+    while [ $STALE -ge $STALEMAX ]
+    do
+      echo "CURRENT STALE is $STALE"
+      if test -f "$SNAPSHOT_DATA_DIR$NETWORK/index.redb.$STALE"; then
+        echo "Removing stale $STALE"
+        rm $SNAPSHOT_DATA_DIR$NETWORK"/index.redb.$STALE"
+        echo "Removed stale"
+      fi
+      ((STALE=STALE-1))
+    done
+  fi
+
+  # UNLOCK
+  rm "$SNAPSHOT_DATA_DIR$NETWORK/lock"
+
+  if test -f "$SNAPSHOT_DATA_DIR$NETWORK/reorg"; then
+    rm "$SNAPSHOT_DATA_DIR$NETWORK/reorg"
+  fi
+
+  echo "Done.."
+else
+  echo "PENDING: Perform self reorg recovery"
+
+  # UNLOCK
+  rm "$SNAPSHOT_DATA_DIR$NETWORK/lock"
+
+  touch "$SNAPSHOT_DATA_DIR$NETWORK/reorg"
+  echo "Has reorg.."
+fi