TrueBlocks · tjayrush · Sep 3, 2023 · Sep 3, 2023 · Sep 3, 2023 · Sep 3, 2023
diff --git a/CHANGES.md b/CHANGES.md
@@ -4,6 +4,23 @@
 
 This file details changes made to TrueBlocks over time. See the [migration notes](./MIGRATIONS.md) for any required actions you must take to stay up to date.
 
+## v1.1.0 (2023/09/05)
+
+Added `--watch_list`, `--commands`, `--batch_size`, and `--run_count` options to `chifra monitors`.
+Added `--run_count` to `chifra scrape` (for debugging purposes).
+Changed names of some rarely used special blocks in `chifra when`. Breaking, but minimal impact.
+Removes a number of previously deprecated options. `chifra abis --sol`, `chifra names --named`, `chifra names --to_custom`, `chifra transactions --trace`, and `chifra --log_level` for all commands.
+Better error reporting when running against non-tracing nodes.
+Fixed an issue with Content-Type in the server.
+Fixed an issue where user could hit cntl+c during caching and corrupt the database.
+Fixed an issue where scraper was missing some smart contract addresses created during out of gas transactions.
+Better error handling from the RPC.
+Added a verbose mode to `chifra when` to include more specials and a description for each special block.
+Now disallows running `chifra scrape` if the node is not a tracing archive node.
+Near complete re-write of block scraper to fix few bugs and prepare for writing v1.0.0 of the spec.
+Add `--run_count` to both the `chifra scrape` and the `chifra monitors --watch`, hides both because they are intended for debugging only.
+Added `--publisher` option to `chifra init` and `chifra chunks`.
+
 ## v1.0.0 (2023/08/20)
 
 **OUR FIRST OFFICIAL RELEASE!**

diff --git a/docs/content/api/openapi.yaml b/docs/content/api/openapi.yaml
@@ -622,15 +622,6 @@ paths:
           schema:
             type: number
             format: uint64
-        - name: runOnce
-          description: >
-            available with --watch option only, only run the monitor --watch commands once then quit
-          required: false
-          style: form
-          in: query
-          explode: true
-          schema:
-            type: boolean
         - name: sleep
           description: >
             available with --watch option only, the number of seconds to sleep between runs
@@ -2077,6 +2068,14 @@ paths:
           explode: true
           schema:
             type: boolean
+        - name: publisher
+          description: for some query options, the publisher of the index
+          required: false
+          style: form
+          in: query
+          explode: true
+          schema:
+            type: string
         - name: remote
           description: >
             prior to processing, retreive the manifest from the Unchained Index smart contract
@@ -2207,6 +2206,14 @@ paths:
           explode: true
           schema:
             type: boolean
+        - name: publisher
+          description: the publisher of the index to download
+          required: false
+          style: form
+          in: query
+          explode: true
+          schema:
+            type: string
         - name: firstBlock
           description: do not download any chunks earlier than this block
           required: false
@@ -3865,9 +3872,6 @@ components:
         rpcProvider:
           type: string
           description: "A valid RPC provider for the chain"
-        apiProvider:
-          type: string
-          description: "A valid API provider for the explorer"
         remoteExplorer:
           type: string
           description: "A remote explorer for the chain such as Etherscan"

diff --git a/docs/content/chifra/accounts.md b/docs/content/chifra/accounts.md
@@ -243,15 +243,14 @@ Flags:
   -a, --watchlist string   available with --watch option only, a file containing the addresses to watch
   -c, --commands string    available with --watch option only, the file containing the list of commands to apply to each watched address
   -b, --batch_size uint    available with --watch option only, the number of monitors to process in each batch (default 8)
-  -r, --run_once           available with --watch option only, only run the monitor --watch commands once then quit
   -s, --sleep float        available with --watch option only, the number of seconds to sleep between runs (default 14)
   -x, --fmt string         export format, one of [none|json*|txt|csv]
   -v, --verbose            enable verbose output
   -h, --help               display this help screen
 
 Notes:
   - An address must be either an ENS name or start with '0x' and be forty-two characters long.
-  - If no address is presented to the --clean command, all existing monitors are be cleaned.
+  - If no address is presented to the --clean command, all existing monitors will be cleaned.
   - The --watch option requires two additional parameters to be specified: --watchlist and --commands.
   - Addresses provided on the command line are ignored in --watch mode.
   - Providing the value existing to the --watchlist monitors all existing monitor files (see --list).

diff --git a/docs/content/chifra/admin.md b/docs/content/chifra/admin.md
@@ -282,6 +282,7 @@ Flags:
   -c, --check              check the manifest, index, or blooms for internal consistency
   -i, --pin                pin the manifest or each index chunk and bloom
   -p, --publish            publish the manifest to the Unchained Index smart contract
+  -P, --publisher string   for some query options, the publisher of the index (default "trueblocks.eth")
   -r, --remote             prior to processing, retreive the manifest from the Unchained Index smart contract
   -b, --belongs strings    in index mode only, checks the address(es) for inclusion in the given index chunk
   -F, --first_block uint   first block to process (inclusive)
@@ -302,6 +303,7 @@ Notes:
   - The --first_block and --last_block options apply only to addresses, appearances, and index --belongs mode.
   - The --pin option requires a locally running IPFS node or a pinning service API key.
   - The --publish option requires a private key.
+  - The --publisher option is ignored with the --publish option since the sender of the transaction is recorded as the publisher.
 ```
 
 Data models produced by this tool:
@@ -355,6 +357,7 @@ Usage:
 Flags:
   -a, --all                in addition to Bloom filters, download full index chunks (recommended)
   -d, --dry_run            display the results of the download without actually downloading
+  -P, --publisher string   the publisher of the index to download (default "trueblocks.eth")
   -F, --first_block uint   do not download any chunks earlier than this block
   -s, --sleep float        seconds to sleep between downloads
   -v, --verbose            enable verbose output

diff --git a/docs/content/data-model/admin.md b/docs/content/data-model/admin.md
@@ -304,7 +304,6 @@ Chains consist of the following fields:
 | chainId        | The chain id as reported by the RPC                              | uint64 |
 | symbol         | The symbol of the base currency on the chain                     | string |
 | rpcProvider    | A valid RPC provider for the chain                               | string |
-| apiProvider    | A valid API provider for the explorer                            | string |
 | remoteExplorer | A remote explorer for the chain such as Etherscan                | string |
 | localExplorer  | The local explorer for the chain (typically TrueBlocks Explorer) | string |
 | ipfsGateway    | An IPFS gateway for pinning the index if enabled                 | string |

diff --git a/docs/content/readmes/accounts-export.md b/docs/content/readmes/accounts-export.md
@@ -77,7 +77,6 @@ Notes:
 Data models produced by this tool:
 
 - [appearance](/data-model/accounts/#appearance)
-- [reconciliation](/data-model/accounts/#reconciliation)
 - [monitor](/data-model/accounts/#monitor)
 - [appearancecount](/data-model/accounts/#appearancecount)
 - [statement](/data-model/accounts/#statement)

diff --git a/docs/content/readmes/accounts-monitors.md b/docs/content/readmes/accounts-monitors.md
@@ -1,23 +1,48 @@
 ## chifra monitors
 
 <!-- markdownlint-disable MD041 -->
-`chifra monitors`` has two purposes: (1) to `--watch` a set of addresses. This function is in its early
-stages and will be better explained elsewhere. Please see an example of what one may do with
-`chifra monitors --watch`, and (2) allows one to manage existing monitored addresses.
+`chifra monitors` has two purposes: (1) to display information about the current set of monitors, and (2)
+to `--watch` a set of addresses. The `--watch` function allows one to "follow" an address (or set
+of addresses) and keep an off-chain database fresh.
 
-A "monitor" is simply a file on a hard drive that represents the transactional history of a given
-Ethereum address. Monitors are very small, being only the `<block_no><tx_id>` pair representing each
-appearance of an address. Monitor files are only created when a user expresses interest in a
-particular address. In this way, TrueBlock is able to continue to work on small desktop or even
-laptop computers. (See `chifra list`.)
+### Crud commands
 
-You may use the `--delete` command to delete (or `--undelete` if already deleted) an address. The
-monitor is not removed from your computer if you delete it. It is just marked as being deleted
-making it invisible to the TrueBlocks explorer.
+`chifra list` creates a new monitor. See that tool's help file for more information.
+
+The `chifra monitors --delete` command deletes (or `--undelete` if already deleted) an address but does not remove it from your hard drive. The monitor is marked as being deleted, making it invisible to other tools.
 
 Use the `--remove` command to permanently remove a monitor from your computer. This is an
 irreversible operation and requires the monitor to have been previously deleted.
 
+The `--decache` option will remove not only the monitor but all of the cached data associated with
+the monitor (for example, transactions or traces). This is an irreversible operation (except
+for the fact that the cache can be easily re-created with `chifra list <address>`). The monitor need not have been previously deleted.
+
+### Watching addresses
+
+The `--watch` command is special. It starts a long-running process that continually reads the blockchain looking for appearances of the addresses it is instructed to watch. It command requires two additional parameters: `--watchlist <filename>` and `--commands <filename>`.  The `--watchlist` file is simply a list of addresses or ENS names, one per line:
+
+```
+0x5e349eca2dc61abcd9dd99ce94d04136151a09ee
+trueblocks.eth
+0x855b26bc8ebabcdbefe82ee5e9d40d20a1a4c11f
+etc.
+```
+
+You may monitor as many addresses as you wish, however, if the commands you specify take longer than the `--sleep` amount you specify (14 seconds by default), the results are undefined. (Adjust `--sleep` if necessary.)
+
+The `--commands` file may contain a list of any valid `chifra` command that operates on addresses. (Currently `export`, `list`, `state`, `tokens`.) Each command in the `--commands` file is executed once for each address in the `--watchlist` file. The `--commands` file may contain any number of commands, one per line with the above proviso. For example:
+
+```
+chifra list [{ADDRESS}]
+chifra export --logs [{ADDRESS}]
+etc.
+```
+
+The `[{ADDRESS}]` token is a stand-in for all addresses in the `--watchlist`. Addresses are processed in groups of `batch_size` (default 8).
+
+Invalid commands or invalid addresses are ignored. If a command fails, the process continues with the next command. If a command fails for a particular address, the process continues with the next address. A warning is generated.
+
 ```[plaintext]
 Purpose:
   Add, remove, clean, and list address monitors.
@@ -29,20 +54,26 @@ Arguments:
   addrs - one or more addresses (0x...) to process
 
 Flags:
-      --delete        delete a monitor, but do not remove it
-      --undelete      undelete a previously deleted monitor
-      --remove        remove a previously deleted monitor
-  -C, --clean         clean (i.e. remove duplicate appearances) from monitors
-  -l, --list          list monitors in the cache (--verbose for more detail)
-  -w, --watch         continually scan for new blocks and extract data for monitored addresses
-  -s, --sleep float   seconds to sleep between monitor passes (default 14)
-  -x, --fmt string    export format, one of [none|json*|txt|csv]
-  -v, --verbose       enable verbose output
-  -h, --help          display this help screen
+      --delete             delete a monitor, but do not remove it
+      --undelete           undelete a previously deleted monitor
+      --remove             remove a previously deleted monitor
+  -C, --clean              clean (i.e. remove duplicate appearances) from monitors
+  -l, --list               list monitors in the cache (--verbose for more detail)
+  -w, --watch              continually scan for new blocks and extract data as per the command file
+  -a, --watchlist string   available with --watch option only, a file containing the addresses to watch
+  -c, --commands string    available with --watch option only, the file containing the list of commands to apply to each watched address
+  -b, --batch_size uint    available with --watch option only, the number of monitors to process in each batch (default 8)
+  -s, --sleep float        available with --watch option only, the number of seconds to sleep between runs (default 14)
+  -x, --fmt string         export format, one of [none|json*|txt|csv]
+  -v, --verbose            enable verbose output
+  -h, --help               display this help screen
 
 Notes:
   - An address must be either an ENS name or start with '0x' and be forty-two characters long.
-  - If no address is presented to the --clean command, all monitors will be cleaned.
+  - If no address is presented to the --clean command, all existing monitors will be cleaned.
+  - The --watch option requires two additional parameters to be specified: --watchlist and --commands.
+  - Addresses provided on the command line are ignored in --watch mode.
+  - Providing the value existing to the --watchlist monitors all existing monitor files (see --list).
 ```
 
 Data models produced by this tool:

diff --git a/docs/content/readmes/chainstate-state.md b/docs/content/readmes/chainstate-state.md
@@ -46,8 +46,8 @@ Notes:
 
 Data models produced by this tool:
 
-- [callresult](/data-model/chainstate/#callresult)
-- [ethcall](/data-model/chainstate/#ethcall)
+- [result](/data-model/chainstate/#result)
+- [state](/data-model/chainstate/#state)
 
 Links:
 

diff --git a/docs/readmes/accounts-monitors.md b/docs/readmes/accounts-monitors.md
@@ -63,15 +63,14 @@ Flags:
   -a, --watchlist string   available with --watch option only, a file containing the addresses to watch
   -c, --commands string    available with --watch option only, the file containing the list of commands to apply to each watched address
   -b, --batch_size uint    available with --watch option only, the number of monitors to process in each batch (default 8)
-  -r, --run_once           available with --watch option only, only run the monitor --watch commands once then quit
   -s, --sleep float        available with --watch option only, the number of seconds to sleep between runs (default 14)
   -x, --fmt string         export format, one of [none|json*|txt|csv]
   -v, --verbose            enable verbose output
   -h, --help               display this help screen
 
 Notes:
   - An address must be either an ENS name or start with '0x' and be forty-two characters long.
-  - If no address is presented to the --clean command, all existing monitors are be cleaned.
+  - If no address is presented to the --clean command, all existing monitors will be cleaned.
   - The --watch option requires two additional parameters to be specified: --watchlist and --commands.
   - Addresses provided on the command line are ignored in --watch mode.
   - Providing the value existing to the --watchlist monitors all existing monitor files (see --list).

diff --git a/docs/readmes/admin-chunks.md b/docs/readmes/admin-chunks.md
@@ -23,6 +23,7 @@ Flags:
   -c, --check              check the manifest, index, or blooms for internal consistency
   -i, --pin                pin the manifest or each index chunk and bloom
   -p, --publish            publish the manifest to the Unchained Index smart contract
+  -P, --publisher string   for some query options, the publisher of the index (default "trueblocks.eth")
   -r, --remote             prior to processing, retreive the manifest from the Unchained Index smart contract
   -b, --belongs strings    in index mode only, checks the address(es) for inclusion in the given index chunk
   -F, --first_block uint   first block to process (inclusive)
@@ -43,6 +44,7 @@ Notes:
   - The --first_block and --last_block options apply only to addresses, appearances, and index --belongs mode.
   - The --pin option requires a locally running IPFS node or a pinning service API key.
   - The --publish option requires a private key.
+  - The --publisher option is ignored with the --publish option since the sender of the transaction is recorded as the publisher.
 ```
 
 Data models produced by this tool:

diff --git a/docs/readmes/admin-init.md b/docs/readmes/admin-init.md
@@ -32,6 +32,7 @@ Usage:
 Flags:
   -a, --all                in addition to Bloom filters, download full index chunks (recommended)
   -d, --dry_run            display the results of the download without actually downloading
+  -P, --publisher string   the publisher of the index to download (default "trueblocks.eth")
   -F, --first_block uint   do not download any chunks earlier than this block
   -s, --sleep float        seconds to sleep between downloads
   -v, --verbose            enable verbose output

diff --git a/docs/templates/api/components.txt b/docs/templates/api/components.txt
@@ -1494,9 +1494,6 @@ components:
         rpcProvider:
           type: string
           description: "A valid RPC provider for the chain"
-        apiProvider:
-          type: string
-          description: "A valid API provider for the explorer"
         remoteExplorer:
           type: string
           description: "A remote explorer for the chain such as Etherscan"

diff --git a/sdk/python/src/_chunks.py b/sdk/python/src/_chunks.py
@@ -10,6 +10,7 @@
     "check": {"hotkey": "-c", "type": "switch"},
     "pin": {"hotkey": "-i", "type": "switch"},
     "publish": {"hotkey": "-p", "type": "switch"},
+    "publisher": {"hotkey": "-P", "type": "flag"},
     "remote": {"hotkey": "-r", "type": "switch"},
     "belongs": {"hotkey": "-b", "type": "flag"},
     "firstBlock": {"hotkey": "-F", "type": "flag"},

diff --git a/sdk/python/src/_init.py b/sdk/python/src/_init.py
@@ -9,6 +9,7 @@
 initOpts = {
     "all": {"hotkey": "-a", "type": "switch"},
     "dryRun": {"hotkey": "-d", "type": "switch"},
+    "publisher": {"hotkey": "-P", "type": "flag"},
     "firstBlock": {"hotkey": "-F", "type": "flag"},
     "sleep": {"hotkey": "-s", "type": "flag"},
     "fmt": {"hotkey": "-x", "type": "flag"},

diff --git a/sdk/python/src/_monitors.py b/sdk/python/src/_monitors.py
@@ -16,7 +16,6 @@
     "watchlist": {"hotkey": "-a", "type": "flag"},
     "commands": {"hotkey": "-c", "type": "flag"},
     "batchSize": {"hotkey": "-b", "type": "flag"},
-    "runOnce": {"hotkey": "-r", "type": "switch"},
     "sleep": {"hotkey": "-s", "type": "flag"},
     "fmt": {"hotkey": "-x", "type": "flag"},
     "verbose:": {"hotkey": "-v", "type": "switch"},

diff --git a/sdk/typescript/src/paths/chunks.ts b/sdk/typescript/src/paths/chunks.ts
@@ -13,6 +13,7 @@ export function getChunks(
     check?: boolean,
     pin?: boolean,
     publish?: boolean,
+    publisher?: address,
     remote?: boolean,
     belongs?: address[],
     firstBlock?: blknum,

diff --git a/sdk/typescript/src/paths/init.ts b/sdk/typescript/src/paths/init.ts
@@ -4,12 +4,13 @@
  * This file was generated with makeClass --sdk. Do not edit it.
  */
 import * as ApiCallers from '../lib/api_callers';
-import { blknum, ChunkRecord, double, Manifest } from '../types';
+import { address, blknum, ChunkRecord, double, Manifest } from '../types';
 
 export function getInit(
   parameters?: {
     all?: boolean,
     dryRun?: boolean,
+    publisher?: address,
     firstBlock?: blknum,
     sleep?: double,
     chain: string,

diff --git a/sdk/typescript/src/paths/monitors.ts b/sdk/typescript/src/paths/monitors.ts
@@ -18,7 +18,6 @@ export function getMonitors(
     watchlist?: string,
     commands?: string,
     batchSize?: uint64,
-    runOnce?: boolean,
     sleep?: double,
     chain: string,
     noHeader?: boolean,

diff --git a/sdk/typescript/src/types/chain.ts b/sdk/typescript/src/types/chain.ts
@@ -10,7 +10,6 @@ export type Chain = {
   chainId: uint64
   symbol: string
   rpcProvider: string
-  apiProvider: string
   remoteExplorer: string
   localExplorer: string
   ipfsGateway: string