Merge pull request #29 from pinax-network/feature/antelope-parquet

Antelope Parquet
pinax-network · Nov 21, 2024 · 9eb2f54 · 9eb2f54
2 parents 306c4f4 + 72baca3
commit 9eb2f54
Show file tree

Hide file tree

Showing 30 changed files with 1,087 additions and 1,535 deletions.
diff --git a/blocks/antelope/Cargo.toml b/blocks/antelope/Cargo.toml
@@ -9,5 +9,6 @@ crate-type = ["cdylib"]
 [dependencies]
 common = { path = "../../common" }
 substreams-antelope = { workspace = true }
-substreams-database-change = { workspace = true }
 substreams = { workspace = true }
+prost = { workspace = true }
+prost-types = { workspace = true }
diff --git a/blocks/antelope/Makefile b/blocks/antelope/Makefile
@@ -1,57 +1,29 @@
 .PHONY: all
 all:
 	make build
-	make pack
-	make graph
-	make info
 
 .PHONY: build
 build:
 	cargo build --target wasm32-unknown-unknown --release
-
-.PHONY: pack
-pack:
 	substreams pack
-
-.PHONY: graph
-graph:
-	substreams graph
-
-.PHONY: info
-info:
 	substreams info
-
-.PHONY: run
-run:
-	substreams run -e eos.substreams.pinax.network:443 ch_out -s 386946801 -t 386946802 -o json
+	substreams graph
 
 .PHONY: gui
 gui:
-	substreams gui -e eos.substreams.pinax.network:443 ch_out -s 386946801 -t 386946802
-
-.PHONY: sql-setup
-sql-setup:
-	# EVM blocks
-	substreams-sink-sql setup clickhouse://default:default@localhost:9000/eos substreams.yaml
-	substreams-sink-sql setup clickhouse://default:default@localhost:9000/wax substreams.yaml
-	substreams-sink-sql setup clickhouse://default:default@localhost:9000/telos substreams.yaml
-
-# Antelope blocks
-.PHONY: sql-run-eos
-sql-run-eos:
-	substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 386841287:387014085 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 1 --development-mode
+	substreams gui -e eos.substreams.pinax.network:443 map_events -s 386946801 -t 386946802
 
-.PHONY: sql-run-eos-test
-sql-run-eos-test:
-	substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 386946800:386946810 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 1 --development-mode
+.PHONY: protogen
+protogen:
+	substreams protogen
 
-.PHONY: sql-run-eos-blocks
-sql-run-eos-blocks:
-	substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 2: --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 100 --params ch_out=blocks
+.PHONY: parquet
+parquet:
+	substreams-sink-files run eos.substreams.pinax.network:443 substreams.yaml map_events './out' 386841287:386841787 --encoder parquet --file-block-count 100 --development-mode
 
-.PHONY: sql-run-wax
-sql-run-wax:
-	substreams-sink-sql run clickhouse://default:default@localhost:9000/wax substreams.yaml -e wax.substreams.pinax.network:443 322339177:322511977 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 100 --development-mode
+.PHONY: parquet-wax
+parquet-wax:
+	substreams-sink-files run wax.substreams.pinax.network:443 substreams.yaml map_events './out' 32233917:32234417 --encoder parquet --file-block-count 100 --development-mode
 
 .PHONY: deploy
 deploy:

diff --git a/blocks/antelope/README.md b/blocks/antelope/README.md
@@ -4,7 +4,7 @@
 > [`sf.antelope.type.v1.Block`](https://buf.build/pinax/firehose-antelope/docs/main:sf.antelope.type.v1)
 
 - [x] **Blocks**
-  - [ ] **Savanna Merkle Roots**
+  - [x] **Savanna Merkle Roots**
 - [x] **Transactions**
   - [x] **Feature Operations**
   - [x] **Permission Operations**
@@ -21,13 +21,23 @@
   - [x] **Account RAM Deltas**
 - [x] **Database Operations**
 
-## Substreams Graph
+## Graph
 
 ```mermaid
 graph TD;
-  raw[sf.antelope.type.v1.Block];
-  raw --> blocks;
-  raw --> transactions;
-  raw --> actions;
-  raw --> db_ops;
+  map_events[map: map_events];
+  sf.substreams.v1.Clock[source: sf.substreams.v1.Clock] --> map_events;
+  sf.antelope.type.v1.Block[source: sf.antelope.type.v1.Block] --> map_events;
+```
+
+## Modules
+
+```bash
+Name: map_events
+Initial block: 0
+Kind: map
+Input: source: sf.substreams.v1.Clock
+Input: source: sf.antelope.type.v1.Block
+Output Type: proto:antelope.Events
+Hash: 2ee4412cab8f461aaf8de27c2e970970a545117c
 ```