From 9fa1253b73b0af019974aea267a533253c317ca4 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Fri, 8 Jul 2022 15:33:32 +0200 Subject: [PATCH 1/6] store solver-list --- dune_api_scripts/requirements.txt | 2 +- dune_api_scripts/store/solver_list.py | 61 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 dune_api_scripts/store/solver_list.py diff --git a/dune_api_scripts/requirements.txt b/dune_api_scripts/requirements.txt index cc1f541..4ce7557 100644 --- a/dune_api_scripts/requirements.txt +++ b/dune_api_scripts/requirements.txt @@ -1,6 +1,6 @@ requests>=2.27.1 pylint==2.11.1 pytest==6.2.5 -duneapi==3.0.3 +duneapi==3.0.7 python-dotenv==0.20.0 diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py new file mode 100644 index 0000000..a088667 --- /dev/null +++ b/dune_api_scripts/store/solver_list.py @@ -0,0 +1,61 @@ +"""Modifies and executed dune query for today's data""" +from __future__ import annotations + +import dataclasses +import json +import os +from dataclasses import dataclass +from typing import Any + +from duneapi.api import DuneAPI +from duneapi.types import DuneQuery, Network +from duneapi.types import Address + + +class EnhancedJSONEncoder(json.JSONEncoder): + def default(self, o): + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +@dataclass +class Solver: + address: str + environment: str + name: str + active: bool + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Solver: + return cls( + address=Address(data["address"]).address, + environment=data["environment"], + name=data["name"], + active=data["active"], + ) + + +def store_solver_list(dune: DuneAPI) -> list[Solver]: + # TODO - fetch for both networks and merge JSON content with chainID + raw_solver_list = dune.fetch( + DuneQuery.from_environment( + name="Solver List", + description="", + raw_sql="select * from gnosis_protocol_v2.view_solvers", + network=Network.MAINNET, + parameters=[], + ) + ) + solver_list = [Solver.from_dict(rec) for rec in raw_solver_list] + + filename = os.path.join(os.environ["DUNE_DATA_FOLDER"], "solvers.json") + with open(filename, "w+", encoding="utf-8") as f: + json.dump(solver_list, f, ensure_ascii=False, indent=4, cls=EnhancedJSONEncoder) + + return solver_list + + +if __name__ == "__main__": + + store_solver_list(dune=DuneAPI.new_from_environment()) From 644e0029c4ea09a4ce8c29203297caa61a0d7052 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 11 Jul 2022 10:50:17 +0200 Subject: [PATCH 2/6] add link to dependent PR --- dune_api_scripts/store/solver_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py index a088667..90d10c6 100644 --- a/dune_api_scripts/store/solver_list.py +++ b/dune_api_scripts/store/solver_list.py @@ -38,6 +38,7 @@ def from_dict(cls, data: dict[str, Any]) -> Solver: def store_solver_list(dune: DuneAPI) -> list[Solver]: # TODO - fetch for both networks and merge JSON content with chainID + # Waiting on https://github.com/duneanalytics/abstractions/pull/1268q raw_solver_list = dune.fetch( DuneQuery.from_environment( name="Solver List", From 106140e4ab2ddeda01a07471b7fad7a603455660 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 11 Jul 2022 10:50:29 +0200 Subject: [PATCH 3/6] typo --- dune_api_scripts/store/solver_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py index 90d10c6..2af9c72 100644 --- a/dune_api_scripts/store/solver_list.py +++ b/dune_api_scripts/store/solver_list.py @@ -38,7 +38,7 @@ def from_dict(cls, data: dict[str, Any]) -> Solver: def store_solver_list(dune: DuneAPI) -> list[Solver]: # TODO - fetch for both networks and merge JSON content with chainID - # Waiting on https://github.com/duneanalytics/abstractions/pull/1268q + # Waiting on https://github.com/duneanalytics/abstractions/pull/1268 raw_solver_list = dune.fetch( DuneQuery.from_environment( name="Solver List", From b97b63858cda014beb78c994a171464061027926 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Tue, 12 Jul 2022 10:38:27 +0200 Subject: [PATCH 4/6] store solver list directly from dune response - both networks --- dune_api_scripts/store/solver_list.py | 63 +++++++++++---------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py index 2af9c72..a262371 100644 --- a/dune_api_scripts/store/solver_list.py +++ b/dune_api_scripts/store/solver_list.py @@ -12,51 +12,40 @@ from duneapi.types import Address -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o): - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - -@dataclass -class Solver: - address: str - environment: str - name: str - active: bool - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> Solver: - return cls( - address=Address(data["address"]).address, - environment=data["environment"], - name=data["name"], - active=data["active"], - ) - - -def store_solver_list(dune: DuneAPI) -> list[Solver]: - # TODO - fetch for both networks and merge JSON content with chainID - # Waiting on https://github.com/duneanalytics/abstractions/pull/1268 - raw_solver_list = dune.fetch( +SOLVER_QUERY = """ +select + concat('0x', encode(address, 'hex')) as address, + environment, + name, + active +from gnosis_protocol_v2.view_solvers +""" + +SHORT_NAMES = { + Network.MAINNET: "mainnet", + Network.GCHAIN: "gc", +} + + +def store_solver_list(dune: DuneAPI, network: Network) -> list[dict[str, str]]: + solver_list = dune.fetch( DuneQuery.from_environment( name="Solver List", description="", - raw_sql="select * from gnosis_protocol_v2.view_solvers", - network=Network.MAINNET, + raw_sql=SOLVER_QUERY, + network=network, parameters=[], ) ) - solver_list = [Solver.from_dict(rec) for rec in raw_solver_list] - - filename = os.path.join(os.environ["DUNE_DATA_FOLDER"], "solvers.json") + filename = os.path.join( + os.environ["DUNE_DATA_FOLDER"], f"{SHORT_NAMES[network]}-solvers.json" + ) with open(filename, "w+", encoding="utf-8") as f: - json.dump(solver_list, f, ensure_ascii=False, indent=4, cls=EnhancedJSONEncoder) - + json.dump(solver_list, f, ensure_ascii=False, indent=4) return solver_list if __name__ == "__main__": - - store_solver_list(dune=DuneAPI.new_from_environment()) + dune_connection = DuneAPI.new_from_environment() + for chain in [Network.MAINNET, Network.GCHAIN]: + store_solver_list(dune=dune_connection, network=chain) From 38d63b4a6763dfc08bb2a32f7b708de84a6dc948 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Tue, 12 Jul 2022 10:40:53 +0200 Subject: [PATCH 5/6] remove unused imports --- dune_api_scripts/store/solver_list.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py index a262371..8740ef0 100644 --- a/dune_api_scripts/store/solver_list.py +++ b/dune_api_scripts/store/solver_list.py @@ -1,15 +1,11 @@ """Modifies and executed dune query for today's data""" from __future__ import annotations -import dataclasses import json import os -from dataclasses import dataclass -from typing import Any from duneapi.api import DuneAPI from duneapi.types import DuneQuery, Network -from duneapi.types import Address SOLVER_QUERY = """ From a400251c5e7c9d70cd81b296f101f39de393d60b Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Wed, 13 Jul 2022 11:55:49 +0200 Subject: [PATCH 6/6] move short names to constants file --- dune_api_scripts/constants.py | 7 +++++++ dune_api_scripts/store/solver_list.py | 7 ++----- 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 dune_api_scripts/constants.py diff --git a/dune_api_scripts/constants.py b/dune_api_scripts/constants.py new file mode 100644 index 0000000..c5e07cc --- /dev/null +++ b/dune_api_scripts/constants.py @@ -0,0 +1,7 @@ +"""A collection of global constants for the project.""" +from duneapi.types import Network + +NETWORK_SHORT_NAMES = { + Network.MAINNET: "mainnet", + Network.GCHAIN: "gc", +} diff --git a/dune_api_scripts/store/solver_list.py b/dune_api_scripts/store/solver_list.py index 8740ef0..b465acc 100644 --- a/dune_api_scripts/store/solver_list.py +++ b/dune_api_scripts/store/solver_list.py @@ -7,6 +7,7 @@ from duneapi.api import DuneAPI from duneapi.types import DuneQuery, Network +from dune_api_scripts.constants import NETWORK_SHORT_NAMES as SHORT_NAMES SOLVER_QUERY = """ select @@ -17,11 +18,6 @@ from gnosis_protocol_v2.view_solvers """ -SHORT_NAMES = { - Network.MAINNET: "mainnet", - Network.GCHAIN: "gc", -} - def store_solver_list(dune: DuneAPI, network: Network) -> list[dict[str, str]]: solver_list = dune.fetch( @@ -38,6 +34,7 @@ def store_solver_list(dune: DuneAPI, network: Network) -> list[dict[str, str]]: ) with open(filename, "w+", encoding="utf-8") as f: json.dump(solver_list, f, ensure_ascii=False, indent=4) + print(f"solver list written to {filename}") return solver_list