From 8d8394a0d755fb45030fd86a2eb6e8d4b1d9bb6f Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:05:39 -0400 Subject: [PATCH 1/7] Update dependencies --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 21258ba..0e28df3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,12 +16,12 @@ docutils==0.20.1 exceptiongroup==1.1.2 fonttools==4.40.0 frozenlist==1.3.3 -fsspec==2023.6.0 +fsspec==2024.9.0 idna==3.4 importlib-metadata==6.7.0 importlib-resources==5.13.0 iniconfig==2.0.0 -ipfsspec==0.4.0 +ipfsspec==0.5.2 jaraco.classes==3.2.3 jsonschema==4.18.0 jsonschema-specifications==2023.6.1 From a3f0eef35aa0a884e910f93d5bc234d76b9686fd Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:15:54 -0400 Subject: [PATCH 2/7] chore: Update IPFS gateway environment variable handling The code changes in this commit update the handling of the IPFS gateway environment variable. Previously, the remote gateway URLs were appended to the existing value of the `IPFS_GATEWAY` environment variable. Now, the code checks if the variable already exists and appends the local gateway URL with the specified gateway port. If the variable doesn't exist, it creates a new one with the local gateway URL. This change is in accordance with modifications to spec IPIP-280 --- ipfs_stac/client.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/ipfs_stac/client.py b/ipfs_stac/client.py index bb13fd8..3668126 100644 --- a/ipfs_stac/client.py +++ b/ipfs_stac/client.py @@ -17,16 +17,29 @@ import rasterio from yaspin import yaspin + +# Global Variables +ENV_VAR_NAME = "IPFS_GATEWAY" + def ensure_data_fetched(func): def wrapper(self, *args, **kwargs): if self.data is None: print("Data for asset has not been fetched yet. Fetching now...") self.fetch() return func(self, *args, **kwargs) + return wrapper + class Web3: - def __init__(self, local_gateway=None, api_port=5001, stac_endpoint=None, remote_gateways=None) -> None: + def __init__( + self, + local_gateway=None, + api_port=5001, + gateway_port=8080, + stac_endpoint=None, + remote_gateways=None, + ) -> None: """ web3 client constructor @@ -38,14 +51,25 @@ def __init__(self, local_gateway=None, api_port=5001, stac_endpoint=None, remote if api_port is None: raise ValueError("api_port must be set") + + if gateway_port is None: + raise ValueError("gateway_port must be set") + self.api_port = api_port + self.gateway_port = gateway_port if self.local_gateway: self.startDaemon() - # Remote_gateways is of type List[str] - if remote_gateways: - os.environ["IPFSSPEC_GATEWAYS"] = f'IPFSSPEC_GATEWAYS="http://{self.local_gateway}:{self.api_port},https://ipfs.io,https://gateway.pinata.cloud,https://cloudflare-ipfs.com,https://dweb.link",{remote_gateways.join(",")}' + # Check if the remote gateway env variable already exists + if ENV_VAR_NAME in os.environ: + os.environ[ENV_VAR_NAME] += ( + os.pathsep + f"http://{self.local_gateway}:{self.gateway_port}" + ) + else: + os.environ[ENV_VAR_NAME] = ( + f"http://{self.local_gateway}:{self.gateway_port}" + ) def forceLocalNode(self) -> None: """ From ea6c35b0b06a4b48adf48d299fd717c31ead5c00 Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:31:02 -0400 Subject: [PATCH 3/7] revised gateway env variable name Revised handling of adding env variables. Note `gateway.pinata.cloud` has been removed. See this issue for more details https://github.com/fsspec/ipfsspec/issues/17#issuecomment-1160571059 --- ipfs_stac/client.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/ipfs_stac/client.py b/ipfs_stac/client.py index 3668126..934b356 100644 --- a/ipfs_stac/client.py +++ b/ipfs_stac/client.py @@ -20,6 +20,12 @@ # Global Variables ENV_VAR_NAME = "IPFS_GATEWAY" +REMOTE_GATEWAYS = [ + "https://ipfs.io", + "https://cloudflare-ipfs.com", + "https://dweb.link", +] + def ensure_data_fetched(func): def wrapper(self, *args, **kwargs): @@ -34,7 +40,7 @@ def wrapper(self, *args, **kwargs): class Web3: def __init__( self, - local_gateway=None, + local_gateway='127.0.0.1', api_port=5001, gateway_port=8080, stac_endpoint=None, @@ -71,15 +77,28 @@ def __init__( f"http://{self.local_gateway}:{self.gateway_port}" ) + # Add default remote gateways + for gateway in REMOTE_GATEWAYS: + os.environ[ENV_VAR_NAME] = os.environ[ENV_VAR_NAME] + ":" + gateway + + # Extend additional remote gateways to the environment variable + if remote_gateways: + os.environ[ENV_VAR_NAME] += os.pathsep + remote_gateways + + def forceLocalNode(self) -> None: """ Forces the use of local node through env file This function needs to be refactored slightly -> currently overwrites .env file which is unideal if user has other variables configured """ if self.local_gateway is None: - os.environ["IPFSSPEC_GATEWAYS"] = f'IPFSSPEC_GATEWAYS="http://{self.local_gateway}:{self.api_port},https://ipfs.io,https://gateway.pinata.cloud,https://cloudflare-ipfs.com,https://dweb.link"' + os.environ["ENV_VAR_NAME"] = ( + f'ENV_VAR_NAME="http://{self.local_gateway}:{self.api_port},https://ipfs.io,https://gateway.pinata.cloud,https://cloudflare-ipfs.com,https://dweb.link"' + ) else: - os.environ["IPFSSPEC_GATEWAYS"] = f'IPFSSPEC_GATEWAYS="{self.local_gateway}"' + os.environ["IPFSSPEC_GATEWAYS"] = ( + f'IPFSSPEC_GATEWAYS="{self.local_gateway}"' + ) def startDaemon(self) -> None: """ From 9bfb7247e6144d5f92fb4d5af8a719e74569c05d Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:32:03 -0400 Subject: [PATCH 4/7] Revised startDaemon method --- ipfs_stac/client.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ipfs_stac/client.py b/ipfs_stac/client.py index 934b356..74d4750 100644 --- a/ipfs_stac/client.py +++ b/ipfs_stac/client.py @@ -100,22 +100,21 @@ def forceLocalNode(self) -> None: f'IPFSSPEC_GATEWAYS="{self.local_gateway}"' ) - def startDaemon(self) -> None: + def startDaemon(self) -> None: """ Starts Kubo CLI Daemon if not already running """ + _process = subprocess.Popen(["ipfs", "daemon"]) try: - requests.get(f"http://{self.local_gateway}:{self.api_port}/") + heartbeat_response = requests.post( + f"http://{self.local_gateway}:{self.api_port}/api/v0/id" + ) + if heartbeat_response.status_code != 200: + warnings.warn( + "IPFS Daemon is running but still can't connect. Check your IPFS configuration." + ) except requests.exceptions.ConnectionError: - warnings.warn("IPFS Daemon is not running... Attempting to launch") - subprocess.Popen(["ipfs", "daemon"]) - - time.sleep(5) - - try: - requests.get(f"http://{self.local_gateway}:{self.api_port}/") - except requests.exceptions.ConnectionError: - raise Exception("Failed to start IPFS daemon") + raise Exception("Failed to start IPFS daemon") def getFromCID(self, cid: str) -> bytes: """ From 6a5acc397daaedfd745c32f2c1db85445d1b4120 Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:50:23 -0400 Subject: [PATCH 5/7] Adding remote gateways to env causes error --- ipfs_stac/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ipfs_stac/client.py b/ipfs_stac/client.py index 74d4750..4b37be6 100644 --- a/ipfs_stac/client.py +++ b/ipfs_stac/client.py @@ -77,13 +77,13 @@ def __init__( f"http://{self.local_gateway}:{self.gateway_port}" ) - # Add default remote gateways - for gateway in REMOTE_GATEWAYS: - os.environ[ENV_VAR_NAME] = os.environ[ENV_VAR_NAME] + ":" + gateway + # # Add default remote gateways + # for gateway in REMOTE_GATEWAYS: + # os.environ[ENV_VAR_NAME] = os.environ[ENV_VAR_NAME] + ":" + gateway - # Extend additional remote gateways to the environment variable - if remote_gateways: - os.environ[ENV_VAR_NAME] += os.pathsep + remote_gateways + # # Extend additional remote gateways to the environment variable + # if remote_gateways: + # os.environ[ENV_VAR_NAME] += os.pathsep + remote_gateways def forceLocalNode(self) -> None: From 7f596a4be4aa5e781a38ada87e60c618952b856d Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:51:27 -0400 Subject: [PATCH 6/7] Mofided for future update --- ipfs_stac/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipfs_stac/client.py b/ipfs_stac/client.py index 4b37be6..0d51f70 100644 --- a/ipfs_stac/client.py +++ b/ipfs_stac/client.py @@ -79,7 +79,7 @@ def __init__( # # Add default remote gateways # for gateway in REMOTE_GATEWAYS: - # os.environ[ENV_VAR_NAME] = os.environ[ENV_VAR_NAME] + ":" + gateway + # os.environ[ENV_VAR_NAME] = os.environ[ENV_VAR_NAME] + os.pathsep + gateway # # Extend additional remote gateways to the environment variable # if remote_gateways: From 7c6e3e54d6b07181144b5de5870d80402286d843 Mon Sep 17 00:00:00 2001 From: SethDocherty Date: Wed, 25 Sep 2024 13:57:23 -0400 Subject: [PATCH 7/7] Updated tests with modifications to Web3 class --- tests/test_client.py | 3 ++- tests/test_client_integration.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 38a184c..2484df8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -17,6 +17,7 @@ LOCAL_GATEWAY = "127.0.0.1" API_PORT = 5001 STAC_ENDPOINT = "fake_endpoint" +GATEWAY_PORT = 8081 @patch('requests.get') @@ -50,7 +51,7 @@ def test_to_pd_df(mock_getFromCID, mock_get): class TestWeb3(SetUp): def setUp(self): - self.client = Web3(local_gateway=LOCAL_GATEWAY, stac_endpoint=STAC_ENDPOINT) + self.client = Web3(local_gateway=LOCAL_GATEWAY, stac_endpoint=STAC_ENDPOINT, gateway_port=GATEWAY_PORT) def test_init(self): self.assertEqual(self.client.local_gateway, LOCAL_GATEWAY) diff --git a/tests/test_client_integration.py b/tests/test_client_integration.py index a91623d..87b4f18 100644 --- a/tests/test_client_integration.py +++ b/tests/test_client_integration.py @@ -12,10 +12,11 @@ LOCAL_GATEWAY = "127.0.0.1" API_PORT = 5001 STAC_ENDPOINT = "http://ec2-54-172-212-55.compute-1.amazonaws.com/api/v1/pgstac/" +GATEWAY_PORT = 8081 class TestIntegration(SetUp): def setUp(self): - self.client = Web3(local_gateway=LOCAL_GATEWAY, stac_endpoint=STAC_ENDPOINT) + self.client = Web3(local_gateway=LOCAL_GATEWAY, stac_endpoint=STAC_ENDPOINT, gateway_port=GATEWAY_PORT) # Colosseum, Rome, Italy self.bbox = [12.490827, 41.889249, 12.494162, 41.891876]