refactor: ♻️ Using generated logger and unified ForexData fetch()

awhipp · Apr 29, 2024 · 5d45e59 · 5d45e59 · github-actions · Apr 29, 2024
1 parent 89f0813
commit 5d45e59
Show file tree

Hide file tree

Showing 12 changed files with 71 additions and 87 deletions.
diff --git a/foresight/indicator_services/indicator.py b/foresight/indicator_services/indicator.py
@@ -2,23 +2,17 @@
 
 import datetime
 import json
-
-# Setup logging and log timestamp prepend
-import logging
 import time
 
 import pandas as pd
 from boto3_type_annotations.sqs import Client
 from utils.aws import get_client
 from utils.database import TimeScaleService
 
+from foresight.utils.logger import generate_logger
+
 
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger = logging.getLogger(__name__)
+logger = generate_logger(name=__name__)
 
 
 class Indicator:

diff --git a/foresight/indicator_services/moving_average_indicator.py b/foresight/indicator_services/moving_average_indicator.py
@@ -1,16 +1,12 @@
-# Setup logging and log timestamp prepend
-import logging
+"""Moving average indicator class"""
 
 import pandas as pd
-from indicator import Indicator
 
+from foresight.indicator_services.indicator import Indicator
+from foresight.utils.logger import generate_logger
 
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger = logging.getLogger(__name__)
+
+logger = generate_logger(name=__name__)
 
 
 class MovingAverageIndicator(Indicator):

diff --git a/foresight/interface_service/app.py b/foresight/interface_service/app.py
@@ -1,7 +1,6 @@
 # app.py
 
 import json
-import logging
 import os
 
 import dotenv
@@ -10,20 +9,15 @@
 from flask import render_template
 
 from foresight.utils.database import TimeScaleService
+from foresight.utils.logger import generate_logger
 
 
 app = Flask(__name__)
 
 # Load environment variables
 dotenv.load_dotenv()
 
-# Setup logging and log timestamp prepend
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger = logging.getLogger(__name__)
+logger = generate_logger(name=__name__)
 
 
 def get_latest() -> list[dict]:

diff --git a/foresight/stream_service/app.py b/foresight/stream_service/app.py
@@ -12,9 +12,9 @@
 import dotenv
 import requests
 
-from foresight.stream_service.models.forex_data import ForexData
 from foresight.stream_service.models.stream import Stream
 from foresight.utils.logger import generate_logger
+from foresight.utils.models.forex_data import ForexData
 
 
 logger = generate_logger(__name__)

diff --git a/foresight/stream_service/models/stream.py b/foresight/stream_service/models/stream.py
@@ -7,8 +7,8 @@
 
 from pydantic import BaseModel
 
-from foresight.stream_service.models.forex_data import ForexData
 from foresight.stream_service.models.pricing import Pricing
+from foresight.utils.models.forex_data import ForexData
 
 
 class Stream(BaseModel):

diff --git a/foresight/utils/database.py b/foresight/utils/database.py
@@ -1,22 +1,18 @@
 """Provides a singleton class to interact with the TimescaleDB database."""
 
-import logging
 import os
 
 import dotenv
 import psycopg2
 import psycopg2.extras
 
+from foresight.utils.logger import generate_logger
+
 
 dotenv.load_dotenv(".env")
 
 
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger = logging.getLogger(__name__)
+logger = generate_logger(name=__name__)
 
 
 class TimeScaleService:

diff --git a/foresight/utils/models/__init__.py b/foresight/utils/models/__init__.py
diff --git a/...sight/stream_service/models/forex_data.py → foresight/utils/models/forex_data.py b/...sight/stream_service/models/forex_data.py → foresight/utils/models/forex_data.py
@@ -2,9 +2,16 @@
 
 from datetime import datetime
 
+import pandas as pd
 from pydantic import BaseModel
 
 from foresight.utils.database import TimeScaleService
+from foresight.utils.logger import generate_logger
+
+
+logger = generate_logger(name=__name__)
+
+timeMap: dict = {"S": "second", "M": "minute", "H": "hour", "D": "day"}
 
 
 class ForexData(BaseModel):
@@ -71,3 +78,42 @@ def drop_table(table_name: str = "forex_data"):
 
         # Execute SQL queries here
         TimeScaleService().execute(query=f"DROP TABLE {table_name}")
+
+    @staticmethod
+    def fetch(instrument: str = "EUR_USD", timescale: str = "M") -> pd.DataFrame:
+        """
+        Fetch all data from the database and return a DataFrame.
+
+        Parameters:
+            instrument (str): The instrument to fetch
+            timescale (str): The timescale to fetch (S = Second, M = Minute)
+
+        Returns:
+            dict: The data from the database
+        """
+        try:
+            df = pd.DataFrame(
+                TimeScaleService().execute(
+                    query=f"""
+                        SELECT
+                        TO_CHAR(
+                            date_trunc(
+                                '{timeMap[timescale.lower()]}', time
+                            ),
+                            'YYYY-MM-DD HH24:MI:SS'
+                        ) as time,
+                        AVG(ask) as ask, AVG(bid) as bid
+                        FROM forex_data
+                        WHERE instrument = '{instrument}'
+                        AND time >= NOW() - INTERVAL '60 minute'
+                        GROUP BY time
+                        ORDER BY time ASC
+                    """,
+                ),
+            )
+
+            return (
+                df.groupby(df["time"]).agg({"ask": "mean", "bid": "mean"}).reset_index()
+            )
+        except Exception as fetch_exception:  # pylint: disable=broad-except
+            logger.error("Error fetching data: %s", fetch_exception)
diff --git a/foresight/window_service/app.py b/foresight/window_service/app.py
@@ -1,62 +1,21 @@
 """Aggregates the data from the database and calculates one-minute averages."""
 
-import logging
 import time
 
 import pandas as pd
 from boto3_type_annotations.sqs import Client
 
 from foresight.utils.aws import get_client
 from foresight.utils.database import TimeScaleService
+from foresight.utils.logger import generate_logger
+from foresight.utils.models.forex_data import ForexData
 
 
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.INFO,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger = logging.getLogger(__name__)
-
-
-timeMap: dict = {"S": "second", "M": "minute", "H": "hour", "D": "day"}
-
-
-def fetch_data(instrument: str = "EUR_USD", timescale: str = "M") -> pd.DataFrame:
-    """
-    Fetch all data from the database and return a DataFrame.
-
-    Parameters:
-        instrument (str): The instrument to fetch
-        timescale (str): The timescale to fetch (T = tick, M = minutes)
-
-    Returns:
-        dict: The data from the database
-    """
-    try:
-        df = pd.DataFrame(
-            TimeScaleService().execute(
-                query=f"""
-                    SELECT
-                    TO_CHAR(
-                        date_trunc('{timeMap[timescale.lower()]}', time), 'YYYY-MM-DD HH24:MI:SS'
-                    ) as time,
-                    AVG(ask) as ask, AVG(bid) as bid
-                    FROM forex_data
-                    WHERE instrument = '{instrument}'
-                    AND time >= NOW() - INTERVAL '60 minute'
-                    GROUP BY time
-                    ORDER BY time ASC
-                """,
-            ),
-        )
-
-        return df.groupby(df["time"]).agg({"ask": "mean", "bid": "mean"}).reset_index()
-    except Exception as fetch_exception:  # pylint: disable=broad-except
-        logger.error("Error fetching data: %s", fetch_exception)
+logger = generate_logger(name=__name__)
 
 
 if __name__ == "__main__":
-    # Execute SQL queries here
+    # ! TODO create subscription_feeds class
     TimeScaleService().create_table(
         query="""CREATE TABLE IF NOT EXISTS subscription_feeds (
         queue_url VARCHAR(255) NOT NULL,
@@ -77,7 +36,7 @@ def fetch_data(instrument: str = "EUR_USD", timescale: str = "M") -> pd.DataFram
 
             # Calculate averages for each subscription
             for subscription in subscriptions:
-                data = fetch_data(
+                data: pd.DataFrame = ForexData.fetch(
                     instrument=subscription["instrument"],
                     timescale=subscription["timescale"],
                 )
@@ -105,7 +64,6 @@ def fetch_data(instrument: str = "EUR_USD", timescale: str = "M") -> pd.DataFram
             logger.error(f"Error: {sending_exception}")
 
         # Run at the start of the next minutes
-        # til_next_minute = round(60 - time.time() % 60, 2)
-        # logger.info(f"Sleeping for {til_next_minute} seconds")
-        # time.sleep(til_next_minute)
-        time.sleep(5)
+        til_next_minute = round(60 - time.time() % 60, 2)
+        logger.info(f"Sleeping for {til_next_minute} seconds")
+        time.sleep(til_next_minute)
diff --git a/tests/stream_service/conftest.py b/tests/stream_service/conftest.py
@@ -2,8 +2,8 @@
 
 import pytest
 
-from foresight.stream_service.models.forex_data import ForexData
 from foresight.utils.database import TimeScaleService
+from foresight.utils.models.forex_data import ForexData
 
 
 @pytest.fixture()

diff --git a/tests/utils/models/__init__.py b/tests/utils/models/__init__.py
diff --git a/.../stream_service/models/test_forex_data.py → tests/utils/models/test_forex_data.py b/.../stream_service/models/test_forex_data.py → tests/utils/models/test_forex_data.py
@@ -4,8 +4,8 @@
 
 import pytest
 
-from foresight.stream_service.models.forex_data import ForexData
 from foresight.utils.database import TimeScaleService
+from foresight.utils.models.forex_data import ForexData
 
 
 def test_valid_forex_data():
File	Stmts	Miss	Cover	Missing
foresight
__init__.py	0	0	100%
foresight/indicator_services
__init__.py	0	0	100%
indicator.py	64	64	0%	3–5, 7–10, 12, 15, 18, 21–24, 26, 33–38, 44, 46–50, 52, 54, 57, 63, 65, 67–69, 73, 77–78, 82–84, 88, 90, 92, 94, 96, 108, 110–111, 117, 119, 122, 125, 128, 130, 132, 134–136, 138–139, 141, 143, 145
moving_average_indicator.py	18	18	0%	3, 5–6, 9, 12, 20–21, 28, 30, 33–34, 36–37, 40, 42, 45–46, 51
foresight/interface_service
__init__.py	0	0	100%
app.py	31	31	0%	3–4, 6–9, 11–12, 15, 18, 20, 23, 25, 33–34, 36, 39–40, 42, 45–46, 48–51, 53–54, 56, 59–61
foresight/stream_service
__init__.py	0	0	100%
app.py	71	13	81%	82, 110, 131, 133–134, 139, 142–148
foresight/stream_service/models
__init__.py	0	0	100%
pricing.py	4	0	100%
stream.py	18	0	100%
foresight/utils
__init__.py	0	0	100%
aws.py	12	12	0%	3, 5, 8, 11, 13–14, 21, 27, 30, 32–33, 40
database.py	51	13	74%	51–52, 67–70, 75, 86–87, 89, 93–95
logger.py	9	0	100%
foresight/utils/models
__init__.py	0	0	100%
forex_data.py	28	5	82%	94–95, 115, 118–119
foresight/window_service
__init__.py	0	0	100%
app.py	34	34	0%	3, 5–6, 8–11, 14, 17, 19, 29, 31–33, 38–39, 44–53, 57–59, 63–64, 67–69
tests
__init__.py	0	0	100%
tests/stream_service
__init__.py	0	0	100%
conftest.py	19	9	52%	21–22, 30, 33, 37–41
test_app.py	40	0	100%
tests/stream_service/models
__init__.py	0	0	100%
tests/utils/models
__init__.py	0	0	100%
test_forex_data.py	30	0	100%
tests/window_service
__init__.py	0	0	100%
test_app.py	0	0	100%
TOTAL	429	199	53%