Merge pull request #47 from Bot-detector/cleanup

Cleanup

.env.example

@@ -1,15 +1,6 @@
-# (str) link to download the proxy list from webshare.io
-# can be any proxy provider, so long as the list is the same format
-# GOTO: https://proxy.webshare.io/proxy/list?
-# Download Proxy List (Txt)
-PROXY_DOWNLOAD_URL = https://proxy.webshare.io/proxy/list/download/PASSWORD/-/http/username/direct/
+# To get an api key please use our referral code (they also have free tier)
+# https://www.webshare.io/?referral_code=qvpjdwxqsblt
 
-# (varchar) api token for auth
-# tokens are found in DB playerdata under the table apiUser
-TOKEN = API_TOKEN
-
-# (int) the batch size of the usernames to scrape
-QUERY_SIZE = 5000
-
-# (str) for local development use the endpoint http://127.0.0.1:5000
-endpoint = https://www.osrsbotdetector.com/dev
+PROXY_API_KEY = "str"
+SESSION_TIMEOUT = "60"
+KAFKA_HOST = "kafka:9092"

kafka_setup/setup_kafka.py

@@ -25,14 +25,19 @@ def create_topics():
         [
             NewTopic(
                 name="player",
-                num_partitions=3,
+                num_partitions=4,
                 replication_factor=1,
             ),
             NewTopic(
                 name="scraper",
                 num_partitions=4,
                 replication_factor=1,
             ),
+            NewTopic(
+                name="scraper-runemetrics",
+                num_partitions=4,
+                replication_factor=1,
+            ),
             NewTopic(
                 name="reports",
                 num_partitions=4,

src/config/config.py

@@ -1,22 +1,18 @@
 import os
 
 import dotenv
-from aiohttp import ClientTimeout
+from pydantic import BaseSettings
 
 from config import logging
-from config.app_config import AppConfig
 
-dotenv.load_dotenv()
 
-app_config = AppConfig(
-    PROXY_DOWNLOAD_URL=os.getenv("PROXY_DOWNLOAD_URL"),
-    PROXY_API_KEY=os.getenv("PROXY_API_KEY"),
-    ENDPOINT=os.getenv("ENDPOINT"),
-    QUERY_SIZE=int(os.getenv("QUERY_SIZE")),
-    TOKEN=os.getenv("TOKEN"),
-    MAX_BYTES=int(os.getenv("MAX_BYTES", 1_000_000)),
-    POST_INTERVAL=os.getenv("POST_INTERVAL", 60),
-    TIMEOUT_SECONDS=10,
-    SESSION_TIMEOUT=os.getenv("SESSION_TIMEOUT", 60),
-    KAFKA_HOST=os.getenv("KAFKA_HOST"),
-)
+class AppConfig(BaseSettings):
+    PROXY_API_KEY: str
+    SESSION_TIMEOUT: int = 60
+    KAFKA_HOST: str = "kafka:9092"
+
+    class Config:
+        env_prefix = ""
+
+
+app_config = AppConfig()