New HS Code - Working

HS Code mapping done Error catching done Further work 1. Better reporting 2. Include other indicators
akoyamp · Feb 25, 2025 · 39a400d · 39a400d
1 parent ef25821
commit 39a400d
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 109 deletions.
diff --git a/Test.ipynb b/Test.ipynb
@@ -9,19 +9,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reading table/s ['Aluminium', 'Antimony', 'Arsenic', 'Asbestos', 'Baryte', 'Bauxite', 'Bentonite', 'Beryllium (conc.)', 'Bismuth', 'Boron Minerals', 'Cadmium', 'Chromium (Cr2O3)', 'Cobalt', 'Coking Coal', 'Copper', 'Diamonds (Gem)', 'Diamonds (Ind)', 'Diatomite', 'Feldspar', 'Fluorspar', 'Gallium', 'Germanium', 'Gold', 'Graphite', 'Gypsum and Anhydrite', 'Indium', 'Iron (Fe)', 'Kaolin (China-Clay)', 'Lead', 'Lignite', 'Lithium (Li2O)', 'logging', 'Magnesite', 'Manganese', 'Mercury', 'Molybdenum', 'Natural Gas', 'Nickel', 'Niobium (Nb2O5)', 'Oil Sands (part of Petroleum)', 'Oil Shales', 'Palladium', 'Perlite', 'Petroleum', 'Phosphate Rock (P2O5)', 'Platinum', 'Potash (K2O)', 'Rare Earths (REO)', 'Rhenium', 'Rhodium', 'Salt (rock, brines, marine)', 'Selenium', 'Silver', 'Steam Coal', 'Sulfur (elementar & industrial)', 'Talc, Steatite & Pyrophyllite', 'Tantalum (Ta2O5)', 'Tellurium', 'Tin', 'Titanium (TiO2)', 'Tungsten (W)', 'Uranium (U3O8)', 'Vanadium (V)', 'Vermiculite', 'Zinc', 'Zircon', 'Country_ISO', 'HS Code Map'] from the library database C:\\Users\\akoyamparamb\\Documents/geopolrisk/databases/world_mining_data.db.: 100%|██████████| 68/68 [00:00<00:00, 1155.63it/s]\n",
-      "Reading table/s ['Normalized'] from the library database C:\\Users\\akoyamparamb\\Documents/geopolrisk/databases/wgi.db.: 100%|██████████| 1/1 [00:00<00:00, 445.16it/s]\n",
-      "Reading table/s ['baci_trade'] from the library database C:\\Users\\akoyamparamb\\Documents/geopolrisk/databases/baci.db.: 100%|██████████| 1/1 [00:48<00:00, 48.54s/it]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from geopolrisk.assessment.main import gprs_calc"
    ]
@@ -35,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -50,87 +40,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Calculating the GeoPolRisk:   0%|          | 0/212 [00:00<?, ?it/s]--- Logging error ---\n",
-      "Traceback (most recent call last):\n",
-      "  File \"c:\\Users\\akoyamparamb\\Documents\\GitHub\\geopolrisk-py\\geopolrisk\\assessment\\main.py\", line 97, in gprs_calc\n",
-      "    Numerator, TotalTrade, Price = importrisk(\n",
-      "                                   ^^^^^^^^^^^\n",
-      "  File \"c:\\Users\\akoyamparamb\\Documents\\GitHub\\geopolrisk-py\\geopolrisk\\assessment\\core.py\", line 126, in importrisk\n",
-      "    return Numerator, TotalTrade, Price\n",
-      "           ^^^^^^^^^\n",
-      "UnboundLocalError: cannot access local variable 'Numerator' where it is not associated with a value\n",
-      "\n",
-      "During handling of the above exception, another exception occurred:\n",
-      "\n",
-      "Traceback (most recent call last):\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\logging\\__init__.py\", line 1110, in emit\n",
-      "    msg = self.format(record)\n",
-      "          ^^^^^^^^^^^^^^^^^^^\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\logging\\__init__.py\", line 953, in format\n",
-      "    return fmt.format(record)\n",
-      "           ^^^^^^^^^^^^^^^^^^\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\logging\\__init__.py\", line 687, in format\n",
-      "    record.message = record.getMessage()\n",
-      "                     ^^^^^^^^^^^^^^^^^^^\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\logging\\__init__.py\", line 377, in getMessage\n",
-      "    msg = msg % self.args\n",
-      "          ~~~~^~~~~~~~~~~\n",
-      "TypeError: not all arguments converted during string formatting\n",
-      "Call stack:\n",
-      "  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
-      "  File \"<frozen runpy>\", line 88, in _run_code\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n",
-      "    app.launch_new_instance()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\traitlets\\config\\application.py\", line 992, in launch_instance\n",
-      "    app.start()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 736, in start\n",
-      "    self.io_loop.start()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 195, in start\n",
-      "    self.asyncio_loop.run_forever()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\asyncio\\base_events.py\", line 607, in run_forever\n",
-      "    self._run_once()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\asyncio\\base_events.py\", line 1922, in _run_once\n",
-      "    handle._run()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\asyncio\\events.py\", line 80, in _run\n",
-      "    self._context.run(self._callback, *self._args)\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 516, in dispatch_queue\n",
-      "    await self.process_one()\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 505, in process_one\n",
-      "    await dispatch(*args)\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 412, in dispatch_shell\n",
-      "    await result\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 740, in execute_request\n",
-      "    reply_content = await reply_content\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 422, in do_execute\n",
-      "    res = shell.run_cell(\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 546, in run_cell\n",
-      "    return super().run_cell(*args, **kwargs)\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3046, in run_cell\n",
-      "    result = self._run_cell(\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3101, in _run_cell\n",
-      "    result = runner(coro)\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
-      "    coro.send(None)\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3306, in run_cell_async\n",
-      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3488, in run_ast_nodes\n",
-      "    if await self.run_code(code, result, async_=asy):\n",
-      "  File \"c:\\Users\\akoyamparamb\\AppData\\Local\\anaconda3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3548, in run_code\n",
-      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
-      "  File \"C:\\Users\\akoyamparamb\\AppData\\Local\\Temp\\ipykernel_37144\\1007593563.py\", line 1, in <module>\n",
-      "    gprs_calc(ListofYear, ListofCountries, ListofMetals)\n",
-      "  File \"c:\\Users\\akoyamparamb\\Documents\\GitHub\\geopolrisk-py\\geopolrisk\\assessment\\main.py\", line 108, in gprs_calc\n",
-      "    logging.debug(\"Unknwon exception at \", e)\n",
-      "Message: 'Unknwon exception at '\n",
-      "Arguments: (UnboundLocalError(\"cannot access local variable 'Numerator' where it is not associated with a value\"),)\n",
-      "Calculating the GeoPolRisk:   0%|          | 0/212 [00:16<?, ?it/s]\n"
+      "Calculating the GeoPolRisk: 100%|██████████| 212/212 [00:58<00:00,  3.63it/s]\n"
      ]
     }
    ],
@@ -503,7 +420,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/geopolrisk/assessment/core.py b/geopolrisk/assessment/core.py
@@ -96,18 +96,24 @@ def wgi_func(x):
             rawmaterial,
             data,
         )  # Dataframe from the utility function
-        QTY = tradedf["qty"].astype(float).tolist()
-        WGI = tradedf["partnerWGI"].apply(wgi_func).astype(float).tolist()
-        VAL = tradedf["cifvalue"].astype(float).tolist()
-        try:
-            Price = sum(VAL) / sum(QTY)
-            TotalTrade = sum(QTY)
-            Numerator = sumproduct(QTY, WGI)
-        except:
+        if tradedf != None:
+            QTY = tradedf["qty"].astype(float).tolist()
+            WGI = tradedf["partnerWGI"].apply(wgi_func).astype(float).tolist()
+            VAL = tradedf["cifvalue"].astype(float).tolist()
+            try:
+                Price = sum(VAL) / sum(QTY)
+                TotalTrade = sum(QTY)
+                Numerator = sumproduct(QTY, WGI)
+            except:
+                logging.debug(
+                    f"Error while making calculations. Raw Material: {rawmaterial}, Country: {country}, Year: {year}"
+                )
+                raise ValueError
+        else:
             logging.debug(
-                f"Error while making calculations. Raw Material: {rawmaterial}, Country: {country}, Year: {year}"
+                f"Data not available for the given inputs, Raw Material: {rawmaterial}, Country: {country}, Year: {year}"
             )
-            raise ValueError
+            Numerator, TotalTrade, Price = 0, 0, 0
     else:
         try:
             Numerator, TotalTrade, Price = aggregateTrade(
@@ -170,6 +176,7 @@ def GeoPolRisk(Numerator, TotalTrade, Price, ProdQty, hhi):
         logging.debug(
             f"Check the Numerator and Denominator. Numerator: {Numerator}, Denominator: {Denominator}"
         )
+        WTA = 0
     Score = hhi * WTA
     CF_Cu = 0.409412948  # Average CF of copper for OECD countries for a period from 2017 - 2021
     CF = (Score * Price) / CF_Cu

diff --git a/geopolrisk/assessment/utils.py b/geopolrisk/assessment/utils.py
@@ -229,8 +229,14 @@ def getbacidata(period: int, country: int, rawmaterial: str, data):
     """
     get the baci-trade-data from the baci_trade dataframe
     """
-    df_query = f"(period == {period}) & (reporterCode == {country}) & (rawMaterial == '{rawmaterial}')"
-    baci_data = data.query(df_query)
+    try:
+        df_query = f"(period == {period}) & (reporterCode == {country}) & (rawMaterial == '{rawmaterial}')"
+        baci_data = data.query(df_query)
+    except Exception as e:
+        logging.debug(
+            f"Error when querying baci database - issue with -- {e}"
+        )
+        return None
     """
     The dataframe is structured as follows:
     'period' -> The year of the trade recorded
@@ -246,16 +252,16 @@ def getbacidata(period: int, country: int, rawmaterial: str, data):
     'partnerWGI' -> The WGI political stability and absence of violence indicator (Normalized) for the partner country
     'Raw Material' -> The Reference name of the commodity traded
     """
-    baci_data.loc[:, "qty"] = baci_data["qty"].apply(replace_func).astype(float)
-    baci_data.loc[:, "cifvalue"] = (
-        baci_data["cifvalue"].apply(replace_func).astype(float)
-    )
     if baci_data is None or isinstance(baci_data, type(None)) or len(baci_data) == 0:
         logging.debug(
-            f"Problem with get the baci-data - {baci_data} - period == {period} - reporterCode == {country} - Raw Material == '{rawmaterial}'"
+            f"baci data error: - {baci_data} - period == {period} - reporterCode == {country} - Raw Material == '{rawmaterial}'"
         )
-        baci_data = None
-    return baci_data
+        return None
+    else:
+        baci_data.loc[:, "qty"] = baci_data["qty"].apply(replace_func).astype(float)
+        baci_data.loc[:, "cifvalue"] = (
+        baci_data["cifvalue"].apply(replace_func).astype(float))
+        return baci_data
 
 
 def aggregateTrade(period: int, country: list, rawmaterial: str, data):
@@ -298,7 +304,13 @@ def wgi_func(x):
         SUMVAL.append(sum(VAL))
         SUMNUM.append(sumproduct(QTY, WGI))
 
-    Price = sum(SUMVAL) / sum(SUMQTY)
+    if sum(SUMQTY) == 0:
+        logging.debug(
+            f"Total trade for the region is 0, Country: {country}, Raw Material: {rawmaterial}, Year: {period}"
+        )
+        Price = 0
+    else:
+        Price = sum(SUMVAL) / sum(SUMQTY)
     """
     The function returns the numerator of the import risk, 
     The total trade for the region,

diff --git a/geopolrisk/lib/world_mining_data.db b/geopolrisk/lib/world_mining_data.db