Add new fields to items

price_scraper/price_scraper/items.py

@@ -11,7 +11,9 @@ class PortfolioPerformanceHistoricalPrice(scrapy.Item):
     Contains data for portfolio performance historical price data source
     """
 
-    # name of the instrument can be ISIN or explicit name depending on the scraped website
+    # name of the file to export the price data to
+    file_name = scrapy.Field()
+    # long name of the instrument
     name = scrapy.Field()
     # date of the price
     date = scrapy.Field()
@@ -23,9 +25,11 @@ class PortfolioPerformanceHistoricalPrice(scrapy.Item):
     day_low = scrapy.Field()
     # highest price during the day
     day_high = scrapy.Field()
-    # long name
-    long_name = scrapy.Field()
     # Currency of the fund
     currency = scrapy.Field()
     # Start date of the fund
     start_date = scrapy.Field()
+    # ISIN if available
+    isin = scrapy.Field()
+    # symbol if available
+    symbol = scrapy.Field()

price_scraper/price_scraper/price_exporter_pipeline.py

@@ -29,17 +29,17 @@ def close_spider(self, spider):
             json_file.close()
 
     def _exporter_for_item(self, adapter):
-        name = self.file_name(adapter["name"])
+        name = self.file_name(adapter["file_name"])
         if name not in self.portfolio_to_exporter:
             self._create_exporter(name)
         return self.portfolio_to_exporter[name][0]
 
     def process_item(self, item, spider):
         adapter = ItemAdapter(item)
         exporter = self._exporter_for_item(adapter)
-        if adapter["date"] in self.stored_dates[self.file_name(adapter["name"])]:
+        if adapter["date"] in self.stored_dates[self.file_name(adapter["file_name"])]:
             raise DropItem(
-                f"Date {adapter['date']} is already stored for instrument {adapter['name']}"
+                f"Date {adapter['date']} is already stored for instrument {adapter['file_name']}"
             )
         exporter.export_item(item)
         return item

price_scraper/price_scraper/spiders/aranykor.py

@@ -32,9 +32,10 @@ def parse(self, response):
             date = prices[1]
             for idx, portfolio in enumerate(header, 2):
                 yield PortfolioPerformanceHistoricalPrice(
-                    name=portfolio,
+                    file_name=portfolio,
                     date=date.replace('.', '-'),
                     price=float(prices[idx]),
-                    long_name=portfolio,
+                    name=f"Aranykor {portfolio}",
                     currency="HUF",
+                    symbol=f"ARANY_{portfolio[:5].upper()}"
                 )

price_scraper/price_scraper/spiders/bamosz.py

@@ -26,12 +26,14 @@ def parse(self, response):
                 fund_data = data_row.css('td::text').getall()
                 fund_data = sanitize_columns(fund_data)
                 yield PortfolioPerformanceHistoricalPrice(
-                    name=isin,
+                    name=long_fund_name,
+                    file_name=isin,
                     # it has a trailing dot in the date but we shouldn't remove it
                     date=fund_data[3].replace('.', '-')[:-1],
                     price=float(fund_data[2].replace(',', '.')),
                     currency=fund_data[1],
-                    long_name=long_fund_name,
+                    isin=isin,
+                    symbol=isin,
                 )
 
 def sanitize_columns(columns):

price_scraper/price_scraper/spiders/mak.py

@@ -53,8 +53,39 @@ def parse_type(self, response):
                     accrued_interest = float(product['accruedInterest'].replace(',', '.'))
                 price = bid_pct + accrued_interest / 100
 
+            long_name = security_type_to_long_name(security_type)
             yield PortfolioPerformanceHistoricalPrice(
-                name=f"{security_type}_{product['name']}",
+                file_name=f"{security_type}_{product['name']}",
                 date=product['settleDate'].replace('.', '-'),
-                price=price
+                price=price,
+                symbol=f"{security_type}_{product['name']}",
+                name=f"{long_name} {product['name']}",
+                start_date=product['issueDate']
             )
+
+def security_type_to_long_name(security_type: str):
+    """
+    Converts short bond security types to long names
+    """
+
+    match security_type:
+        case "1MÁP":
+            return "Egyéves Magyar Állampapír"
+        case "BABA":
+            return "Babakötvény"
+        case "BMÁP":
+            return "Bónusz Magyar Állampapír"
+        case "DKJ":
+            return "Diszkont Kincstárjegy"
+        case "EMÁP":
+            return "Euro Magyar Állampapír"
+        case "FixMÁP":
+            return "Fix Magyar Állampapír"
+        case "KTV":
+            return "Magyar Államkötvény"
+        case "MÁPP":
+            return "Magyar Állampapír Plusz"
+        case "PEMÁP":
+            return "Prémium Euró Magyar Állampapír"
+        case "PMÁP":
+            return "Prémium Magyar Állampapír"