Make LIFO/HIFO accounting methods O(n*log(m))

src/rp2/abstract_accounting_method.py

@@ -14,12 +14,18 @@
 
 
 from enum import Enum
-from typing import Dict, List, NamedTuple, Optional
+from heapq import heappop, heappush
+from typing import Dict, List, NamedTuple, Optional, Tuple
 
 from rp2.abstract_transaction import AbstractTransaction
 from rp2.in_transaction import InTransaction
 from rp2.rp2_decimal import ZERO, RP2Decimal
-from rp2.rp2_error import RP2RuntimeError
+from rp2.rp2_error import RP2RuntimeError, RP2TypeError
+
+
+class AbstractAccountingMethodIterator:
+    def __next__(self) -> InTransaction:
+        raise NotImplementedError("abstract function")
 
 
 class AcquiredLotAndAmount(NamedTuple):
@@ -32,29 +38,43 @@ class AcquiredLotCandidatesOrder(Enum):
     NEWER_TO_OLDER: str = "newer_to_older"
 
 
-class AcquiredLotCandidates:
+class AcquiredLotHeapSortKey(NamedTuple):
+    spot_price: RP2Decimal
+    timestamp: float
+    internal_id_int: int
+
+
+class AbstractAcquiredLotCandidates:
     def __init__(
         self,
         accounting_method: "AbstractAccountingMethod",
         acquired_lot_list: List[InTransaction],
         acquired_lot_2_partial_amount: Dict[InTransaction, RP2Decimal],
     ) -> None:
-        self.__accounting_method: AbstractAccountingMethod = accounting_method
+        self._accounting_method: AbstractAccountingMethod = accounting_method
         self.__acquired_lot_list = acquired_lot_list
         self.__acquired_lot_2_partial_amount = acquired_lot_2_partial_amount
-        self.__to_index = 0
+        self._to_index = 0
         self.__from_index = 0
 
-    def set_to_index(self, to_index: int) -> None:
-        self.__to_index = to_index
-
     def set_from_index(self, from_index: int) -> None:
         self.__from_index = from_index
 
+    def set_to_index(self, to_index: int) -> None:
+        raise NotImplementedError("abstract")
+
     @property
     def from_index(self) -> int:
         return self.__from_index
 
+    @property
+    def to_index(self) -> int:
+        return self._to_index
+
+    @property
+    def acquired_lot_list(self) -> List[InTransaction]:
+        return self.__acquired_lot_list
+
     def has_partial_amount(self, acquired_lot: InTransaction) -> bool:
         return acquired_lot in self.__acquired_lot_2_partial_amount
 
@@ -69,11 +89,11 @@ def set_partial_amount(self, acquired_lot: InTransaction, amount: RP2Decimal) ->
     def clear_partial_amount(self, acquired_lot: InTransaction) -> None:
         self.set_partial_amount(acquired_lot, ZERO)
 
-    def __iter__(self) -> "AccountingMethodIterator":
-        return AccountingMethodIterator(self.__acquired_lot_list, self.__from_index, self.__to_index, self.__accounting_method.lot_candidates_order())
+    def __iter__(self) -> AbstractAccountingMethodIterator:
+        return self._accounting_method._get_accounting_method_iterator(self)
 
 
-class AccountingMethodIterator:
+class ListAccountingMethodIterator(AbstractAccountingMethodIterator):
     def __init__(self, acquired_lot_list: List[InTransaction], from_index: int, to_index: int, order_type: AcquiredLotCandidatesOrder) -> None:
         self.__acquired_lot_list = acquired_lot_list
         self.__start_index = from_index if order_type == AcquiredLotCandidatesOrder.OLDER_TO_NEWER else to_index
@@ -96,10 +116,55 @@ def __next__(self) -> InTransaction:
         raise StopIteration(self)
 
 
+class HeapAccountingMethodIterator(AbstractAccountingMethodIterator):
+    def __init__(self, acquired_lot_heap: List[Tuple[AcquiredLotHeapSortKey, InTransaction]]) -> None:
+        self.__acquired_lot_heap = acquired_lot_heap
+
+    def __next__(self) -> InTransaction:
+        while len(self.__acquired_lot_heap) > 0:
+            _, result = heappop(self.__acquired_lot_heap)
+            return result
+        raise StopIteration(self)
+
+
+class ListAcquiredLotCandidates(AbstractAcquiredLotCandidates):
+    def set_to_index(self, to_index: int) -> None:
+        self._to_index = to_index  # pylint: disable=unused-private-member
+
+
+class HeapAcquiredLotCandidates(AbstractAcquiredLotCandidates):
+    _accounting_method: "AbstractHeapAccountingMethod"
+
+    def __init__(
+        self,
+        accounting_method: "AbstractAccountingMethod",
+        acquired_lot_list: List[InTransaction],
+        acquired_lot_2_partial_amount: Dict[InTransaction, RP2Decimal],
+    ) -> None:
+        super().__init__(accounting_method, acquired_lot_list, acquired_lot_2_partial_amount)
+        self.__acquired_lot_heap: List[Tuple[AcquiredLotHeapSortKey, InTransaction]] = []
+
+    def set_to_index(self, to_index: int) -> None:
+        # Control how far to advance the iterator, caller is responsible for updating
+        for i in range(self.to_index, to_index + 1):
+            lot = self.acquired_lot_list[i]
+            self._accounting_method.add_selected_lot_to_heap(self.acquired_lot_heap, lot)
+        self._to_index = to_index
+
+    @property
+    def acquired_lot_heap(self) -> List[Tuple[AcquiredLotHeapSortKey, InTransaction]]:
+        return self.__acquired_lot_heap
+
+
 class AbstractAccountingMethod:
+    def create_lot_candidates(
+        self, acquired_lot_list: List[InTransaction], acquired_lot_2_partial_amount: Dict[InTransaction, RP2Decimal]
+    ) -> AbstractAcquiredLotCandidates:
+        raise NotImplementedError("abstract")
+
     def seek_non_exhausted_acquired_lot(
         self,
-        lot_candidates: AcquiredLotCandidates,
+        lot_candidates: AbstractAcquiredLotCandidates,
         taxable_event: Optional[AbstractTransaction],
         taxable_event_amount: RP2Decimal,
     ) -> Optional[AcquiredLotAndAmount]:
@@ -114,3 +179,38 @@ def name(self) -> str:
 
     def __repr__(self) -> str:
         return self.name
+
+    def _get_accounting_method_iterator(self, lot_candidates: AbstractAcquiredLotCandidates) -> AbstractAccountingMethodIterator:
+        raise NotImplementedError("Abstract function")
+
+
+class AbstractListAccountingMethod(AbstractAccountingMethod):
+    def create_lot_candidates(
+        self, acquired_lot_list: List[InTransaction], acquired_lot_2_partial_amount: Dict[InTransaction, RP2Decimal]
+    ) -> ListAcquiredLotCandidates:
+        return ListAcquiredLotCandidates(self, acquired_lot_list, acquired_lot_2_partial_amount)
+
+    def lot_candidates_order(self) -> AcquiredLotCandidatesOrder:
+        raise NotImplementedError("Abstract function")
+
+    def _get_accounting_method_iterator(self, lot_candidates: AbstractAcquiredLotCandidates) -> ListAccountingMethodIterator:
+        return ListAccountingMethodIterator(lot_candidates.acquired_lot_list, lot_candidates.from_index, lot_candidates.to_index, self.lot_candidates_order())
+
+
+class AbstractHeapAccountingMethod(AbstractAccountingMethod):
+    def create_lot_candidates(
+        self, acquired_lot_list: List[InTransaction], acquired_lot_2_partial_amount: Dict[InTransaction, RP2Decimal]
+    ) -> HeapAcquiredLotCandidates:
+        return HeapAcquiredLotCandidates(self, acquired_lot_list, acquired_lot_2_partial_amount)
+
+    def add_selected_lot_to_heap(self, heap: List[Tuple[AcquiredLotHeapSortKey, InTransaction]], lot: InTransaction) -> None:
+        heap_item = (self.heap_key(lot), lot)
+        heappush(heap, heap_item)
+
+    def heap_key(self, lot: InTransaction) -> AcquiredLotHeapSortKey:
+        raise NotImplementedError("Abstract function")
+
+    def _get_accounting_method_iterator(self, lot_candidates: AbstractAcquiredLotCandidates) -> HeapAccountingMethodIterator:
+        if not isinstance(lot_candidates, HeapAcquiredLotCandidates):
+            raise RP2TypeError(f"Internal error: lot_candidates is not of type HeapAcquiredLotCandidates, but of type {type(lot_candidates)}")
+        return HeapAccountingMethodIterator(lot_candidates.acquired_lot_heap)

src/rp2/abstract_transaction.py

@@ -30,7 +30,7 @@ def __init__(
         asset: str,
         transaction_type: str,
         spot_price: RP2Decimal,
-        internal_id: Optional[int] = None,
+        row: Optional[int] = None,
         unique_id: Optional[str] = None,
         notes: Optional[str] = None,
     ) -> None:
@@ -39,7 +39,9 @@ def __init__(
         self.__timestamp: datetime = configuration.type_check_timestamp_from_string("timestamp", timestamp)
         self.__transaction_type: TransactionType = TransactionType.type_check_from_string("transaction_type", transaction_type)
         self.__spot_price: RP2Decimal = configuration.type_check_positive_decimal("spot_price", spot_price)
-        self.__internal_id: int = configuration.type_check_internal_id("internal_id", internal_id) if internal_id is not None else id(self)
+        # TODO: behavior when row is not provided does not semantically match "row", make non-optional  # pylint: disable=fixme
+        self.__row: int = configuration.type_check_internal_id("row", row) if row is not None else id(self)
+        self.__internal_id: int = self.__row
         self.__unique_id: str = configuration.type_check_string_or_integer("unique_id", unique_id) if unique_id is not None else ""
         self.__notes = configuration.type_check_string("notes", notes) if notes else ""
 
@@ -63,6 +65,9 @@ def __eq__(self, other: object) -> bool:
     def __ne__(self, other: object) -> bool:
         return not self.__eq__(other)
 
+    def __lt__(self, other: "AbstractTransaction") -> bool:
+        return self.timestamp < other.timestamp
+
     def __hash__(self) -> int:
         # By definition, internal_id can uniquely identify a transaction: this works even if it's the ODS line from the spreadsheet,
         # since there are no cross-asset transactions (so a spreadsheet line points to a unique transaction for that asset).
@@ -86,6 +91,10 @@ def to_string(self, indent: int = 0, repr_format: bool = True, extra_data: Optio
     def internal_id(self) -> str:
         return str(self.__internal_id)
 
+    @property
+    def row(self) -> int:
+        return self.__row
+
     @property
     def timestamp(self) -> datetime:
         return self.__timestamp

src/rp2/accounting_engine.py

@@ -20,8 +20,8 @@
 
 from rp2.abstract_accounting_method import (
     AbstractAccountingMethod,
+    AbstractAcquiredLotCandidates,
     AcquiredLotAndAmount,
-    AcquiredLotCandidates,
 )
 from rp2.abstract_transaction import AbstractTransaction
 from rp2.in_transaction import InTransaction
@@ -84,7 +84,7 @@ def type_check(cls, name: str, instance: "AccountingEngine") -> "AccountingEngin
 
     def __init__(self, years_2_methods: AVLTree[int, AbstractAccountingMethod]) -> None:
         self.__years_2_methods: AVLTree[int, AbstractAccountingMethod] = years_2_methods
-        self.__years_2_lot_candidates: AVLTree[int, AcquiredLotCandidates] = AVLTree()
+        self.__years_2_lot_candidates: AVLTree[int, AbstractAcquiredLotCandidates] = AVLTree()
         if not self.__years_2_methods:
             raise RP2RuntimeError("Internal error: no accounting method defined")
 
@@ -119,7 +119,7 @@ def initialize(
         node = self.__years_2_methods.root
         while node is not None:
             self.__years_2_lot_candidates.insert_node(
-                node.key, AcquiredLotCandidates(node.value, self.__acquired_lot_list, self.__acquired_lot_2_partial_amount)
+                node.key, node.value.create_lot_candidates(self.__acquired_lot_list, self.__acquired_lot_2_partial_amount)
             )
             if node.left:
                 to_visit.append(node.left)
@@ -206,7 +206,7 @@ def get_acquired_lot_for_taxable_event(
             if acquired_lot_and_index.acquired_lot != self.__acquired_lot_list[acquired_lot_and_index.index]:
                 raise RP2RuntimeError("Internal error: acquired_lot incongruence in accounting logic")
             method = self._get_accounting_method(taxable_event.timestamp.year)
-            lot_candidates: Optional[AcquiredLotCandidates] = self.__years_2_lot_candidates.find_max_value_less_than(taxable_event.timestamp.year)
+            lot_candidates: Optional[AbstractAcquiredLotCandidates] = self.__years_2_lot_candidates.find_max_value_less_than(taxable_event.timestamp.year)
             # lot_candidates is 1:1 with acquired_lot_and_index, should always be True
             if lot_candidates:
                 lot_candidates.set_to_index(acquired_lot_and_index.index)

src/rp2/in_transaction.py

@@ -45,15 +45,21 @@ def __init__(
         fiat_in_no_fee: Optional[RP2Decimal] = None,
         fiat_in_with_fee: Optional[RP2Decimal] = None,
         fiat_fee: Optional[RP2Decimal] = None,
-        internal_id: Optional[int] = None,
+        row: Optional[int] = None,
         unique_id: Optional[str] = None,
         notes: Optional[str] = None,
     ) -> None:
-        super().__init__(configuration, timestamp, asset, transaction_type, spot_price, internal_id, unique_id, notes)
+        super().__init__(configuration, timestamp, asset, transaction_type, spot_price, row, unique_id, notes)
 
         self.__exchange: str = configuration.type_check_exchange("exchange", exchange)
         self.__holder: str = configuration.type_check_holder("holder", holder)
-        self.__crypto_in: RP2Decimal = configuration.type_check_positive_decimal("crypto_in", crypto_in, non_zero=True)
+        self.__crypto_in: RP2Decimal
+        if self.transaction_type == TransactionType.STAKING:
+            # Staking income can be negative: in certain cases the protocol can remove from the stash rather than
+            # add to it (e.g. if the node stays offline too long).
+            self.__crypto_in = configuration.type_check_decimal("crypto_in", crypto_in)
+        else:
+            self.__crypto_in = configuration.type_check_positive_decimal("crypto_in", crypto_in, non_zero=True)
         self.__crypto_fee: RP2Decimal = configuration.type_check_positive_decimal("crypto_fee", crypto_fee) if crypto_fee else ZERO
         self.__fiat_fee: RP2Decimal = configuration.type_check_positive_decimal("fiat_fee", fiat_fee) if fiat_fee else ZERO
 

src/rp2/intra_transaction.py

@@ -34,7 +34,7 @@ def __init__(
         spot_price: Optional[RP2Decimal],
         crypto_sent: RP2Decimal,
         crypto_received: RP2Decimal,
-        internal_id: Optional[int] = None,
+        row: Optional[int] = None,
         unique_id: Optional[str] = None,
         notes: Optional[str] = None,
     ) -> None:
@@ -52,7 +52,7 @@ def __init__(
                 raise RP2ValueError(
                     f"crypto_fee is non-zero ({self.__crypto_fee}) but spot_price is empty or zero: {timestamp} {asset} {crypto_sent} {unique_id} "
                 )
-        super().__init__(configuration, timestamp, asset, "MOVE", spot_price, internal_id, unique_id, notes)
+        super().__init__(configuration, timestamp, asset, "MOVE", spot_price, row, unique_id, notes)
 
         self.__from_exchange: str = configuration.type_check_exchange("from_exchange", from_exchange)
         self.__from_holder: str = configuration.type_check_holder("from_holder", from_holder)

src/rp2/ods_parser.py

@@ -191,7 +191,7 @@ def _create_and_process_transaction(
                 fiat_in_no_fee=transaction.fiat_in_no_fee,
                 fiat_in_with_fee=transaction.fiat_in_with_fee,
                 fiat_fee=transaction.fiat_fee,
-                internal_id=internal_id,
+                row=internal_id,
                 unique_id=transaction.unique_id,
                 notes=notes,
             )
@@ -207,7 +207,7 @@ def _create_and_process_transaction(
                 spot_price=transaction.spot_price,
                 crypto_out_no_fee=ZERO,
                 crypto_fee=transaction.crypto_fee,
-                internal_id=artificial_internal_id,
+                row=artificial_internal_id,
                 unique_id=transaction.unique_id,
                 notes=(
                     f"Artificial transaction modeling the crypto fee of {transaction.crypto_fee} {transaction.asset} "
@@ -243,7 +243,7 @@ def _process_constructor_argument_pack(
     internal_id: int,
     class_name: str,
 ) -> Dict[str, Any]:
-    argument_pack.update({"configuration": configuration, "internal_id": internal_id})
+    argument_pack.update({"configuration": configuration, "row": internal_id})
     numeric_parameters: List[str] = _get_decimal_constructor_argument_names(class_name)
     for numeric_parameter in numeric_parameters:
         if numeric_parameter in argument_pack:

src/rp2/out_transaction.py

@@ -38,11 +38,11 @@ def __init__(
         crypto_out_with_fee: Optional[RP2Decimal] = None,
         fiat_out_no_fee: Optional[RP2Decimal] = None,
         fiat_fee: Optional[RP2Decimal] = None,
-        internal_id: Optional[int] = None,
+        row: Optional[int] = None,
         unique_id: Optional[str] = None,
         notes: Optional[str] = None,
     ) -> None:
-        super().__init__(configuration, timestamp, asset, transaction_type, spot_price, internal_id, unique_id, notes)
+        super().__init__(configuration, timestamp, asset, transaction_type, spot_price, row, unique_id, notes)
 
         self.__exchange: str = configuration.type_check_exchange("exchange", exchange)
         self.__holder: str = configuration.type_check_holder("holder", holder)

src/rp2/plugin/accounting_method/fifo.py

@@ -15,9 +15,9 @@
 from typing import Optional
 
 from rp2.abstract_accounting_method import (
-    AbstractAccountingMethod,
+    AbstractAcquiredLotCandidates,
+    AbstractListAccountingMethod,
     AcquiredLotAndAmount,
-    AcquiredLotCandidates,
     AcquiredLotCandidatesOrder,
 )
 from rp2.abstract_transaction import AbstractTransaction
@@ -28,10 +28,10 @@
 # FIFO (First In, First Out) plugin. See https://www.investopedia.com/terms/l/fifo.asp. This plugin uses universal application, not per-wallet application:
 # this means there is one queue for each coin across every wallet and exchange and the accounting method is applied to each such queue.
 # More on this at https://www.forbes.com/sites/shehanchandrasekera/2020/09/17/what-crypto-taxpayers-need-to-know-about-fifo-lifo-hifo-specific-id/
-class AccountingMethod(AbstractAccountingMethod):
+class AccountingMethod(AbstractListAccountingMethod):
     def seek_non_exhausted_acquired_lot(
         self,
-        lot_candidates: AcquiredLotCandidates,
+        lot_candidates: AbstractAcquiredLotCandidates,
         taxable_event: Optional[AbstractTransaction],
         taxable_event_amount: RP2Decimal,
     ) -> Optional[AcquiredLotAndAmount]: