From 899a448002ed9953297e76cebc4d75179d5b86e8 Mon Sep 17 00:00:00 2001 From: arilieb Date: Tue, 28 Jan 2025 07:13:10 -0800 Subject: [PATCH 1/2] Unverefied event indexed escrowed couples (uwes) database refactor to use B64OnIoDupSuber. Working, needs cleanup and refactor of two more tests. --- src/keri/core/eventing.py | 183 ++++++++++++++++++++++++-------------- src/keri/db/basing.py | 8 +- 2 files changed, 119 insertions(+), 72 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 24bd4708..cee4b195 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -114,6 +114,7 @@ + # Future make Cues dataclasses instead of dicts. Dataclasses so may be converted # to/from dicts easily example: dict(kin="receipt", serder=serder) @@ -5156,8 +5157,25 @@ def escrowUWReceipt(self, serder, wigers, said): # don't know witness pre yet without witness list so no verfer in wiger # if wiger.verfer.transferable: # skip transferable verfers # continue # skip invalid triplets - couple = said.encode("utf-8") + wiger.qb64b - self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) + print(f"escrowUWReceipt SERDER: {serder}, SERDER TYPE: {type(serder)}") + # couple = said.encode("utf-8") + ">" + wiger.qb64b + couple = said + ">" + wiger.qb64 + # print(f"escrowUWReceipt SERDER.PREB: {serder.preb}, SERDER.PREB TYPE: {type(serder.preb)}") + # SERDER.PREB: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', SERDER.PREB TYPE: + print(f"escrowUWReceipt SAID: {said}, SAID TYPE: {type(said)}") + # SAID: EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9, SAID TYPE: + print(f"escrowUWReceipt COUPLE: {couple}, COUPLE TYPE: {type(couple)}") + # COUPLE: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9AADbyebqZQKwn7TqU92Vtw8n2wy5FptP42F1HEmCc9nQLzbXrXuA9SMl9nCZ-vi2bdaeT3aqInXGFAW70QPzM4kJ', COUPLE TYPE: + print(f"escrowUWReceipt SAID.ENCODE(UTF-8): {said.encode("utf-8")}, SAID.ENCODE(UTF-8) TYPE: {type(said.encode("utf-8"))}") + # SAID.ENCODE(UTF-8): b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', SAID.ENCODE(UTF-8) TYPE: + # print(f"SERDER.SNH {serder.qb64}") + print(Saider(qb64=said)) + # print(Siger(qb64=said)) + # print(Siger.index) + # self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) + self.db.uwes.addOn(keys=serder.preb, on=serder.sn, val=(said, wiger.qb64)) + # print(self.db.uwes.getItemIter()) + # log escrowed logger.debug("Kevery process: escrowed unverified witness indexed receipt" " of pre= %s sn=%x dig=%s", serder.pre, serder.sn, said) @@ -6044,78 +6062,104 @@ def processEscrowUnverWitness(self): If successful then remove from escrow table """ - ims = bytearray() - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, ecouple in self.db.getUweItemIter(key=key): - try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key - - # get escrowed receipt's rdiger of receipted event and - # wiger indexed signature of receipted event - rdiger, wiger = deWitnessCouple(ecouple) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(rdiger.qb64b))) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(rdiger.qb64b)) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(rdiger.qb64b)) - - # lookup database dig of the receipted event in pwes escrow - # using pre and sn lastEvt - found = self._processEscrowFindUnver(pre=pre, - sn=sn, - rsaider=rdiger, - wiger=wiger) + # ims = bytearray() + # key = ekey = b'' # both start same. when not same means escrows found + # while True: # break when done + for (pre, snh), (rdiger, wiger) in self.db.uwes.getItemIter(): + print(pre, snh, rdiger, wiger) + try: + # print(f"EKEY: {ekey}, EKEY TYPE: {type(ekey)}") + # # EKEY: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9.00000000000000000000000000000000', EKEY TYPE: + # + # pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key + # print(f"PRE: {pre}, PRE TYPE: {type(pre)}") + # # PRE: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', PRE TYPE: + # + # print(f"SN: {sn}, SN TYPE: {type(sn)}") + # # SN: 0, SN TYPE: + # + # # get escrowed receipt's rdiger of receipted event and + # # wiger indexed signature of receipted event + # print(f"ECOUPLE: {ecouple}, ECOUPLE TYPE: {type(ecouple)}") + # # ECOUPLE: , ECOUPLE TYPE: + # + # rdiger, wiger = deWitnessCouple(ecouple) + # print(f"RDIGER: {rdiger}, RDIGER TYPE: {type(rdiger)}") + # # RDIGER: , RDIGER TYPE: + # print(f"WIGER: {wiger}, WIGER TYPE: {type(wiger)}") + # # WIGER: , WIGER TYPE: + # + # print(f"RDIGER QB64B: {rdiger.qb64b}, RDIGER QB64B TYPE: {type(rdiger.qb64b)}") + # # RDIGER QB64B: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', RDIGER QB64B TYPE: + # print(f"RDIGER QB64B BYTES: {bytes(rdiger.qb64b)}, RDIGER QB64B BYTES TYPE: {type(bytes(rdiger.qb64b))}") + # # RDIGER QB64B BYTES: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', RDIGER QB64B BYTES TYPE: + brdiger = rdiger.encode('utf-8') + # check date if expired then remove escrow. + dtb = self.db.getDts(dgKey(pre, bytes(brdiger))) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", brdiger) - if not found: # no partial witness escrow of event found - # so keep in escrow by raising UnverifiedWitnessReceiptError - logger.debug("Kevery unescrow error: Missing witness " - "receipted evt at pre=%s sn=%x", (pre, sn)) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(brdiger)) - raise UnverifiedWitnessReceiptError("Missing witness " - "receipted evt at pre={} sn={:x}".format(pre, sn)) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", rdiger.qb64b) - except UnverifiedWitnessReceiptError as ex: - # still waiting on missing prior event to validate - # only happens if we process above - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(rdiger.qb64b)) + + # lookup database dig of the receipted event in pwes escrow + # using pre and sn lastEvt + sn = int(snh, 16) + rrdiger = Diger(qb64=rdiger) + wwiger = Siger(qb64=wiger) + found = self._processEscrowFindUnver(pre=pre, + sn=sn, + rsaider=rrdiger, + wiger=wwiger) + + if not found: # no partial witness escrow of event found + # so keep in escrow by raising UnverifiedWitnessReceiptError + logger.debug("Kevery unescrow error: Missing witness " + "receipted evt at pre=%s sn=%x", (pre, sn)) + + raise UnverifiedWitnessReceiptError("Missing witness " + "receipted evt at pre={} sn={:x}".format(pre, sn)) + + except UnverifiedWitnessReceiptError as ex: + # still waiting on missing prior event to validate + # only happens if we process above + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - except Exception as ex: # log diagnostics errors etc - # error other than out of order so remove from OO escrow - self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + except Exception as ex: # log diagnostics errors etc + # error other than out of order so remove from OO escrow + # self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val + self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event pre=%s " - "sn=%s", pre, sn) + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + # self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val + self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) + logger.info("Kevery unescrow succeeded for event pre=%s " + "sn=%s", pre, sn) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + # if ekey == key: # still same so no escrows found on last while iteration + # break + # key = ekey # setup next while iteration, with key after ekey def processEscrowUnverNonTrans(self): """ @@ -6551,6 +6595,9 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): # lookup the database dig of the receipted event in pwes escrow using # snKey(pre,sn) where pre is controller and sn is event sequence number # compare dig to rdiger derived from receipt's dig of receipted event + # print(f"_processEscrowFindUnver PRE: {pre}, PRE TYPE: {type(pre)}") + # print(f"_processEscrowFindUnver SN: {sn}, SN TYPE: {type(sn)}") + found = False for dig in self.db.getPwesIter(key=snKey(pre, sn)): # search entries dig = bytes(dig) # database dig of receipted event diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 958a544e..5e4f0414 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1021,10 +1021,10 @@ def reopen(self, **kwa): self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) - self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) - #self.uwes = subing.CatCesrIoSetSuber(db=self, subkey='uwes.', - #klas=(coring.Saider, indexing.Siger)) - + # self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) + # self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.', + # klas=(coring.Saider, indexing.Siger), sep=">") + self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.', sep='>') self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) From 4e5ddc6964493c1062a31a197c1a68d09d23d6dc Mon Sep 17 00:00:00 2001 From: arilieb Date: Tue, 28 Jan 2025 08:32:10 -0800 Subject: [PATCH 2/2] Fixed tests that were breaking with invalid type errors. Cleaned up commented code and made slight revisions to documentation to reflect changes. --- src/keri/core/eventing.py | 75 ++++----------------------- src/keri/db/basing.py | 106 ++------------------------------------ tests/db/test_basing.py | 83 +++++++++++++++-------------- 3 files changed, 55 insertions(+), 209 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index cee4b195..3bb45e9b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -114,7 +114,6 @@ - # Future make Cues dataclasses instead of dicts. Dataclasses so may be converted # to/from dicts easily example: dict(kin="receipt", serder=serder) @@ -5157,24 +5156,7 @@ def escrowUWReceipt(self, serder, wigers, said): # don't know witness pre yet without witness list so no verfer in wiger # if wiger.verfer.transferable: # skip transferable verfers # continue # skip invalid triplets - print(f"escrowUWReceipt SERDER: {serder}, SERDER TYPE: {type(serder)}") - # couple = said.encode("utf-8") + ">" + wiger.qb64b - couple = said + ">" + wiger.qb64 - # print(f"escrowUWReceipt SERDER.PREB: {serder.preb}, SERDER.PREB TYPE: {type(serder.preb)}") - # SERDER.PREB: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', SERDER.PREB TYPE: - print(f"escrowUWReceipt SAID: {said}, SAID TYPE: {type(said)}") - # SAID: EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9, SAID TYPE: - print(f"escrowUWReceipt COUPLE: {couple}, COUPLE TYPE: {type(couple)}") - # COUPLE: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9AADbyebqZQKwn7TqU92Vtw8n2wy5FptP42F1HEmCc9nQLzbXrXuA9SMl9nCZ-vi2bdaeT3aqInXGFAW70QPzM4kJ', COUPLE TYPE: - print(f"escrowUWReceipt SAID.ENCODE(UTF-8): {said.encode("utf-8")}, SAID.ENCODE(UTF-8) TYPE: {type(said.encode("utf-8"))}") - # SAID.ENCODE(UTF-8): b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', SAID.ENCODE(UTF-8) TYPE: - # print(f"SERDER.SNH {serder.qb64}") - print(Saider(qb64=said)) - # print(Siger(qb64=said)) - # print(Siger.index) - # self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) self.db.uwes.addOn(keys=serder.preb, on=serder.sn, val=(said, wiger.qb64)) - # print(self.db.uwes.getItemIter()) # log escrowed logger.debug("Kevery process: escrowed unverified witness indexed receipt" @@ -6035,8 +6017,7 @@ def processEscrowUnverWitness(self): This allows FIFO processing of escrows for events with same prefix and sn but different digest. - Uses .uwes reads .db.getUwe - was put there by.db.addUwe(self, key, val) which is IOVal with dups. + Uses .uwes reads .db.uwes.get() which is B64OnIoDupSuber Value is couple @@ -6062,47 +6043,18 @@ def processEscrowUnverWitness(self): If successful then remove from escrow table """ - # ims = bytearray() - # key = ekey = b'' # both start same. when not same means escrows found - # while True: # break when done for (pre, snh), (rdiger, wiger) in self.db.uwes.getItemIter(): - print(pre, snh, rdiger, wiger) try: - # print(f"EKEY: {ekey}, EKEY TYPE: {type(ekey)}") - # # EKEY: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9.00000000000000000000000000000000', EKEY TYPE: - # - # pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key - # print(f"PRE: {pre}, PRE TYPE: {type(pre)}") - # # PRE: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', PRE TYPE: - # - # print(f"SN: {sn}, SN TYPE: {type(sn)}") - # # SN: 0, SN TYPE: - # - # # get escrowed receipt's rdiger of receipted event and - # # wiger indexed signature of receipted event - # print(f"ECOUPLE: {ecouple}, ECOUPLE TYPE: {type(ecouple)}") - # # ECOUPLE: , ECOUPLE TYPE: - # - # rdiger, wiger = deWitnessCouple(ecouple) - # print(f"RDIGER: {rdiger}, RDIGER TYPE: {type(rdiger)}") - # # RDIGER: , RDIGER TYPE: - # print(f"WIGER: {wiger}, WIGER TYPE: {type(wiger)}") - # # WIGER: , WIGER TYPE: - # - # print(f"RDIGER QB64B: {rdiger.qb64b}, RDIGER QB64B TYPE: {type(rdiger.qb64b)}") - # # RDIGER QB64B: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', RDIGER QB64B TYPE: - # print(f"RDIGER QB64B BYTES: {bytes(rdiger.qb64b)}, RDIGER QB64B BYTES TYPE: {type(bytes(rdiger.qb64b))}") - # # RDIGER QB64B BYTES: b'EJufgwH347N2kobmes1IQw_1pfMipEFFy0RwinZTtah9', RDIGER QB64B BYTES TYPE: - brdiger = rdiger.encode('utf-8') + rdigerBytes = rdiger.encode('utf-8') # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(brdiger))) + dtb = self.db.getDts(dgKey(pre, bytes(rdigerBytes))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", brdiger) + " at dig = %s", rdigerBytes) raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(brdiger)) + "at dig = {}.".format(rdigerBytes)) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() @@ -6118,12 +6070,12 @@ def processEscrowUnverWitness(self): # lookup database dig of the receipted event in pwes escrow # using pre and sn lastEvt sn = int(snh, 16) - rrdiger = Diger(qb64=rdiger) - wwiger = Siger(qb64=wiger) + rdigerClassed = Diger(qb64=rdiger) + wigerClassed = Siger(qb64=wiger) found = self._processEscrowFindUnver(pre=pre, sn=sn, - rsaider=rrdiger, - wiger=wwiger) + rsaider=rdigerClassed, + wiger=wigerClassed) if not found: # no partial witness escrow of event found # so keep in escrow by raising UnverifiedWitnessReceiptError @@ -6141,7 +6093,6 @@ def processEscrowUnverWitness(self): except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow - # self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) if logger.isEnabledFor(logging.DEBUG): # adds exception data logger.exception("Kevery unescrowed: %s", ex.args[0]) @@ -6152,15 +6103,10 @@ def processEscrowUnverWitness(self): # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. - # self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger)) logger.info("Kevery unescrow succeeded for event pre=%s " "sn=%s", pre, sn) - # if ekey == key: # still same so no escrows found on last while iteration - # break - # key = ekey # setup next while iteration, with key after ekey - def processEscrowUnverNonTrans(self): """ Process escrowed unverified event receipts from nontrans receiptors @@ -6595,9 +6541,6 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): # lookup the database dig of the receipted event in pwes escrow using # snKey(pre,sn) where pre is controller and sn is event sequence number # compare dig to rdiger derived from receipt's dig of receipted event - # print(f"_processEscrowFindUnver PRE: {pre}, PRE TYPE: {type(pre)}") - # print(f"_processEscrowFindUnver SN: {sn}, SN TYPE: {type(sn)}") - found = False for dig in self.db.getPwesIter(key=snKey(pre, sn)): # search entries dig = bytes(dig) # database dig of receipted event diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5e4f0414..b5f9d977 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -813,7 +813,6 @@ class Baser(dbing.LMDBer): witness nontrans identifier prefix from witness list and index is offset into witness list of latest establishment event for receipted event - snKey DB is keyed by receipted event controller prefix plus sn of serialized event More than one value per DB key is allowed @@ -1021,10 +1020,7 @@ def reopen(self, **kwa): self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) - # self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) - # self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.', - # klas=(coring.Saider, indexing.Siger), sep=">") - self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.', sep='>') + self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.') self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) @@ -1395,15 +1391,14 @@ def clearEscrows(self): self.delPses(key=k) for (k, _) in self.getPweItemIter(): self.delPwes(key=k) - for (k, _) in self.getUweItemIter(): - self.delUwes(key=k) for (k, _) in self.getOoeItemIter(): self.delOoes(key=k) for (k, _) in self.getLdeItemIter(): self.delLdes(key=k) for (pre, said), edig in self.qnfs.getItemIter(): self.qnfs.rem(keys=(pre, said)) - + for (pre, snh), rdigerWigerTuple in self.uwes.getItemIter(): + self.uwes.rem(keys=(pre, snh)) for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: @@ -2894,101 +2889,6 @@ def delPwe(self, key, val): """ return self.delIoDupVal(self.pwes, key, val) - def putUwes(self, key, vals): - """ - Use snKey() - Write each entry from list of bytes witness receipt couples vals to key - Witness couple is edig+wig - Adds to existing receipts at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.putIoDupVals(self.uwes, key, vals) - - def addUwe(self, key, val): - """ - Use snKey() - Add receipt couple val bytes as dup to key in db - Witness couple is edig+wig - Adds to existing values at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.addIoDupVal(self.uwes, key, val) - - def getUwes(self, key): - """ - Use snKey() - Return list of receipt couples at key - Witness couple is edig+wig - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupVals(self.uwes, key) - - def getUwesIter(self, key): - """ - Use snKey() - Return iterator of receipt couples at key - Witness couple is edig+wig - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoDupValsIter(self.uwes, key) - - def getUweLast(self, key): - """ - Use snKey() - Return last inserted dup partial signed escrowed receipt couple val at key - Witness couple is edig+wig - Returns None if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupValLast(self.uwes, key) - - def getUweItemIter(self, key=b''): - """ - Use sgKey() - Return iterator of partial signed escrowed receipt couple items at next - key after key. - Items is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getTopIoDupItemIter(self.uwes, key) - #return self.getIoDupItemsNextIter(self.uwes, key, skip) - - def cntUwes(self, key): - """ - Use snKey() - Return count of receipt couples at key - Returns zero if no entry at key - """ - return self.cntIoDupVals(self.uwes, key) - - def delUwes(self, key): - """ - Use snKey() - Deletes all values at key in db. - Returns True If key exists in database Else False - """ - return self.delIoDupVals(self.uwes, key) - - def delUwe(self, key, val): - """ - Use snKey() - Deletes dup val at key in db. - Returns True If dup at exists in db Else False - - Parameters: - key is bytes of key within sub db's keyspace - val is dup val (does not include insertion ordering proem) - """ - return self.delIoDupVal(self.uwes, key, val) - def putOoes(self, key, vals): """ Use snKey() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index ad4a17ee..cd88511f 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -899,52 +899,52 @@ def test_baser(): # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order key = b'A' - vals = [b"z", b"m", b"x", b"a"] - - assert db.getUwes(key) == [] - assert db.getUweLast(key) == None - assert db.cntUwes(key) == 0 - assert db.delUwes(key) == False - assert db.putUwes(key, vals) == True - assert db.getUwes(key) == vals # preserved insertion order - assert db.cntUwes(key) == len(vals) == 4 - assert db.getUweLast(key) == vals[-1] - assert db.putUwes(key, vals=[b'a']) == False # duplicate - assert db.getUwes(key) == vals # no change - assert db.addUwe(key, b'a') == False # duplicate - assert db.addUwe(key, b'b') == True - assert db.getUwes(key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in db.getUwesIter(key)] == [b"z", b"m", b"x", b"a", b"b"] - assert db.delUwes(key) == True - assert db.getUwes(key) == [] + vals = [('z',), ('m',), ('x',), ('a',)] + + assert db.uwes.get(key) == [] + assert db.uwes.getLast(key) == None + assert db.uwes.cnt(key) == 0 + assert db.uwes.rem(key) == False + assert db.uwes.put(key, vals) == True + assert db.uwes.get(key) == vals # preserved insertion order + assert db.uwes.cnt(key) == len(vals) == 4 + assert db.uwes.getLast(key) == vals[-1] + assert db.uwes.put(key, vals=[b'a']) == False # duplicate + assert db.uwes.get(key) == vals # no change + assert db.uwes.add(key, b'a') == False # duplicate + assert db.uwes.add(key, b'b') == True + assert db.uwes.get(key) == [('z',), ('m',), ('x',), ('a',), ('b',)] + assert [val for key, val in db.uwes.getItemIter(key)] == [('z',), ('m',), ('x',), ('a',), ('b',)] + assert db.uwes.rem(key) == True + assert db.uwes.get(key) == [] # Setup Tests for getUweItemsNext and getUweItemsNextIter - aKey = snKey(pre=b'A', sn=1) - aVals = [b"z", b"m", b"x"] - bKey = snKey(pre=b'A', sn=2) - bVals = [b"o", b"r", b"z"] - cKey = snKey(pre=b'A', sn=4) - cVals = [b"h", b"n"] - dKey = snKey(pre=b'A', sn=7) - dVals = [b"k", b"b"] + aKey = ('A', '00000000000000000000000000000001') + aVals = [('z',), ('m',), ('x',)] + bKey = ('A', '00000000000000000000000000000002') + bVals = [('o',), ('r',), ('z',)] + cKey = ('A', '00000000000000000000000000000004') + cVals = [('h',), ('n',)] + dKey = ('A', '00000000000000000000000000000007') + dVals = [('k',), ('b',)] - assert db.putUwes(key=aKey, vals=aVals) - assert db.putUwes(key=bKey, vals=bVals) - assert db.putUwes(key=cKey, vals=cVals) - assert db.putUwes(key=dKey, vals=dVals) + assert db.uwes.put(keys=aKey, vals=aVals) + assert db.uwes.put(keys=bKey, vals=bVals) + assert db.uwes.put(keys=cKey, vals=cVals) + assert db.uwes.put(keys=dKey, vals=dVals) # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemIter()] + items = [item for item in db.uwes.getItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemIter(key=aKey)] + items = [item for item in db.uwes.getItemIter(keys=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey @@ -952,34 +952,34 @@ def test_baser(): assert vals == aVals # bVals - items = [item for item in db.getUweItemIter(key=bKey)] + items = [item for item in db.uwes.getItemIter(keys=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True # cVals - items = [item for item in db.getUweItemIter(key=cKey)] + items = [item for item in db.uwes.getItemIter(keys=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey vals = [val for key, val in items] assert vals == cVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True # dVals - items = [item for item in db.getUweItemIter(key=dKey)] + items = [item for item in db.uwes.getItemIter(keys=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey vals = [val for key, val in items] assert vals == dVals for key, val in items: - assert db.delUwe(ikey, val) == True + assert db.uwes.rem(ikey, val) == True @@ -1825,13 +1825,16 @@ def test_clear_escrows(): db.putVres(key, vals) db.putPses(key, vals) db.putPwes(key, vals) - db.putUwes(key, vals) db.putOoes(key, vals) db.putLdes(key, vals) pre = b'k' snh = b'snh' saidb = b'saidb' + + db.uwes.add(keys=(pre, snh), val=saidb) + assert db.uwes.cnt(keys=(pre, snh)) == 1 + db.qnfs.add(keys=(pre, saidb), val=b"z") assert db.qnfs.cnt(keys=(pre, saidb)) == 1 @@ -1888,7 +1891,7 @@ def test_clear_escrows(): assert db.getVres(key) == [] assert db.getPses(key) == [] assert db.getPwes(key) == [] - assert db.getUwes(key) == [] + assert db.uwes.get(key) == [] assert db.getOoes(key) == [] assert db.getLdes(key) == [] assert db.qnfs.cnt(keys=(pre, saidb)) == 0