Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

uwes Refactor #931

Merged
merged 2 commits into from
Jan 29, 2025
Merged

uwes Refactor #931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
899a448
Unverefied event indexed escrowed couples (uwes) database refactor to…
arilieb Jan 28, 2025
4e5ddc6
Fixed tests that were breaking with invalid type errors. Cleaned up c…
arilieb Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 61 additions & 71 deletions src/keri/core/eventing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5156,8 +5156,8 @@ def escrowUWReceipt(self, serder, wigers, said):
# don't know witness pre yet without witness list so no verfer in wiger
# if wiger.verfer.transferable: # skip transferable verfers
# continue # skip invalid triplets
couple = said.encode("utf-8") + wiger.qb64b
self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple)
self.db.uwes.addOn(keys=serder.preb, on=serder.sn, val=(said, wiger.qb64))

# log escrowed
logger.debug("Kevery process: escrowed unverified witness indexed receipt"
" of pre= %s sn=%x dig=%s", serder.pre, serder.sn, said)
Expand Down Expand Up @@ -6017,8 +6017,7 @@ def processEscrowUnverWitness(self):
This allows FIFO processing of escrows for events with same prefix and
sn but different digest.

Uses .uwes reads .db.getUwe
was put there by.db.addUwe(self, key, val) which is IOVal with dups.
Uses .uwes reads .db.uwes.get() which is B64OnIoDupSuber

Value is couple

Expand All @@ -6044,78 +6043,69 @@ def processEscrowUnverWitness(self):
If successful then remove from escrow table
"""

ims = bytearray()
key = ekey = b'' # both start same. when not same means escrows found
while True: # break when done
for ekey, ecouple in self.db.getUweItemIter(key=key):
try:
pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key

# get escrowed receipt's rdiger of receipted event and
# wiger indexed signature of receipted event
rdiger, wiger = deWitnessCouple(ecouple)

# check date if expired then remove escrow.
dtb = self.db.getDts(dgKey(pre, bytes(rdiger.qb64b)))
if dtb is None: # othewise is a datetime as bytes
# no date time so raise ValidationError which unescrows below
logger.info("Kevery unescrow error: Missing event datetime"
" at dig = %s", rdiger.qb64b)

raise ValidationError("Missing escrowed event datetime "
"at dig = {}.".format(rdiger.qb64b))

# do date math here and discard if stale nowIso8601() bytes
dtnow = helping.nowUTC()
dte = helping.fromIso8601(bytes(dtb))
if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE):
# escrow stale so raise ValidationError which unescrows below
logger.info("Kevery unescrow error: Stale event escrow "
" at dig = %s", rdiger.qb64b)

raise ValidationError("Stale event escrow "
"at dig = {}.".format(rdiger.qb64b))

# lookup database dig of the receipted event in pwes escrow
# using pre and sn lastEvt
found = self._processEscrowFindUnver(pre=pre,
sn=sn,
rsaider=rdiger,
wiger=wiger)

if not found: # no partial witness escrow of event found
# so keep in escrow by raising UnverifiedWitnessReceiptError
logger.debug("Kevery unescrow error: Missing witness "
"receipted evt at pre=%s sn=%x", (pre, sn))
for (pre, snh), (rdiger, wiger) in self.db.uwes.getItemIter():
try:
rdigerBytes = rdiger.encode('utf-8')
# check date if expired then remove escrow.
dtb = self.db.getDts(dgKey(pre, bytes(rdigerBytes)))
if dtb is None: # othewise is a datetime as bytes
# no date time so raise ValidationError which unescrows below
logger.info("Kevery unescrow error: Missing event datetime"
" at dig = %s", rdigerBytes)

raise UnverifiedWitnessReceiptError("Missing witness "
"receipted evt at pre={} sn={:x}".format(pre, sn))
raise ValidationError("Missing escrowed event datetime "
"at dig = {}.".format(rdigerBytes))

except UnverifiedWitnessReceiptError as ex:
# still waiting on missing prior event to validate
# only happens if we process above
if logger.isEnabledFor(logging.DEBUG): # adds exception data
logger.exception("Kevery unescrow failed: %s", ex.args[0])
# do date math here and discard if stale nowIso8601() bytes
dtnow = helping.nowUTC()
dte = helping.fromIso8601(bytes(dtb))
if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE):
# escrow stale so raise ValidationError which unescrows below
logger.info("Kevery unescrow error: Stale event escrow "
" at dig = %s", rdiger.qb64b)

except Exception as ex: # log diagnostics errors etc
# error other than out of order so remove from OO escrow
self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val
if logger.isEnabledFor(logging.DEBUG): # adds exception data
logger.exception("Kevery unescrowed: %s", ex.args[0])
else:
logger.error("Kevery unescrowed: %s", ex.args[0])
raise ValidationError("Stale event escrow "
"at dig = {}.".format(rdiger.qb64b))

# lookup database dig of the receipted event in pwes escrow
# using pre and sn lastEvt
sn = int(snh, 16)
rdigerClassed = Diger(qb64=rdiger)
wigerClassed = Siger(qb64=wiger)
found = self._processEscrowFindUnver(pre=pre,
sn=sn,
rsaider=rdigerClassed,
wiger=wigerClassed)

if not found: # no partial witness escrow of event found
# so keep in escrow by raising UnverifiedWitnessReceiptError
logger.debug("Kevery unescrow error: Missing witness "
"receipted evt at pre=%s sn=%x", (pre, sn))

raise UnverifiedWitnessReceiptError("Missing witness "
"receipted evt at pre={} sn={:x}".format(pre, sn))

except UnverifiedWitnessReceiptError as ex:
# still waiting on missing prior event to validate
# only happens if we process above
if logger.isEnabledFor(logging.DEBUG): # adds exception data
logger.exception("Kevery unescrow failed: %s", ex.args[0])

else: # unescrow succeeded, remove from escrow
# We don't remove all escrows at pre,sn because some might be
# duplicitous so we process remaining escrows in spite of found
# valid event escrow.
self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val
logger.info("Kevery unescrow succeeded for event pre=%s "
"sn=%s", pre, sn)
except Exception as ex: # log diagnostics errors etc
# error other than out of order so remove from OO escrow
self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger))
if logger.isEnabledFor(logging.DEBUG): # adds exception data
logger.exception("Kevery unescrowed: %s", ex.args[0])
else:
logger.error("Kevery unescrowed: %s", ex.args[0])

if ekey == key: # still same so no escrows found on last while iteration
break
key = ekey # setup next while iteration, with key after ekey
else: # unescrow succeeded, remove from escrow
# We don't remove all escrows at pre,sn because some might be
# duplicitous so we process remaining escrows in spite of found
# valid event escrow.
self.db.uwes.rem(keys=(pre, snh), val=(rdiger, wiger))
logger.info("Kevery unescrow succeeded for event pre=%s "
"sn=%s", pre, sn)

def processEscrowUnverNonTrans(self):
"""
Expand Down
106 changes: 3 additions & 103 deletions src/keri/db/basing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,7 +813,6 @@ class Baser(dbing.LMDBer):
witness nontrans identifier prefix from witness list and index
is offset into witness list of latest establishment event for
receipted event
snKey
DB is keyed by receipted event controller prefix plus sn
of serialized event
More than one value per DB key is allowed
Expand Down Expand Up @@ -1021,10 +1020,7 @@ def reopen(self, **kwa):
self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.')
self.udes = subing.CatCesrSuber(db=self, subkey='udes.',
klas=(coring.Seqner, coring.Saider))
self.uwes = self.env.open_db(key=b'uwes.', dupsort=True)
#self.uwes = subing.CatCesrIoSetSuber(db=self, subkey='uwes.',
#klas=(coring.Saider, indexing.Siger))

self.uwes = subing.B64OnIoDupSuber(db=self, subkey='uwes.')
self.ooes = self.env.open_db(key=b'ooes.', dupsort=True)
self.dels = self.env.open_db(key=b'dels.', dupsort=True)
self.ldes = self.env.open_db(key=b'ldes.', dupsort=True)
Expand Down Expand Up @@ -1395,15 +1391,14 @@ def clearEscrows(self):
self.delPses(key=k)
for (k, _) in self.getPweItemIter():
self.delPwes(key=k)
for (k, _) in self.getUweItemIter():
self.delUwes(key=k)
for (k, _) in self.getOoeItemIter():
self.delOoes(key=k)
for (k, _) in self.getLdeItemIter():
self.delLdes(key=k)
for (pre, said), edig in self.qnfs.getItemIter():
self.qnfs.rem(keys=(pre, said))

for (pre, snh), rdigerWigerTuple in self.uwes.getItemIter():
self.uwes.rem(keys=(pre, snh))

for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi,
self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]:
Expand Down Expand Up @@ -2894,101 +2889,6 @@ def delPwe(self, key, val):
"""
return self.delIoDupVal(self.pwes, key, val)

def putUwes(self, key, vals):
"""
Use snKey()
Write each entry from list of bytes witness receipt couples vals to key
Witness couple is edig+wig
Adds to existing receipts at key if any
Returns True If at least one of vals is added as dup, False otherwise
Duplicates are inserted in insertion order.
"""
return self.putIoDupVals(self.uwes, key, vals)

def addUwe(self, key, val):
"""
Use snKey()
Add receipt couple val bytes as dup to key in db
Witness couple is edig+wig
Adds to existing values at key if any
Returns True If at least one of vals is added as dup, False otherwise
Duplicates are inserted in insertion order.
"""
return self.addIoDupVal(self.uwes, key, val)

def getUwes(self, key):
"""
Use snKey()
Return list of receipt couples at key
Witness couple is edig+wig
Returns empty list if no entry at key
Duplicates are retrieved in insertion order.
"""
return self.getIoDupVals(self.uwes, key)

def getUwesIter(self, key):
"""
Use snKey()
Return iterator of receipt couples at key
Witness couple is edig+wig
Raises StopIteration Error when empty
Duplicates are retrieved in insertion order.
"""
return self.getIoDupValsIter(self.uwes, key)

def getUweLast(self, key):
"""
Use snKey()
Return last inserted dup partial signed escrowed receipt couple val at key
Witness couple is edig+wig
Returns None if no entry at key
Duplicates are retrieved in insertion order.
"""
return self.getIoDupValLast(self.uwes, key)

def getUweItemIter(self, key=b''):
"""
Use sgKey()
Return iterator of partial signed escrowed receipt couple items at next
key after key.
Items is (key, val) where proem has already been stripped from val
val is couple edig+wig
If key is b'' empty then returns dup items at first key.
If skip is False and key is not b'' empty then returns dup items at key
Raises StopIteration Error when empty
Duplicates are retrieved in insertion order.
"""
return self.getTopIoDupItemIter(self.uwes, key)
#return self.getIoDupItemsNextIter(self.uwes, key, skip)

def cntUwes(self, key):
"""
Use snKey()
Return count of receipt couples at key
Returns zero if no entry at key
"""
return self.cntIoDupVals(self.uwes, key)

def delUwes(self, key):
"""
Use snKey()
Deletes all values at key in db.
Returns True If key exists in database Else False
"""
return self.delIoDupVals(self.uwes, key)

def delUwe(self, key, val):
"""
Use snKey()
Deletes dup val at key in db.
Returns True If dup at exists in db Else False

Parameters:
key is bytes of key within sub db's keyspace
val is dup val (does not include insertion ordering proem)
"""
return self.delIoDupVal(self.uwes, key, val)

def putOoes(self, key, vals):
"""
Use snKey()
Expand Down
Loading