From 7f3c6a97ebfe2529500d156f58cd1ec990c2e6a8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Jan 2026 10:40:49 -0700 Subject: [PATCH 01/27] use uuid1 instead of uuid4 so that memos are lexocographic in time --- src/hio/core/memo/memoing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 474f3dc..1ad137c 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -943,7 +943,7 @@ def rend(self, memo, vid=None): vid = b"" if vid is None else vid[:vs].encode() # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned - mid = encodeB64(bytes([0] * ps) + uuid.uuid4().bytes)[ps:] # prepad, convert, and strip + mid = encodeB64(bytes([0] * ps) + uuid.uuid1().bytes)[ps:] # prepad, convert, and strip mid = self.code.encode() + mid # fully qualified mid with prefix code ml = len(memo) From 470b378ee2d3390116b0beb38e12c50bfe3c2998 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Jan 2026 14:18:17 -0700 Subject: [PATCH 02/27] clean up doc strings and comments in Memoer --- src/hio/core/memo/memoing.py | 95 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 1ad137c..a202d34 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -3,9 +3,10 @@ hio.core.memoing Module Mixin Base Classes that add support for memograms over datagram based transports. -In this context, a memogram is a larger memogram that is partitioned into -smaller parts as transport specific datagrams. This allows larger messages to -be transported over datagram transports than the underlying transport can support. +In this context, a memo made is a larger message that is partitioned into +smaller memograms parts that fit into transport specific datagrams. +This allows larger messages (memos) to be transported over datagram transports +which messages are larger than a single datagram in the underlying transport. """ import socket @@ -31,17 +32,21 @@ """Sizage: namedtuple for gram header part size entries in Memoer code tables -cs is the code part size int number of chars in code part +cs is the code part size int number of chars in code part (serialization code) ms is the mid part size int number of chars in the mid part (memoID) vs is the vid part size int number of chars in the vid part (verification ID) ns is the neck part size int number of chars for the gram number in all grams - and the additional neck that also appears in first gram for gram count + it is also the size of the gram count that only appears in the zeroth gram + The zeroth gram has a long neck, two ns sized fields all other grams have a + short neck (1 ns sized field) ss is the signature part size int number of chars in the signature part. the signature part is discontinuously attached after the body part but - its size is included in over head computation for the body size -hs is the head size int number of chars for other grams no-neck. - hs = cs + ms + vs + ns + ss. The size for the first gram with neck is - hs + ns + its size is included in the overhead size computation for the body size +hs is the head size int number of chars + short next + hs = cs + ms + vs + ns + ss + zeroth long neck + hs = cs + ms + vs + ns + ns + ss """ Sizage = namedtuple("Sizage", "cs ms vs ss ns hs") @@ -78,36 +83,38 @@ def __iter__(self): class Memoer(hioing.Mixin): - """ - Memoer mixin base class to adds memogram support to a transport class. - Memoer supports asynchronous memograms. Provides common methods for subclasses. + """Memoer mixin base class to adds memogram support to a transport class. + Memoer supports memos composed of asynchronous memograms. + Provides common methods for subclasses. - A memogram is a higher level construct that sits on top of a datagram. + A memogram is a higher level construct that sits on top of a datagram transport. A memogram supports the segmentation and desegmentation of memos to respect the underlying datagram size, buffer, and fragmentation behavior. - Layering a reliable transaction protocol on top of a memogram enables reliable - asynchronous messaging over unreliable datagram transport protocols. + Layering a reliable transaction protocol for memos on top of a memogram + transport enables reliable asynchronous messaging over unreliable datagram + transport protocols. - When the datagram protocol is already reliable, then a memogram enables + When the datagram protocol is already reliable, then a memogram transport enables larger memos (messages) than that natively supported by the datagram. Usage: Do not instantiate directly but use as a mixin with a transport class in order to create a new subclass that adds memogram support to the - transport class. For example MemoerUdp or MemoerUxd + datagram transport class. For example MemoerUdp or MemoerUxd Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. - On the receive side each complete datagram (gram) is put in a gram receive - deque as a segment of a memo. These deques are indexed by the sender's - source addr. The grams in the gram recieve deque are then desegmented into a memo + On the receive side each complete memogram (gram) is put in a gram receive + deque as a memogram (datagram sized) segment of a memo. + These deques are indexed by the sender's source addr. + The grams in the gram recieve deque are then desegmented into a memo and placed in the memo deque for consumption by the application or some other higher level protocol. On the transmit side memos are placed in a deque (double ended queue). Each - memo is then segmented into grams (datagrams) that respect the size constraints + memo is then segmented into grams (memograms) that respect the size constraints of the underlying datagram transport. These grams are placed in the outgoing gram deque. Each entry in this deque is a duple of form: (gram: bytes, dst: str). Each duple is pulled off the deque and its @@ -144,7 +151,8 @@ class Memoer(hioing.Mixin): Inherited Attributes: - name (str): unique name for Memoer transport. Used to manage. + name (str): unique name for Memoer transport. + Used to manage multiple instances. opened (bool): True means transport open for use False otherwise bc (int | None): count of transport buffers of MaxGramSize @@ -152,13 +160,10 @@ class Memoer(hioing.Mixin): Attributes: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - rxgs (dict): holding rx (receive) (data) gram deques of grams. - Each item in dict has key=src and val=deque of grames received - rxgs (dict): keyed by mid (memoID) with value of dict where each deque - holds incomplete memo grams for that mid. - The mid appears in every gram from the same memo. - The value dict is keyed by the gram number with value - that is the gram bytes. + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. sources (dict): keyed by mid (memoID) that holds the src for the memo. This enables reattaching src to fused memo in rxms deque tuple. counts (dict): keyed by mid (memoID) that holds the gram count from @@ -188,15 +193,6 @@ class Memoer(hioing.Mixin): echos (deque): holding echo receive duples for testing. Each duple of form: (gram: bytes, dst: str). - - Hidden: - _code (bytes | None): see size property - _curt (bool): see curt property - _size (int): see size property - _verific (bool): see verific property - - - Properties: code (bytes | None): gram code for gram header when rending for tx curt (bool): True means when rending for tx encode header in base2 @@ -211,6 +207,11 @@ class Memoer(hioing.Mixin): False otherwise + Hidden: + _code (bytes | None): see size property + _curt (bool): see curt property + _size (int): see size property + _verific (bool): see verific property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex @@ -457,7 +458,6 @@ def wiff(self, gram): """Determines encoding of gram bytes header when parsing grams. The encoding maybe either base2 or base64. - Returns: curt (bool): True means base2 encoding False means base64 encoding @@ -489,7 +489,7 @@ def wiff(self, gram): the count from 0 to 63. There is one collision, however, between Base2 and Base 64 for invalid gram headers. This is because 0o27 is the base2 code for ascii 'V'. This means that Base2 encodings - that begin 0o27 witch is the Base2 encoding of the ascii 'V, would be + that begin with 0o27 wich is the Base2 encoding of the ascii 'V, would be incorrectly detected as text not binary. """ @@ -571,11 +571,10 @@ def pick(self, gram): raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" f" < {hs + 1}.") - #mid = encodeB64(bytes([0] * ps) + gram[cs:cms])[ps:].decode() # prepad, convert, strip mid = encodeB64(gram[:cms]).decode() # fully qualified with prefix code vid = encodeB64(gram[cms:cms+vs]).decode() # must be on 24 bit boundary gn = int.from_bytes(gram[cms+vs:cms+vs+ns]) - if gn == 0: # first (zeroth) gram so get neck + if gn == 0: # first (zeroth) gram so long neck if len(gram) < hs + ns + 1: raise hioing.MemoerError(f"Not enough rx bytes for b2 " f"gram < {hs + ns + 1}.") @@ -585,7 +584,7 @@ def pick(self, gram): del gram[-ss if ss else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram del gram[:hs-ss+ns] # strip of fore head leaving body in gram - else: # non-first gram no neck + else: # non-zeroth gram so short neck gc = None sig = encodeB64(gram[-ss if ss else len(gram):]) del gram[-ss if ss else len(gram):] # strip sig @@ -609,7 +608,7 @@ def pick(self, gram): mid = bytes(gram[:cs+ms]).decode() # fully qualified with prefix code vid = bytes(gram[cs+ms:cs+ms+vs]).decode() # must be on 24 bit boundary gn = helping.b64ToInt(gram[cs+ms+vs:cs+ms+vs+ns]) - if gn == 0: # first (zeroth) gram so get neck + if gn == 0: # first (zeroth) gram so long neck if len(gram) < hs + ns + 1: raise hioing.MemoerError(f"Not enough rx bytes for b64 " f"gram < {hs + ns + 1}.") @@ -619,7 +618,7 @@ def pick(self, gram): del gram[-ss if ss else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram del gram[:hs-ss+ns] # strip of fore head leaving body in gram - else: # non-first gram no neck + else: # non-zeroth gram short neck gc = None sig = bytes(gram[-ss if ss else len(gram):]) del gram[-ss if ss else len(gram):] # strip sig @@ -635,7 +634,7 @@ def pick(self, gram): def receive(self, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. + """Attemps to receive bytes from remote source. Must be overridden in subclass. This is a stub to define mixin interface. @@ -646,7 +645,7 @@ def receive(self, *, echoic=False): indicates nothing to receive of form (b'', None) Returns: - duple (tuple): of form (data: bytes, src: str) where data is the + duple (tuple): of form (data: bytes, src: str|None) where data is the bytes of received data and src is the source address. When no data the duple is (b'', None) unless echoic is True then pop off echo from .echos @@ -702,7 +701,7 @@ def _serviceOneReceived(self, *, echoic=False): if not gram: # no received data return False # so try again later - gram = bytearray(gram)# make copy bytearray so can strip off header + gram = bytearray(gram) # make copy bytearray so can strip off header try: mid, vid, gn, gc = self.pick(gram) # parse and strip off head leaving body From 4777a831d29a28b07778d7dcc2b23852e6977088 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 6 Jan 2026 09:14:19 -0700 Subject: [PATCH 03/27] some clean up review and doc string clarification --- src/hio/core/memo/memoing.py | 84 ++++++++++++++++++++---------------- src/hio/core/udp/udping.py | 3 +- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index a202d34..fdb18e3 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -106,6 +106,16 @@ class Memoer(hioing.Mixin): Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. + On the transmit side memos are placed in a memo deque (double ended queue). + Each memo is then segmented into grams (memograms) that respect the size constraints + of the underlying datagram transport. These grams are placed in the outgoing + gram deque. Each entry in this deque is a duple of form: + (gram: bytes, dst: str). Each duple is pulled off the deque and its + gram is put in bytearray for transport. + + memo -> .txms deque -> rend -> grams -> .txgs deque -> send -> .txbs + + On the receive side each complete memogram (gram) is put in a gram receive deque as a memogram (datagram sized) segment of a memo. These deques are indexed by the sender's source addr. @@ -113,12 +123,9 @@ class Memoer(hioing.Mixin): and placed in the memo deque for consumption by the application or some other higher level protocol. - On the transmit side memos are placed in a deque (double ended queue). Each - memo is then segmented into grams (memograms) that respect the size constraints - of the underlying datagram transport. These grams are placed in the outgoing - gram deque. Each entry in this deque is a duple of form: - (gram: bytes, dst: str). Each duple is pulled off the deque and its - gram is put in bytearray for transport. + receive -> (gram, src) -> grams parsed to .rxgs .counts .vids .sources -> + fuse -> memo .rxms deque + When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation @@ -867,26 +874,6 @@ def serviceAllRx(self): self.serviceRxMemos() - def send(self, txbs, dst, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. - Must be overridden in subclass. - This is a stub to define mixin interface. - - Returns: - count (int): bytes actually sent - - Parameters: - txbs (bytes | bytearray): of bytes to send - dst (str): remote destination address - echoic (bool): True means echo sends into receives via. echos - False measn do not echo - """ - if echoic: - self.echos.append((bytes(txbs), dst)) # make copy - return len(txbs) # all sent - - - def memoit(self, memo, dst, vid=None): """Append (memo, dst, vid) tuple to .txms deque @@ -1003,6 +990,29 @@ def rend(self, memo, vid=None): return grams + def send(self, gram, dst, *, echoic=False): + """Attemps to send bytes in txbs to remote destination dst. + Must be overridden in subclass. + This is a stub to define mixin interface. + + Returns: + count (int): bytes actually sent + + Parameters: + gram (bytearray): of bytes to send + dst (str): remote destination address + echoic (bool): True means echo sends into receives via. echos + False measn do not echo + """ + if echoic: + self.echos.append((bytes(gram), dst)) # make copy + + # for example: cnt = self.ls.sendto(gram, dst) + # cnt == int number of bytes sent + cnt = len(gram) # all sent + return cnt # bytes sent + + def _serviceOneTxMemo(self): """Service one (memo, dst, vid) tuple from .txms deque where tuple is of form: (memo: str, dst: str, vid: str) where: @@ -1040,7 +1050,7 @@ def serviceTxMemos(self): def gramit(self, gram, dst): - """Append (gram, dst) duple to .txgs deque + """Append (gram, dst) duple to .txgs deque. Utility method for testing. Parameters: gram (bytes): gram to be sent @@ -1057,12 +1067,13 @@ def _serviceOnceTxGrams(self, *, echoic=False): echoic (bool): True means echo sends into receives via. echos False measn do not echo - Each entry in.txgs is duple of form: (gram: bytes, dst: str) + Each entry in .txgs is duple of form: (gram: bytes, dst: str) where: gram (bytes): is outgoing gram segment from associated memo dst (str): is far peer destination address - .txbs is duple of form: (gram: bytearray, dst: str) + The latest gram from .txgs is put in .txbs whose value is duple of form: + (gram: bytearray, dst: str) bytearray so can partial send where: gram (bytearray): holds incompletly sent gram portion if any dst (str | None): destination or None if last completely sent @@ -1084,17 +1095,17 @@ def _serviceOnceTxGrams(self, *, echoic=False): send remainder of a datagram when using nonblocking sends. When the far side peer is unavailable the gram is dropped. This means - that unreliable transports need to have a timeour retry mechanism. + that unreliable transports need to have a timeout retry mechanism. Internally, a dst of None in the .txbs duple indicates its ok to take another gram from the .txgs deque if any and start sending it. """ - gram, dst = self.txbs - if dst == None: + gram, dst = self.txbs # split out saved partial send if any + if dst == None: # no partial send remaining so get new gram try: - gram, dst = self.txgs.popleft() - gram = bytearray(gram) + gram, dst = self.txgs.popleft() # next gram + gram = bytearray(gram) # ensure bytearray except IndexError: return False # nothing more to send, return False to try later @@ -1126,8 +1137,8 @@ def _serviceOnceTxGrams(self, *, echoic=False): if cnt: del gram[:cnt] # remove from buffer those bytes sent if not gram: # all sent - dst = None # indicate by setting dst to None - self.txbs = (gram, dst) # update txbs to indicate if completely sent + dst = None # done indicated by setting dst to None + self.txbs = (gram, dst) # update .txbs to indicate if completely sent return (False if dst else True) # incomplete return False, else True @@ -1159,6 +1170,7 @@ def serviceTxGrams(self, *, echoic=False): break # try again later + def serviceAllTxOnce(self): """Service the transmit side once (non-greedy) one transmission. """ diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 8eb6f53..78d2ea2 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -208,6 +208,7 @@ def receive(self, **kwa): return (data, sa) + def send(self, data, dst, **kwa): """Perform non blocking send on socket. @@ -215,7 +216,7 @@ def send(self, data, dst, **kwa): cnt (int): count of bytes actually sent, may be less than len(data). Parameters: - data (bytes): payload to send + data (bytes | bytearray): payload to send (txbs) dst (str): udp destination addr duple of form (host: str, port: int) """ try: From 4261af379f75d9d8cd15ffc87ba81d074aae6bc5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 8 Jan 2026 10:21:03 -0700 Subject: [PATCH 04/27] some cleanup normalizing interface of uxd and udp for memoer --- src/hio/core/memo/memoing.py | 6 ++++-- src/hio/core/udp/udping.py | 25 ++++++++++++++++++++----- src/hio/core/uxd/uxding.py | 6 ++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index fdb18e3..7c1aee3 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -143,7 +143,8 @@ class Memoer(hioing.Mixin): Memo segmentation/desegmentation information is embedded in the grams. Inherited Class Attributes: - MaxGramSize (int): absolute max gram size on tx with overhead + MaxGramSize (int): absolute max gram size on tx with overhead, override + in subclass Class Attributes: Version (Versionage): default version consisting of namedtuple of form @@ -233,7 +234,8 @@ class Memoer(hioing.Mixin): #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size MaxGramCount = 16777215 # (2**24-1) absolute max gram count - MaxGramSize = 65535 # (2**16-1) Overridden in subclass + MaxGramSize = 65535 # (2**16-1) absolute max gram size overridden in subclass + def __init__(self, *, diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 78d2ea2..53407a8 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -17,33 +17,48 @@ logger = help.ogler.getLogger() + +UDP_IPV4_MAX_SAFE_PAYLOAD = 548 # IPV4 MTU 576 - 28 headers +UDP_IPv6_MAX_SAFE_PAYLOAD = 1240 # IPV6 MTU 1280 - 40 headers UDP_MAX_DATAGRAM_SIZE = (2 ** 16) - 1 # 65535 -UDP_MAX_SAFE_PAYLOAD = 548 # IPV4 MTU 576 - udp headers 28 -# IPV6 MTU is 1280 but headers are bigger UDP_MAX_PACKET_SIZE = min(1024, UDP_MAX_DATAGRAM_SIZE) # assumes IPV6 capable equipment - +# the only way to fragment ipv6 packet is for source to do it. Never done by +# routers en-route. IPV6 has many extension headers that are only used if set +# up at the source. The fragment header is an 8 byte extension header. +# https://www.geeksforgeeks.org/computer-networks/ipv6-fragmentation-header/ class Peer(hioing.Mixin): """Class to manage non blocking I/O on UDP socket. + Class Attributes: + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max bytes in in datagram for this transport + + Attributes: name (str): unique identifier of peer for managment purposes ha (tuple): host address of form (host,port) of type (str, int) of this peer's socket address. + + bc (int | None): count of transport buffers of MaxGramSize bs (int): buffer size wl (WireLog): instance ref for debug logging of over the wire tx and rx - bcast (bool): True enables sending to broadcast addresses from local socket - False otherwise + ls (socket.socket): local socket of this Peer opened (bool): True local socket is created and opened. False otherwise + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + Properties: host (str): element of .ha duple port (int): element of .ha duple """ + BufSize = 65535 # 2 ** 16 - 1 default buffersize + MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass def __init__(self, *, name='main', diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 6e268e1..734b565 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -62,8 +62,9 @@ class Peer(filing.Filer): Class Attributes: Umask (int): octal default umask permissions such as 0o022 MaxUxdPathSize (int:) max characters in uxd file path + BufSize (int): used to set default buffer size for transport datagram buffers MaxGramSize (int): max bytes in in datagram for this transport - BufSize (int): used to set buffer size for transport datagram buffers + Attributes: @@ -91,8 +92,9 @@ class Peer(filing.Filer): Fext = "uxd" Umask = 0o022 # default MaxUxdPathSize = 108 - MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass BufSize = 65535 # 2 ** 16 - 1 default buffersize + MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass + def __init__(self, *, umask=None, bc=None, bs=None, wl=None, From 81a400b9c14e6873507d71f9a0781b5a7d86a4d2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 8 Jan 2026 11:07:28 -0700 Subject: [PATCH 05/27] normalized udp and uxd interfaces --- src/hio/core/udp/udping.py | 28 +++++--- src/hio/core/uxd/peermemoing.py | 4 -- src/hio/core/uxd/uxding.py | 17 +++-- tests/core/udp/test_udp.py | 101 +++++++++++++++++++++++++++++ tests/core/uxd/test_peer_memoer.py | 2 - 5 files changed, 133 insertions(+), 19 deletions(-) diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 53407a8..4f37ef7 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -32,7 +32,7 @@ class Peer(hioing.Mixin): """Class to manage non blocking I/O on UDP socket. Class Attributes: - BufSize (int): used to set default buffer size for transport datagram buffers + BufSize (int): used to set default buffer size for transport datagram buffers MaxGramSize (int): max bytes in in datagram for this transport @@ -58,14 +58,15 @@ class Peer(hioing.Mixin): """ BufSize = 65535 # 2 ** 16 - 1 default buffersize - MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass + MaxGramSize = UDP_IPv6_MAX_SAFE_PAYLOAD # 1240 def __init__(self, *, name='main', ha=None, host='', port=55000, - bufsize=1024, + bc=None, + bs=None, wl=None, bcast=False, **kwa): @@ -76,25 +77,36 @@ def __init__(self, *, ha (tuple): local socket (host, port) address duple of type (str, int) host (str): address where '' means any interface on host port (int): socket port - bs (int): buffer size - wl (WireLog): instance to log over the wire tx and rx + bc (int | None): count of transport buffers of MaxGramSize + bs (int | None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize + wl (WireLog): instance ref for debug logging of over the wire tx and rx bcast (bool): True enables sending to broadcast addresses from local socket False otherwise """ - super(Peer, self).__init__(**kwa) + self.name = name self.ha = ha or (host, port) # ha = host address duple (host, port) host, port = self.ha host = coring.normalizeHost(host) # ip host address self.ha = (host, port) - self.bs = bufsize + self.bc = int(bc) if bc is not None and bc > 0 else None + if self.bc: + self.bs = self.MaxGramSize * self.bc + else: + self.bs = bs if bs is not None else self.BufSize + self.wl = wl self.bcast = bcast - self.ls = None # local socket for this Peer + self.ls = None # local socket for this Peer needs to be opened/bound self.opened = False + super(Peer, self).__init__(**kwa) + @property def host(self): diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index 606d4bd..d3d7b0f 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -29,13 +29,9 @@ class PeerMemoer(Peer, Memoer): Class Attributes: - Attributes: - - """ - def __init__(self, *, bc=4, **kwa): """Initialization method for instance. diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 734b565..ac8df36 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -97,9 +97,16 @@ class Peer(filing.Filer): - def __init__(self, *, umask=None, bc=None, bs=None, wl=None, - reopen=False, clear=True, - filed=False, extensioned=True, **kwa): + def __init__(self, *, + umask=None, + bc=None, + bs=None, + wl=None, + reopen=False, + clear=True, + filed=False, + extensioned=True, + **kwa): """Initialization method for instance. Inherited Parameters: @@ -125,13 +132,13 @@ def __init__(self, *, umask=None, bc=None, bs=None, wl=None, wl (WireLog): instance ref for debug logging of over the wire tx and rx """ self.umask = umask # only change umask if umask is not None below - self.bc = bc + + self.bc = int(bc) if bc is not None and bc > 0 else None if self.bc: self.bs = self.MaxGramSize * self.bc else: self.bs = bs if bs is not None else self.BufSize - self.wl = wl self.ls = None # local socket of this Peer, needs to be opened/bound diff --git a/tests/core/udp/test_udp.py b/tests/core/udp/test_udp.py index cf24662..c8a8ad8 100644 --- a/tests/core/udp/test_udp.py +++ b/tests/core/udp/test_udp.py @@ -105,6 +105,106 @@ def addrBytes(ha): assert not wl.opened """Done Test""" +def test_udp(): + """ Test the udp connection between two peers with bc and bs + + """ + + tymist = tyming.Tymist() + with (wiring.openWL(samed=True, filed=True) as wl): + host = '127.0.0.1' + alphaPort = 6101 + betaPort = 6102 + alphaHa = (host, alphaPort) + betaHa = (host, betaPort) + bc = 1024 + + alpha = udping.Peer(host=host, port = alphaPort, wl=wl, bc=bc) + assert not alpha.opened + assert alpha.name == 'main' # default + assert alpha.bc == bc + assert alpha.MaxGramSize == 1240 # max safe payload + assert alpha.bs == bc * alpha.MaxGramSize == 1269760 + assert alpha.actualBufSizes() == (0, 0) # not opened yet + assert alpha.reopen() + assert alpha.opened + assert alpha.ha == alphaHa + assert alpha.actualBufSizes() == (alpha.bs, alpha.bs) == (1269760, 1269760) + + bs = 2 ** 15 - 1 + assert bs == 32767 + beta = udping.Peer(name='beta',host=host, port = betaPort, wl=wl, bs=bs) # host on port 6102 + assert not beta.opened + assert beta.name == 'beta' + assert beta.bc is None + assert beta.bs == bs + assert beta.actualBufSizes() == (0, 0) # not opened yet + assert beta.reopen() + assert beta.opened + assert beta.ha == betaHa + bses = beta.actualBufSizes() + assert bses[0] >= beta.bs + assert bses[1] >= beta.bs + + msgOut = b"alpha sends to beta" + alpha.send(msgOut, beta.ha) + time.sleep(0.05) + msgIn, src = beta.receive() + assert msgOut == msgIn + assert src[1] == alpha.port # ports equal + + msgOut = b"alpha sends to alpha" + alpha.send(msgOut, alpha.ha) + time.sleep(0.05) + msgIn, src = alpha.receive() + assert msgOut == msgIn + assert src[1] == alpha.port # ports equal + + msgOut = b"beta sends to alpha" + beta.send(msgOut, alpha.ha) + time.sleep(0.05) + msgIn, src = alpha.receive() + assert msgOut == msgIn + assert src[1] == beta.port # ports equal + + msgOut = b"beta sends to beta" + beta.send(msgOut, beta.ha) + time.sleep(0.05) + msgIn, src = beta.receive() + assert msgOut == msgIn + assert src[1] == beta.port # ports equal + + alpha.close() + beta.close() + + assert not alpha.opened + assert not beta.opened + + wl.flush() # just to test + assert wl.samed # rx and tx same buffer + + def addrBytes(ha): + return f"('{ha[0]}', {ha[1]})".encode("ascii") + + assert wl.readRx() == (b"\nTx ('127.0.0.1', 6102):\nalpha sends to beta\n\nRx ('127.0.0.1', 6101)" + b":\nalpha sends to beta\n\nTx ('127.0.0.1', 6101):\nalpha sends to alpha\n" + b"\nRx ('127.0.0.1', 6101):\nalpha sends to alpha\n\nTx ('127.0.0.1', 6101" + b"):\nbeta sends to alpha\n\nRx ('127.0.0.1', 6102):\nbeta sends to alpha\n" + b"\nTx ('127.0.0.1', 6102):\nbeta sends to beta\n\nRx ('127.0.0.1', 6102):" + b'\nbeta sends to beta\n') + assert wl.readTx() == (b"\nTx ('127.0.0.1', 6102):\nalpha sends to beta\n\nRx ('127.0.0.1', 6101)" + b":\nalpha sends to beta\n\nTx ('127.0.0.1', 6101):\nalpha sends to alpha\n" + b"\nRx ('127.0.0.1', 6101):\nalpha sends to alpha\n\nTx ('127.0.0.1', 6101" + b"):\nbeta sends to alpha\n\nRx ('127.0.0.1', 6102):\nbeta sends to alpha\n" + b"\nTx ('127.0.0.1', 6102):\nbeta sends to beta\n\nRx ('127.0.0.1', 6102):" + b'\nbeta sends to beta\n') + + + assert wl.readTx() == wl.readRx() + + assert not wl.opened + """Done Test""" + def test_open_peer(): """Test the udp openPeer context manager connection between two peers @@ -402,6 +502,7 @@ def test_peer_doer(): if __name__ == "__main__": test_udp_basic() + test_udp() test_open_peer() test_peer_doer() #test_udp_broadcast() diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoer.py index 3511ed8..156c2ef 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoer.py @@ -14,8 +14,6 @@ from hio.core.uxd import uxding, peermemoing - - def test_memoer_peer_basic(): """Test MemoerPeer class""" if platform.system() == "Windows": From e2f8a3fa005d3f5bf5bae0e29856c1d2ddfc47b2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 8 Jan 2026 14:45:23 -0700 Subject: [PATCH 06/27] now udp supports peermemo functionality --- src/hio/core/memo/memoing.py | 2 +- src/hio/core/udp/peermemoing.py | 139 ++++++++++++ src/hio/core/udp/udping.py | 16 +- src/hio/core/uxd/peermemoing.py | 10 +- tests/core/memo/test_memoing.py | 12 +- tests/core/udp/test_peer_memoing.py | 323 ++++++++++++++++++++++++++++ tests/core/udp/test_udp.py | 2 + tests/core/uxd/test_peer_memoer.py | 12 +- 8 files changed, 495 insertions(+), 21 deletions(-) create mode 100644 src/hio/core/udp/peermemoing.py create mode 100644 tests/core/udp/test_peer_memoing.py diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 7c1aee3..af0b479 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -341,7 +341,7 @@ def __init__(self, *, self.opened = False # mixed with subclass should provide this. if not hasattr(self, "bc"): # stub so mixin works in isolation. - self.bc = bc if bc is not None else 4 # mixed with subclass should provide this. + self.bc = bc if bc is not None else 1024 # mixed with subclass should provide this. @property diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py new file mode 100644 index 0000000..77c184a --- /dev/null +++ b/src/hio/core/udp/peermemoing.py @@ -0,0 +1,139 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.udp.peermemoing Module +""" +from contextlib import contextmanager + +from ... import help + +from ...base import doing +from ..udp import Peer +from ..memo import Memoer + +logger = help.ogler.getLogger() + + +class PeerMemoer(Peer, Memoer): + """Class for sending memograms over UXD transport + Mixin base classes Peer and Memoer to attain memogram over uxd transport. + + + Inherited Class Attributes: + MaxGramSize (int): absolute max gram size on tx with overhead + See memoing.Memoer Class + See Peer Class + + Inherited Attributes: + See memoing.Memoer Class + See Peer Class + + Class Attributes: + + Attributes: + """ + + def __init__(self, *, bc=1024, **kwa): + """Initialization method for instance. + + Inherited Parameters: + bc (int | None): count of transport buffers of MaxGramSize + + See memoing.Memoer for other inherited paramters + See Peer for other inherited paramters + + + Parameters: + + """ + super(PeerMemoer, self).__init__(bc=bc, **kwa) + + + +@contextmanager +def openPM(cls=None, name="test", temp=True, reopen=True, **kwa): + """ + Wrapper to create and open UXD PeerMemoer instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of PeerMemoer peer. + Enables management of transport by name. + Provides unique path part so can have many peers each at + different paths but in same directory. + temp (bool): True means open in temporary directory, clear on close + Otherwise open in persistent directory, do not clear on close + reopen (bool): True (re)open with this init (default) + False not (re)open with this init but later + + See udping.Peer for other keyword parameter passthroughs + + Usage: + with openPM() as peer: + peer.receive() + + with openPM(cls=PeerMemoerSubclass) as peer: + peer.receive() + + """ + peer = None + if cls is None: + cls = PeerMemoer + try: + peer = cls(name=name, temp=temp, reopen=reopen, **kwa) + + yield peer + + finally: + if peer: + peer.close() + + + +class PeerMemoerDoer(doing.Doer): + """PeerMemoerDoer Doer for reliable UXD transport. + Does not require retry tymers. + + See Doer for inherited attributes, properties, and methods. + To test in WingIde must configure Debug I/O to use external console + + Attributes: + .peer (PeerMemoerDoer): underlying transport instance subclass of Memoer + + """ + + def __init__(self, peer, **kwa): + """Initialize instance. + + Parameters: + peer (Peer): is Memoer Subclass instance + """ + super(PeerMemoerDoer, self).__init__(**kwa) + self.peer = peer + + + def enter(self, *, temp=None): + """Do 'enter' context actions. Override in subclass. Not a generator method. + Set up resources. Comparable to context manager enter. + + Parameters: + temp (bool | None): True means use temporary file resources if any + None means ignore parameter value. Use self.temp + + Inject temp or self.temp into file resources here if any + + Doist or DoDoer winds its doers on enter + """ + # inject temp into file resources here if any + self.peer.reopen(temp=temp) + + + def recur(self, tyme): + """""" + self.peer.service() + + + def exit(self): + """""" + self.peer.close() + diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 4f37ef7..b4f482b 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -54,6 +54,7 @@ class Peer(hioing.Mixin): Properties: host (str): element of .ha duple port (int): element of .ha duple + path (tuple): .ha (host, port) alias to match .uxd """ @@ -63,12 +64,13 @@ class Peer(hioing.Mixin): def __init__(self, *, name='main', ha=None, - host='', + host='127.0.0.1', port=55000, bc=None, bs=None, wl=None, bcast=False, + reopen=False, **kwa): """ Initialization method for instance. @@ -85,6 +87,8 @@ def __init__(self, *, wl (WireLog): instance ref for debug logging of over the wire tx and rx bcast (bool): True enables sending to broadcast addresses from local socket False otherwise + reopen (bool): True (re)open with this init + False not (re)open with this init but later (default) """ self.name = name @@ -106,7 +110,16 @@ def __init__(self, *, self.opened = False super(Peer, self).__init__(**kwa) + if reopen: + self.reopen() + @property + def path(self): + """ + Property that returns .ha duple + stub to match uxd interface + """ + return self.ha @property def host(self): @@ -115,7 +128,6 @@ def host(self): """ return self.ha[0] - @host.setter def host(self, value): """ diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index d3d7b0f..c6fb029 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -5,10 +5,10 @@ from contextlib import contextmanager from ... import help -from ... import hioing + from ...base import doing from ..uxd import Peer -from ..memo import Memoer, GramDex +from ..memo import Memoer logger = help.ogler.getLogger() @@ -32,7 +32,7 @@ class PeerMemoer(Peer, Memoer): Attributes: """ - def __init__(self, *, bc=4, **kwa): + def __init__(self, *, bc=1024, **kwa): """Initialization method for instance. Inherited Parameters: @@ -64,8 +64,8 @@ def openPM(cls=None, name="test", temp=True, reopen=True, clear=True, different paths but in same directory. temp (bool): True means open in temporary directory, clear on close Otherwise open in persistent directory, do not clear on close - reopen (bool): True (re)open with this init - False not (re)open with this init but later (default) + reopen (bool): True (re)open with this init (default) + False not (re)open with this init but later clear (bool): True means remove directory upon close when reopening False means do not remove directory upon close when reopening filed (bool): True means .path is file path not directory path diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index a74b1d7..6a92c8c 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -68,7 +68,7 @@ def test_memoer_basic(): peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -204,7 +204,7 @@ def test_memoer_small_gram_size(): peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -368,7 +368,7 @@ def test_memoer_multiple(): assert peer.size == 38 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt assert not peer.verific @@ -436,7 +436,7 @@ def test_memoer_basic_signed(): peer = memoing.Memoer(code=GramDex.Signed) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -586,7 +586,7 @@ def test_memoer_multiple_signed(): assert peer.size == 170 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt assert not peer.verific @@ -712,7 +712,7 @@ def test_memoer_verific(): peer = memoing.Memoer(verific=True) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc == 1024 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py new file mode 100644 index 0000000..f236d67 --- /dev/null +++ b/tests/core/udp/test_peer_memoing.py @@ -0,0 +1,323 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_peer_memoer module + +""" +import os +import platform +import time + +import pytest + +from hio.help import helping +from hio.base import doing, tyming +from hio.core.memo import GramDex +from hio.core.udp import udping, peermemoing + + +def test_memoer_peer_basic(): + """Test MemoerPeer class""" + + alphaPort = 6103 + betaPort = 6104 + + alpha = peermemoing.PeerMemoer(name="alpha", temp=True) + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + # (code, mid, vid, sig, neck, head) part sizes + assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.size == 1240 # default MaxGramSize for udp + assert alpha.bc == 1024 + assert not alpha.opened + assert alpha.ha == ('127.0.0.1', 55000) + assert alpha.path == alpha.ha + + + size = 38 # force gram size to be smaller than default so forces segmentation + alpha = peermemoing.PeerMemoer(name="alpha", temp=True, size=size, port=alphaPort) + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + # (code, mid, vid, sig, neck, head) part sizes + assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.size == size + assert alpha.bc == 1024 + assert not alpha.opened + assert alpha.reopen() + assert alpha.opened + assert alpha.ha == ('127.0.0.1', alphaPort) + assert alpha.path == alpha.ha + #assert alpha.actualBufSizes() == (1269760, 1269760) == (alpha.bc * alpha.MaxGramSize, alpha.bc * alpha.MaxGramSize) + + beta = peermemoing.PeerMemoer(name="beta", temp=True, size=size, port=betaPort) + assert beta.reopen() + assert beta.opened + assert beta.ha == ('127.0.0.1', betaPort) + assert beta.path == beta.ha + #assert beta.actualBufSizes() == (1269760, 1269760) == (beta.bc * beta.MaxGramSize, beta.bc * beta.MaxGramSize) + + # alpha sends + alpha.memoit("Hello there.", beta.ha) + alpha.memoit("How ya doing?", beta.path) + assert len(alpha.txms) == 2 + alpha.serviceTxMemos() + assert not alpha.txms + assert len(alpha.txgs) == 4 + for m, d in alpha.txgs: + assert not alpha.wiff(m) # base64 + assert d == beta.path + alpha.serviceTxGrams() + assert not alpha.txgs + assert alpha.txbs == (b'', None) + assert not alpha.rxgs + assert not alpha.rxms + assert not alpha.counts + assert not alpha.sources + + # beta receives + beta.serviceReceives() + time.sleep(0.05) + assert not beta.echos + assert len(beta.rxgs) == 2 + assert len(beta.counts) == 2 + assert len(beta.sources) == 2 + + mid = list(beta.rxgs.keys())[0] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'Hello ') + assert beta.rxgs[mid][1] == bytearray(b'there.') + + mid = list(beta.rxgs.keys())[1] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'How ya') + assert beta.rxgs[mid][1] == bytearray(b' doing?') + + beta.serviceRxGrams() + assert not beta.rxgs + assert not beta.counts + assert not beta.sources + assert len(beta.rxms) == 2 + assert beta.rxms[0] == ('Hello there.', alpha.path, None) + assert beta.rxms[1] == ('How ya doing?', alpha.path, None) + beta.serviceRxMemos() + assert not beta.rxms + + # beta sends + beta.memoit("Well is not this a fine day?", alpha.path) + beta.memoit("Do you want to do lunch?", alpha.path) + assert len(beta.txms) == 2 + beta.serviceTxMemos() + assert not beta.txms + beta.serviceTxGrams() + assert not beta.txgs + assert beta.txbs == (b'', None) + assert not beta.rxgs + assert not beta.rxms + assert not beta.counts + assert not beta.sources + + # alpha receives + alpha.serviceReceives() + time.sleep(0.05) + assert not alpha.echos + assert len(alpha.rxgs) == 2 + assert len(alpha.counts) == 2 + assert len(alpha.sources) == 2 + + mid = list(alpha.rxgs.keys())[0] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 4 + assert len(alpha.rxgs[mid]) == 4 + + mid = list(alpha.rxgs.keys())[1] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 3 + assert len(alpha.rxgs[mid]) == 3 + + alpha.serviceRxGrams() + assert not alpha.rxgs + assert not alpha.counts + assert not alpha.sources + assert len(alpha.rxms) == 2 + assert alpha.rxms[0] == ('Well is not this a fine day?', beta.path, None) + assert alpha.rxms[1] == ('Do you want to do lunch?', beta.path, None) + alpha.serviceRxMemos() + assert not alpha.rxms + + assert beta.close() + assert not beta.opened + + assert alpha.close() + assert not alpha.opened + + """Done Test""" + + +def test_memoer_peer_open(): + """Test MemoerPeer class with context manager openPM""" + + host = '127.0.0.1' # default + alphaPort = 6103 + betaPort = 6104 + size = 38 + + with (peermemoing.openPM(name='alpha', size=size, port=alphaPort) as alpha, + peermemoing.openPM(name='beta', size=size, port=betaPort) as beta): + + + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + # (code, mid, vid, sig, neck, head) part sizes + assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.size == size + assert alpha.bc == 1024 + + assert alpha.opened + assert alpha.ha == alpha.path == (host, alphaPort) + + assert beta.bc == 1024 + assert beta.opened + assert beta.ha == beta.path == (host, betaPort) + + # alpha sends + alpha.memoit("Hello there.", beta.path) + alpha.memoit("How ya doing?", beta.path) + assert len(alpha.txms) == 2 + alpha.serviceTxMemos() + assert not alpha.txms + assert len(alpha.txgs) == 4 + for m, d in alpha.txgs: + assert not alpha.wiff(m) # base64 + assert d == beta.path + alpha.serviceTxGrams() + assert not alpha.txgs + assert alpha.txbs == (b'', None) + assert not alpha.rxgs + assert not alpha.rxms + assert not alpha.counts + assert not alpha.sources + + # beta receives + beta.serviceReceives() + time.sleep(0.05) + time.sleep(0.05) + assert not beta.echos + assert len(beta.rxgs) == 2 + assert len(beta.counts) == 2 + assert len(beta.sources) == 2 + + mid = list(beta.rxgs.keys())[0] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'Hello ') + assert beta.rxgs[mid][1] == bytearray(b'there.') + + mid = list(beta.rxgs.keys())[1] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'How ya') + assert beta.rxgs[mid][1] == bytearray(b' doing?') + + beta.serviceRxGrams() + assert not beta.rxgs + assert not beta.counts + assert not beta.sources + assert len(beta.rxms) == 2 + assert beta.rxms[0] == ('Hello there.', alpha.path, None) + assert beta.rxms[1] == ('How ya doing?', alpha.path, None) + beta.serviceRxMemos() + assert not beta.rxms + + # beta sends + beta.memoit("Well is not this a fine day?", alpha.path) + beta.memoit("Do you want to do lunch?", alpha.path) + assert len(beta.txms) == 2 + beta.serviceTxMemos() + assert not beta.txms + beta.serviceTxGrams() + assert not beta.txgs + assert beta.txbs == (b'', None) + assert not beta.rxgs + assert not beta.rxms + assert not beta.counts + assert not beta.sources + + # alpha receives + alpha.serviceReceives() + time.sleep(0.05) + assert not alpha.echos + assert len(alpha.rxgs) == 2 + assert len(alpha.counts) == 2 + assert len(alpha.sources) == 2 + + mid = list(alpha.rxgs.keys())[0] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 4 + assert len(alpha.rxgs[mid]) == 4 + + mid = list(alpha.rxgs.keys())[1] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 3 + assert len(alpha.rxgs[mid]) == 3 + + alpha.serviceRxGrams() + assert not alpha.rxgs + assert not alpha.counts + assert not alpha.sources + assert len(alpha.rxms) == 2 + assert alpha.rxms[0] == ('Well is not this a fine day?', beta.path, None) + assert alpha.rxms[1] == ('Do you want to do lunch?', beta.path, None) + alpha.serviceRxMemos() + assert not alpha.rxms + + assert not beta.opened + assert not alpha.opened + + """Done Test""" + + + +def test_peermemoer_doer(): + """Test PeerMemoerDoer class + """ + + tock = 0.03125 + ticks = 4 + limit = ticks * tock + doist = doing.Doist(tock=tock, real=True, limit=limit) + assert doist.tyme == 0.0 # on next cycle + assert doist.tock == tock == 0.03125 + assert doist.real == True + assert doist.limit == limit == 0.125 + assert doist.doers == [] + + peer = peermemoing.PeerMemoer(name="test", temp=True, reopen=False) + assert peer.opened == False + assert peer.ha == peer.path == ('127.0.0.1', 55000) + + doer = peermemoing.PeerMemoerDoer(peer=peer) + assert doer.peer == peer + assert not doer.peer.opened + assert doer.tock == 0.0 # ASAP + + doers = [doer] + doist.do(doers=doers) + assert doist.tyme == limit + assert peer.opened == False + """Done Test""" + + +if __name__ == "__main__": + test_memoer_peer_basic() + test_memoer_peer_open() + test_peermemoer_doer() + + diff --git a/tests/core/udp/test_udp.py b/tests/core/udp/test_udp.py index c8a8ad8..04a4f01 100644 --- a/tests/core/udp/test_udp.py +++ b/tests/core/udp/test_udp.py @@ -38,6 +38,7 @@ def test_udp_basic(): assert alpha.reopen() assert alpha.opened assert alpha.ha == alphaHa + assert alpha.path == alpha.ha beta = udping.Peer(name='beta',host=host, port = betaPort, wl=wl) # host on port 6102 assert not beta.opened @@ -45,6 +46,7 @@ def test_udp_basic(): assert beta.reopen() assert beta.opened assert beta.ha == betaHa + assert beta.path == beta.ha msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoer.py index 156c2ef..f1978a5 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoer.py @@ -26,7 +26,7 @@ def test_memoer_peer_basic(): assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert alpha.size == 38 - assert alpha.bc == 4 + assert alpha.bc == 1024 assert not alpha.opened assert alpha.reopen() assert alpha.opened @@ -145,22 +145,22 @@ def test_memoer_peer_open(): """Test MemoerPeer class with context manager openPM""" if platform.system() == "Windows": return - with (peermemoing.openPM(name='alpha', size=38) as alpha, - peermemoing.openPM(name='beta', size=38) as beta): + with (peermemoing.openPM(name='alpha', size=38) as alpha, + peermemoing.openPM(name='beta', size=38) as beta): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt # (code, mid, vid, sig, neck, head) part sizes assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert alpha.size == 38 - assert alpha.bc == 4 + assert alpha.bc == 1024 assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert beta.bc == 4 + assert beta.bc == 1024 assert beta.opened assert beta.path.endswith("beta.uxd") @@ -255,11 +255,9 @@ def test_memoer_peer_open(): assert not alpha.rxms - assert beta.close() assert not beta.opened assert not os.path.exists(beta.path) - assert alpha.close() assert not alpha.opened assert not os.path.exists(alpha.path) From 7d0e4ff82682ebc610cbc5ea04b1d9c4a58d81f5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 8 Jan 2026 21:48:43 -0700 Subject: [PATCH 07/27] made peermemoer udp a subclass of TymeeMemoer Added tymers dict for all the tx retry tymers for each in flight tx --- src/hio/core/memo/memoing.py | 16 +++-- src/hio/core/udp/peermemoing.py | 115 +++++++++++++++++++++++++++++--- tests/core/memo/test_memoing.py | 5 ++ 3 files changed, 121 insertions(+), 15 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index af0b479..748f47f 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -23,6 +23,7 @@ from ... import hioing, help from ...base import tyming, doing +from ...base.tyming import Tymer, Tymee from ...help import helping logger = help.ogler.getLogger() @@ -158,7 +159,7 @@ class Memoer(hioing.Mixin): MaxGramCount (int): absolute max gram count - Inherited Attributes: + Stubbed Attributes: name (str): unique name for Memoer transport. Used to manage multiple instances. opened (bool): True means transport open for use @@ -1289,7 +1290,7 @@ def exit(self): -class TymeeMemoer(tyming.Tymee, Memoer): +class TymeeMemoer(Tymee, Memoer): """TymeeMemoer mixin base class to add tymer support for unreliable transports that need retry tymers. Subclass of tyming.Tymee @@ -1323,7 +1324,8 @@ def __init__(self, *, tymeout=None, **kwa): """ super(TymeeMemoer, self).__init__(**kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout - #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer + self.tymers = {} + #Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer def wind(self, tymth): @@ -1332,7 +1334,8 @@ def wind(self, tymth): Updates winds .tymer .tymth """ super(TymeeMemoer, self).wind(tymth) - #self.tymer.wind(tymth) + for tid, tymer in self.tymers.items(): + tymer.wind(tymth) def serviceTymers(self): """Service all retry tymers @@ -1428,6 +1431,7 @@ def wind(self, tymth): self.peer.wind(tymth) + def enter(self, *, temp=None): """Do 'enter' context actions. Override in subclass. Not a generator method. Set up resources. Comparable to context manager enter. @@ -1440,10 +1444,10 @@ def enter(self, *, temp=None): Doist or DoDoer winds its doers on enter """ - # inject temp into file resources here if any if self.tymth: self.peer.wind(self.tymth) - self.peer.reopen(temp=temp) + + self.peer.reopen(temp=temp) # inject temp into file resources here if any def recur(self, tyme): diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index 77c184a..c1b0582 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -7,29 +7,115 @@ from ... import help from ...base import doing +from ...base.tyming import Tymer from ..udp import Peer -from ..memo import Memoer +from ..memo import Memoer, TymeeMemoer logger = help.ogler.getLogger() -class PeerMemoer(Peer, Memoer): +class PeerMemoer(Peer, TymeeMemoer): """Class for sending memograms over UXD transport Mixin base classes Peer and Memoer to attain memogram over uxd transport. Inherited Class Attributes: - MaxGramSize (int): absolute max gram size on tx with overhead - See memoing.Memoer Class - See Peer Class + (Peer) + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max gram bytes for this transport + + (Memoer) + Version (Versionage): default version consisting of namedtuple of form + (major: int, minor: int) + Codex (GramDex): dataclass ref to gram codex + Codes (dict): maps codex names to codex values + Names (dict): maps codex values to codex names + Sodex (SGDex): dataclass ref to signed gram codex + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes + MaxMemoSize (int): absolute max memo size + MaxGramCount (int): absolute max gram count Inherited Attributes: - See memoing.Memoer Class - See Peer Class + (Peer) + name (str): unique identifier of peer for managment purposes + ha (tuple): host address of form (host,port) of type (str, int) of this + peer's socket address. + + bc (int | None): count of transport buffers of MaxGramSize + bs (int): buffer size + wl (WireLog): instance ref for debug logging of over the wire tx and rx + + ls (socket.socket): local socket of this Peer + opened (bool): True local socket is created and opened. False otherwise + + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + + (Memoer) + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. + sources (dict): keyed by mid (memoID) that holds the src for the memo. + This enables reattaching src to fused memo in rxms deque tuple. + counts (dict): keyed by mid (memoID) that holds the gram count from + the first gram for the memo. This enables lookup of the gram count when + fusing its grams. + vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + the memo indexed by its mid (memoID). This enables reattaching + the vid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise vid is None + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is tuple of form: + (memo: str, src: str, vid: str) where: + memo is fused memo, src is source addr, vid is verification ID + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is tuple of form + (memo: str, dst: str, vid: str | None) + memo is memo to be partitioned into gram + dst is dst addr for grams + vid is verification id when gram is to be signed or None otherwise + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when Encodesdatagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) + echos (deque): holding echo receive duples for testing. Each duple of + form: (gram: bytes, dst: str). + + + Inherited Properties: + (Peer) + host (str): element of .ha duple + port (int): element of .ha duple + path (tuple): .ha (host, port) alias to match .uxd + + (Tymee) + tyme is float relative cycle time of associated Tymist .tyme obtained + via injected .tymth function wrapper closure. + tymth is function wrapper closure returned by Tymist .tymeth() method. + When .tymth is called it returns associated Tymist .tyme. + .tymth provides injected dependency on Tymist tyme base. + + (Memoer) + code (bytes | None): gram code for gram header when rending for tx + curt (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = over head size + neck size + body size. + other gram size = over head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. + verific (bool): True means any rx grams must be signed. + False otherwise + - Class Attributes: - Attributes: """ def __init__(self, *, bc=1024, **kwa): @@ -112,6 +198,14 @@ def __init__(self, peer, **kwa): self.peer = peer + def wind(self, tymth): + """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + Updates winds .tymer .tymth + """ + super(PeerMemoerDoer, self).wind(tymth) + self.peer.wind(tymth) + + def enter(self, *, temp=None): """Do 'enter' context actions. Override in subclass. Not a generator method. Set up resources. Comparable to context manager enter. @@ -124,6 +218,8 @@ def enter(self, *, temp=None): Doist or DoDoer winds its doers on enter """ + if self.tymth: # Doist or DoDoer already winds its doers on enter + self.peer.wind(self.tymth) # inject temp into file resources here if any self.peer.reopen(temp=temp) @@ -137,3 +233,4 @@ def exit(self): """""" self.peer.close() + diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 6a92c8c..81bffd7 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -805,6 +805,8 @@ def test_memoer_doer(): """End Test """ + + def test_tymee_memoer_basic(): """Test TymeeMemoer class basic """ @@ -1009,6 +1011,9 @@ def test_tymee_memoer_doer(): test_memoer_verific() test_open_memoer() test_memoer_doer() + test_tymer_memoer_basic() + test_open_tmr() + test_tymer_memoer_doer() test_tymee_memoer_basic() test_open_tm() test_tymee_memoer_doer() From 68fc5f221af18bb7b46d73068f492fca2860652b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 8 Jan 2026 22:04:46 -0700 Subject: [PATCH 08/27] some cleanup of tyme handling in tymememoer --- src/hio/core/memo/memoing.py | 10 ++++++---- tests/core/memo/test_memoing.py | 13 ++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 748f47f..dfba257 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -1305,6 +1305,8 @@ class TymeeMemoer(Tymee, Memoer): see superclass Attributes: + tymers (dict): keys are tid and values are Tymers for retry tymers for + each inflight tx tymeout (float): default timeout for retry tymer(s) if any @@ -1333,7 +1335,7 @@ def wind(self, tymth): Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoer, self).wind(tymth) + super(TymeeMemoer, self).wind(tymth) # wind Tymee superclass for tid, tymer in self.tymers.items(): tymer.wind(tymth) @@ -1427,8 +1429,8 @@ def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoerDoer, self).wind(tymth) - self.peer.wind(tymth) + super(TymeeMemoerDoer, self).wind(tymth) # wind this doer + self.peer.wind(tymth) # wind its peer @@ -1445,7 +1447,7 @@ def enter(self, *, temp=None): Doist or DoDoer winds its doers on enter """ if self.tymth: - self.peer.wind(self.tymth) + self.peer.wind(self.tymth) # Doist or DoDoer winds its doers on enter self.peer.reopen(temp=temp) # inject temp into file resources here if any diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 81bffd7..6094cbb 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -820,6 +820,7 @@ def test_tymee_memoer_basic(): assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize assert not peer.verific + assert peer.tymers == {} peer.reopen() assert peer.opened == True @@ -957,6 +958,7 @@ def test_open_tm(): assert zeta.opened assert zeta.name == 'zeta' assert zeta.tymeout == 0.0 + assert zeta.tymers == {} assert not zeta.opened @@ -991,11 +993,11 @@ def test_tymee_memoer_doer(): tymist = tyming.Tymist(tock=1.0) tmgdoer.wind(tymth=tymist.tymen()) - assert tmgdoer.tyme == tymist.tyme == 0.0 - assert peer.tyme == tymist.tyme == 0.0 + assert tmgdoer.tymth == tmgdoer.peer.tymth + assert tmgdoer.tyme == peer.tyme == tymist.tyme == 0.0 + tymist.tick() - assert tmgdoer.tyme == tymist.tyme == 1.0 - assert peer.tyme == tymist.tyme == 1.0 + assert tmgdoer.tyme == peer.tyme == tymist.tyme == 1.0 """End Test """ @@ -1011,9 +1013,6 @@ def test_tymee_memoer_doer(): test_memoer_verific() test_open_memoer() test_memoer_doer() - test_tymer_memoer_basic() - test_open_tmr() - test_tymer_memoer_doer() test_tymee_memoer_basic() test_open_tm() test_tymee_memoer_doer() From 489b925ee02f6480bd333961d043571246a5022d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Jan 2026 10:10:33 -0700 Subject: [PATCH 09/27] Some refactor to set up for safe peer memoing --- src/hio/core/memo/memoing.py | 34 ++++++++-- src/hio/core/udp/peermemoing.py | 66 +++++++++++++++---- src/hio/core/uxd/peermemoing.py | 10 ++- src/hio/core/uxd/uxding.py | 15 ++--- tests/core/memo/test_memoing.py | 18 +++-- tests/core/udp/test_peer_memoing.py | 3 +- .../core/udp/{test_udp.py => test_udping.py} | 0 ...st_peer_memoer.py => test_peer_memoing.py} | 6 +- .../core/uxd/{test_uxd.py => test_uxding.py} | 0 9 files changed, 105 insertions(+), 47 deletions(-) rename tests/core/udp/{test_udp.py => test_udping.py} (100%) rename tests/core/uxd/{test_peer_memoer.py => test_peer_memoing.py} (99%) rename tests/core/uxd/{test_uxd.py => test_uxding.py} (100%) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index dfba257..4a33607 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -157,6 +157,7 @@ class Memoer(hioing.Mixin): Sizes (dict): gram head part sizes Sizage instances keyed by gram codes MaxMemoSize (int): absolute max memo size MaxGramCount (int): absolute max gram count + BufSize (int): used to set default buffer size for transport datagram buffers Stubbed Attributes: @@ -164,7 +165,15 @@ class Memoer(hioing.Mixin): Used to manage multiple instances. opened (bool): True means transport open for use False otherwise - bc (int | None): count of transport buffers of MaxGramSize + bc (int|None): count of transport buffers of MaxGramSize + bs (int|None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize + + Stubbed Methods: + send(gram, dst, *, echoic=False) -> int # send gram over transport to dst + receive(self, *, echoic=False) -> (bytes, str|tuple|None) # receive gram Attributes: version (Versionage): version for this memoir instance consisting of @@ -236,12 +245,13 @@ class Memoer(hioing.Mixin): MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size MaxGramCount = 16777215 # (2**24-1) absolute max gram count MaxGramSize = 65535 # (2**16-1) absolute max gram size overridden in subclass - + BufSize = 65535 # 2 ** 16 - 1 default buffersize def __init__(self, *, name=None, bc=None, + bs=None, version=None, rxgs=None, sources=None, @@ -264,6 +274,10 @@ def __init__(self, *, opened (bool): True means opened for transport False otherwise bc (int | None): count of transport buffers of MaxGramSize + bs (int | None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize Parameters: version (Versionage): version for this memoir instance consisting of @@ -333,7 +347,7 @@ def __init__(self, *, self.size = size # property sets size given .code and constraints self._verific = True if verific else False - super(Memoer, self).__init__(name=name, bc=bc, **kwa) + super(Memoer, self).__init__(name=name, bc=bc, bs=bs, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. self.name = name if name is not None else "main" # mixed with subclass should provide this. @@ -342,7 +356,13 @@ def __init__(self, *, self.opened = False # mixed with subclass should provide this. if not hasattr(self, "bc"): # stub so mixin works in isolation. - self.bc = bc if bc is not None else 1024 # mixed with subclass should provide this. + self.bc = bc # mixin subclass should provide this. + + if not hasattr(self, "bs"): # stub so mixin works in isolation. + self.bs = bs # mixin subclass should provide this. + + if self.bs is None: # default if not provided by mixin + self.bs = self.BufSize @property @@ -643,7 +663,7 @@ def pick(self, gram): return (mid, vid if vid else None, gn, gc) - def receive(self, *, echoic=False): + def receive(self, *, echoic=False) -> (bytes, str|tuple|None): """Attemps to receive bytes from remote source. Must be overridden in subclass. This is a stub to define mixin interface. @@ -655,7 +675,7 @@ def receive(self, *, echoic=False): indicates nothing to receive of form (b'', None) Returns: - duple (tuple): of form (data: bytes, src: str|None) where data is the + duple (tuple): of form (data: bytes, src: str|tuple|None) where data is the bytes of received data and src is the source address. When no data the duple is (b'', None) unless echoic is True then pop off echo from .echos @@ -993,7 +1013,7 @@ def rend(self, memo, vid=None): return grams - def send(self, gram, dst, *, echoic=False): + def send(self, gram, dst, *, echoic=False) -> int: """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index c1b0582..a2c4137 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -9,12 +9,12 @@ from ...base import doing from ...base.tyming import Tymer from ..udp import Peer -from ..memo import Memoer, TymeeMemoer +from ..memo import Memoer logger = help.ogler.getLogger() -class PeerMemoer(Peer, TymeeMemoer): +class PeerMemoer(Peer, Memoer): """Class for sending memograms over UXD transport Mixin base classes Peer and Memoer to attain memogram over uxd transport. @@ -94,13 +94,6 @@ class PeerMemoer(Peer, TymeeMemoer): port (int): element of .ha duple path (tuple): .ha (host, port) alias to match .uxd - (Tymee) - tyme is float relative cycle time of associated Tymist .tyme obtained - via injected .tymth function wrapper closure. - tymth is function wrapper closure returned by Tymist .tymeth() method. - When .tymth is called it returns associated Tymist .tyme. - .tymth provides injected dependency on Tymist tyme base. - (Memoer) code (bytes | None): gram code for gram header when rending for tx curt (bool): True means when rending for tx encode header in base2 @@ -175,9 +168,8 @@ def openPM(cls=None, name="test", temp=True, reopen=True, **kwa): peer.close() - class PeerMemoerDoer(doing.Doer): - """PeerMemoerDoer Doer for reliable UXD transport. + """PeerMemoerDoer Doer for unreliable UDP transport. Does not require retry tymers. See Doer for inherited attributes, properties, and methods. @@ -198,11 +190,59 @@ def __init__(self, peer, **kwa): self.peer = peer + def enter(self, *, temp=None): + """Do 'enter' context actions. Override in subclass. Not a generator method. + Set up resources. Comparable to context manager enter. + + Parameters: + temp (bool | None): True means use temporary file resources if any + None means ignore parameter value. Use self.temp + + Inject temp or self.temp into file resources here if any + + Doist or DoDoer winds its doers on enter + """ + # inject temp into file resources here if any + self.peer.reopen(temp=temp) + + + def recur(self, tyme): + """""" + self.peer.service() + + + def exit(self): + """""" + self.peer.close() + + +class SafePeerMemoerDoer(doing.Doer): + """PeerMemoerDoer Doer for unreliable UDP transport. + Does not require retry tymers. + + See Doer for inherited attributes, properties, and methods. + To test in WingIde must configure Debug I/O to use external console + + Attributes: + .peer (PeerMemoerDoer): underlying transport instance subclass of Memoer + + """ + + def __init__(self, peer, **kwa): + """Initialize instance. + + Parameters: + peer (Peer): is Memoer Subclass instance + """ + super(SafePeerMemoerDoer, self).__init__(**kwa) + self.peer = peer + + def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(PeerMemoerDoer, self).wind(tymth) + super(SafePeerMemoerDoer, self).wind(tymth) self.peer.wind(tymth) @@ -232,5 +272,3 @@ def recur(self, tyme): def exit(self): """""" self.peer.close() - - diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index c6fb029..2ea55e7 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -19,20 +19,18 @@ class PeerMemoer(Peer, Memoer): Inherited Class Attributes: - MaxGramSize (int): absolute max gram size on tx with overhead - See memoing.Memoer Class See Peer Class + See memoing.Memoer Class + Inherited Attributes: - See memoing.Memoer Class See Peer Class + See Memoer Class - Class Attributes: - Attributes: """ - def __init__(self, *, bc=1024, **kwa): + def __init__(self, *, bc=4, **kwa): """Initialization method for instance. Inherited Parameters: diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index ac8df36..72f832f 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -44,6 +44,12 @@ class Peer(filing.Filer): Mode (str): open mode such as "r+" Fext (str): default file extension such as "text" for "fname.text" + Class Attributes: + Umask (int): octal default umask permissions such as 0o022 + MaxUxdPathSize (int:) max characters in uxd file path + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max bytes in in datagram for this transport + Inherited Attributes: name (str): unique path component used in directory or file path name base (str): another unique path component inserted before name @@ -59,14 +65,6 @@ class Peer(filing.Filer): opened (bool): True means directory path, uxd file, and socket are created and opened. False otherwise - Class Attributes: - Umask (int): octal default umask permissions such as 0o022 - MaxUxdPathSize (int:) max characters in uxd file path - BufSize (int): used to set default buffer size for transport datagram buffers - MaxGramSize (int): max bytes in in datagram for this transport - - - Attributes: umask (int): unpermission mask for uxd file, usually octal 0o022 .umask is applied after .perm is set if any @@ -96,7 +94,6 @@ class Peer(filing.Filer): MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass - def __init__(self, *, umask=None, bc=None, diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 6094cbb..00c883d 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -68,7 +68,8 @@ def test_memoer_basic(): peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -204,7 +205,8 @@ def test_memoer_small_gram_size(): peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -368,7 +370,8 @@ def test_memoer_multiple(): assert peer.size == 38 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt assert not peer.verific @@ -436,7 +439,8 @@ def test_memoer_basic_signed(): peer = memoing.Memoer(code=GramDex.Signed) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -586,7 +590,8 @@ def test_memoer_multiple_signed(): assert peer.size == 170 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt assert not peer.verific @@ -712,7 +717,8 @@ def test_memoer_verific(): peer = memoing.Memoer(verific=True) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 1024 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index f236d67..246f683 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -205,8 +205,7 @@ def test_memoer_peer_open(): # beta receives beta.serviceReceives() - time.sleep(0.05) - time.sleep(0.05) + time.sleep(0.1) assert not beta.echos assert len(beta.rxgs) == 2 assert len(beta.counts) == 2 diff --git a/tests/core/udp/test_udp.py b/tests/core/udp/test_udping.py similarity index 100% rename from tests/core/udp/test_udp.py rename to tests/core/udp/test_udping.py diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoing.py similarity index 99% rename from tests/core/uxd/test_peer_memoer.py rename to tests/core/uxd/test_peer_memoing.py index f1978a5..13ce8af 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoing.py @@ -26,7 +26,7 @@ def test_memoer_peer_basic(): assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert alpha.size == 38 - assert alpha.bc == 1024 + assert alpha.bc == 4 assert not alpha.opened assert alpha.reopen() assert alpha.opened @@ -155,12 +155,12 @@ def test_memoer_peer_open(): # (code, mid, vid, sig, neck, head) part sizes assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert alpha.size == 38 - assert alpha.bc == 1024 + assert alpha.bc == 4 assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert beta.bc == 1024 + assert beta.bc == 4 assert beta.opened assert beta.path.endswith("beta.uxd") diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxding.py similarity index 100% rename from tests/core/uxd/test_uxd.py rename to tests/core/uxd/test_uxding.py From 0d6db6c2a1408d654bdcd41cee2f81d0028d7f6c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Jan 2026 10:21:37 -0700 Subject: [PATCH 10/27] refactor to SureMemoer class for reliable delivery services over unreliable datagram transports --- src/hio/core/memo/__init__.py | 2 +- src/hio/core/memo/memoing.py | 38 +++++++++++++++++---------------- tests/core/memo/test_memoing.py | 22 +++++++++---------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/hio/core/memo/__init__.py b/src/hio/core/memo/__init__.py index 6924ae2..a27fc1d 100644 --- a/src/hio/core/memo/__init__.py +++ b/src/hio/core/memo/__init__.py @@ -6,5 +6,5 @@ from .memoing import (Versionage, Sizage, GramDex, SGDex, openMemoer, Memoer, MemoerDoer, - openTM, TymeeMemoer, TymeeMemoerDoer) + openSM, SureMemoer, SureMemoerDoer) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 4a33607..9c82a68 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -1310,9 +1310,11 @@ def exit(self): -class TymeeMemoer(Tymee, Memoer): - """TymeeMemoer mixin base class to add tymer support for unreliable transports - that need retry tymers. Subclass of tyming.Tymee +class SureMemoer(Tymee, Memoer): + """SureMemoer mixin base class that supports reliable (sure) memo delivery + over unreliable datagram transports. These need retry tymers and acknowledged + delivery services. + Subclass of Tymee and Memoer Inherited Class Attributes: @@ -1325,9 +1327,9 @@ class TymeeMemoer(Tymee, Memoer): see superclass Attributes: + tymeout (float): default timeout for retry tymer(s) if any tymers (dict): keys are tid and values are Tymers for retry tymers for each inflight tx - tymeout (float): default timeout for retry tymer(s) if any @@ -1344,7 +1346,7 @@ def __init__(self, *, tymeout=None, **kwa): tymeout (float): default for retry tymer if any """ - super(TymeeMemoer, self).__init__(**kwa) + super(SureMemoer, self).__init__(**kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout self.tymers = {} #Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer @@ -1355,7 +1357,7 @@ def wind(self, tymth): Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoer, self).wind(tymth) # wind Tymee superclass + super(SureMemoer, self).wind(tymth) # wind Tymee superclass for tid, tymer in self.tymers.items(): tymer.wind(tymth) @@ -1392,9 +1394,8 @@ def serviceAll(self): @contextmanager -def openTM(cls=None, name="test", **kwa): - """ - Wrapper to create and open TymeeMemoer instances +def openSM(cls=None, name="test", **kwa): + """Wrapper to create and open SureMemoer instances When used in with statement block, calls .close() on exit of with block Parameters: @@ -1402,17 +1403,17 @@ def openTM(cls=None, name="test", **kwa): name (str): unique identifer of Memoer peer. Enables management of transport by name. Usage: - with openTM() as peer: + with openSM() as peer: peer.receive() - with openTM(cls=MemoerSub) as peer: + with openSM(cls=SureMemoerSub) as peer: peer.receive() """ peer = None if cls is None: - cls = TymeeMemoer + cls = SureMemoer try: peer = cls(name=name, **kwa) peer.reopen() @@ -1424,13 +1425,14 @@ def openTM(cls=None, name="test", **kwa): peer.close() -class TymeeMemoerDoer(doing.Doer): - """TymeeMemoer Doer for unreliable transports that require retry tymers. +class SureMemoerDoer(doing.Doer): + """SureMemoerDoer Doer to provide reliable delivery over unreliable + datagram transports. This requires retry tymers and acknowldged services. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (TymeeMemoer) is underlying transport instance subclass of TymeeMemoer + .peer (SureMemoer) is underlying transport instance subclass of SureMemoer """ @@ -1438,9 +1440,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (TymeeMemoer): subclass instance + peer (SureMemoer): subclass instance """ - super(TymeeMemoerDoer, self).__init__(**kwa) + super(SureMemoerDoer, self).__init__(**kwa) self.peer = peer @@ -1449,7 +1451,7 @@ def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoerDoer, self).wind(tymth) # wind this doer + super(SureMemoerDoer, self).wind(tymth) # wind this doer self.peer.wind(tymth) # wind its peer diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 00c883d..619f62a 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -813,10 +813,10 @@ def test_memoer_doer(): -def test_tymee_memoer_basic(): +def test_sure_memoer_basic(): """Test TymeeMemoer class basic """ - peer = memoing.TymeeMemoer() + peer = memoing.SureMemoer() assert peer.tymeout == 0.0 assert peer.name == "main" assert peer.opened == False @@ -956,10 +956,10 @@ def test_tymee_memoer_basic(): """ End Test """ -def test_open_tm(): +def test_open_sm(): """Test contextmanager decorator openTM for openTymeeMemoer """ - with (memoing.openTM(name='zeta') as zeta): + with (memoing.openSM(name='zeta') as zeta): assert zeta.opened assert zeta.name == 'zeta' @@ -972,8 +972,8 @@ def test_open_tm(): """ End Test """ -def test_tymee_memoer_doer(): - """Test TymeeMemoerDoer class +def test_sure_memoer_doer(): + """Test SureMemoerDoer class """ tock = 0.03125 ticks = 4 @@ -985,9 +985,9 @@ def test_tymee_memoer_doer(): assert doist.limit == limit == 0.125 assert doist.doers == [] - peer = memoing.TymeeMemoer() + peer = memoing.SureMemoer() - tmgdoer = memoing.TymeeMemoerDoer(peer=peer) + tmgdoer = memoing.SureMemoerDoer(peer=peer) assert tmgdoer.peer == peer assert not tmgdoer.peer.opened assert tmgdoer.tock == 0.0 # ASAP @@ -1019,7 +1019,7 @@ def test_tymee_memoer_doer(): test_memoer_verific() test_open_memoer() test_memoer_doer() - test_tymee_memoer_basic() - test_open_tm() - test_tymee_memoer_doer() + test_sure_memoer_basic() + test_open_sm() + test_sure_memoer_doer() From 7f0918b516cea1f40b86c4e0fda8b77e9c50e1ed Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Jan 2026 12:04:34 -0700 Subject: [PATCH 11/27] added .echoic property to Memoer to facilitate easier setup in testing for the added complexity of Reliable services with subclass SureMemoer --- src/hio/base/tyming.py | 26 ++--- src/hio/core/memo/memoing.py | 173 +++++++++++++++++++++++++++++--- src/hio/core/udp/peermemoing.py | 10 +- src/hio/core/uxd/peermemoing.py | 4 +- tests/core/memo/test_memoing.py | 31 ++++-- 5 files changed, 203 insertions(+), 41 deletions(-) diff --git a/src/hio/base/tyming.py b/src/hio/base/tyming.py index 990f30e..c3e92be 100644 --- a/src/hio/base/tyming.py +++ b/src/hio/base/tyming.py @@ -104,14 +104,16 @@ class Tymee(hioing.Mixin): Tymee has .tyme property that returns the artificial or simulated or cycle time from its referenced Tymist instance ._tymist. + Class Attributes: + Attributes: Properties: - .tyme (float | None): relative cycle time of associated Tymist which is + tyme (float | None): relative cycle time of associated Tymist which is provided by calling .tymth function wrapper closure which is obtained from Tymist.tymen(). None means not assigned yet. - .tymth (Callable | None): function wrapper closure returned by + tymth (Callable | None): function wrapper closure returned by Tymist.tymen() method. When .tymth is called it returns associated Tymist.tyme. Provides injected dependency on Tymist cycle tyme base. None means not assigned yet. @@ -120,7 +122,7 @@ class Tymee(hioing.Mixin): .wind injects ._tymth dependency from associated Tymist to get its .tyme Hidden: - ._tymth is injected function wrapper closure returned by .tymen() of + _tymth is injected function wrapper closure returned by .tymen() of associated Tymist instance that returns Tymist .tyme. when called. """ @@ -184,17 +186,17 @@ class Tymer(Tymee): Attributes: Inherited Properties: - .tyme is float relative cycle time of associated Tymist .tyme obtained + tyme is float relative cycle time of associated Tymist .tyme obtained via injected .tymth function wrapper closure. - .tymth is function wrapper closure returned by Tymist .tymeth() method. + tymth is function wrapper closure returned by Tymist .tymeth() method. When .tymth is called it returns associated Tymist .tyme. .tymth provides injected dependency on Tymist tyme base. Properties: - .duration = tyme duration of tymer in seconds from ._start to ._stop - .elaspsed = tyme elasped in seconds since ._start - .remaining = tyme remaining in seconds until ._stop - .expired = True if expired, False otherwise, i.e. .tyme >= ._stop + duration (float): tyme duration in seconds from ._start to ._stop + elaspsed (float): tyme elasped in seconds since ._start + remaining (float): tyme remaining in seconds until ._stop + expired (bool): True if expired, False otherwise, i.e. .tyme >= ._stop Inherited Methods: .wind is injects ._tymth dependency @@ -204,10 +206,10 @@ class Tymer(Tymee): .restart() = restart tymer at last ._stop so no time lost Hidden: - ._tymth is injected function wrapper closure returned by .tymen() of + _tymth (closure): injected function wrapper closure returned by .tymen() of associated Tymist instance that returns Tymist .tyme. when called. - ._start is start tyme in seconds - ._stop is stop tyme in seconds + _start (float): start tyme in seconds + _stop (float): stop tyme in seconds """ Duration = 0.0 # default duration when not provided diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 9c82a68..b95524d 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -116,7 +116,6 @@ class Memoer(hioing.Mixin): memo -> .txms deque -> rend -> grams -> .txgs deque -> send -> .txbs - On the receive side each complete memogram (gram) is put in a gram receive deque as a memogram (datagram sized) segment of a memo. These deques are indexed by the sender's source addr. @@ -127,7 +126,6 @@ class Memoer(hioing.Mixin): receive -> (gram, src) -> grams parsed to .rxgs .counts .vids .sources -> fuse -> memo .rxms deque - When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation of the sent datagram which means the whole datagram is not sent at once via @@ -223,13 +221,21 @@ class Memoer(hioing.Mixin): MaxMemoSize. verific (bool): True means any rx grams must be signed. False otherwise - + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter Hidden: _code (bytes | None): see size property _curt (bool): see curt property _size (int): see size property _verific (bool): see verific property + _echoic (bool): see echoic property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex @@ -241,6 +247,8 @@ class Memoer(hioing.Mixin): '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), } + + # Base2 Binary index representation of Text Base64 Char Codes #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size MaxGramCount = 16777215 # (2**24-1) absolute max gram count @@ -265,6 +273,7 @@ def __init__(self, *, curt=False, size=None, verific=False, + echoic=False, **kwa ): """Setup instance @@ -326,6 +335,14 @@ def __init__(self, *, MaxMemoSize. verific (bool): True means any rx grams must be signed. False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter """ # initialize attributes @@ -364,6 +381,7 @@ def __init__(self, *, if self.bs is None: # default if not provided by mixin self.bs = self.BufSize + self._echoic = True if echoic else False @property def code(self): @@ -457,6 +475,21 @@ def verific(self): """ return self._verific + @property + def echoic(self): + """Property getter for ._echoic + + Returns: + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + """ + return self._echoic + def open(self): """Opens transport in nonblocking mode @@ -665,22 +698,33 @@ def pick(self, gram): def receive(self, *, echoic=False) -> (bytes, str|tuple|None): """Attemps to receive bytes from remote source. + + May use echoic=True and .echos to mock a transport layer for testing + Puts sent gram into .echos so that .recieve can extract it when using + same Memoer to send and recieve to itself via its own .echos + When using different Memoers each for send and recieve then must + manually copy from sender Memoer .echos to Receiver Memoer .echos + Must be overridden in subclass. This is a stub to define mixin interface. Parameters: - echoic (bool): True means use .echos in .receive debugging purposes - where echo is duple of form: (gram: bytes, src: str) - False means do not use .echos default is duple that + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that indicates nothing to receive of form (b'', None) + Returns: duple (tuple): of form (data: bytes, src: str|tuple|None) where data is the bytes of received data and src is the source address. When no data the duple is (b'', None) unless echoic is True then pop off echo from .echos """ - + echoic = echoic or self.echoic # use parm when True else default .echoic if echoic: try: result = self.echos.popleft() @@ -1015,6 +1059,13 @@ def rend(self, memo, vid=None): def send(self, gram, dst, *, echoic=False) -> int: """Attemps to send bytes in txbs to remote destination dst. + + May use echoic=True and .echos to mock a transport layer for testing + Puts sent gram into .echos so that .recieve can extract it when using + same Memoer to send and recieve to itself via its own .echos + When using different Memoers each for send and recieve then must + manually copy from sender Memoer .echos to Receiver Memoer .echos + Must be overridden in subclass. This is a stub to define mixin interface. @@ -1024,9 +1075,16 @@ def send(self, gram, dst, *, echoic=False) -> int: Parameters: gram (bytearray): of bytes to send dst (str): remote destination address - echoic (bool): True means echo sends into receives via. echos - False measn do not echo + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + """ + echoic = echoic or self.echoic # use parm when True else default .echoic if echoic: self.echos.append((bytes(gram), dst)) # make copy @@ -1316,22 +1374,107 @@ class SureMemoer(Tymee, Memoer): delivery services. Subclass of Tymee and Memoer - - Inherited Class Attributes: - see superclass + Inherited Class Attributes (Memoer): + Version (Versionage): default version consisting of namedtuple of form + (major: int, minor: int) + Codex (GramDex): dataclass ref to gram codex + Codes (dict): maps codex names to codex values + Names (dict): maps codex values to codex names + Sodex (SGDex): dataclass ref to signed gram codex + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes + MaxMemoSize (int): absolute max memo size + MaxGramCount (int): absolute max gram count + BufSize (int): used to set default buffer size for transport datagram buffers Class Attributes: Tymeout (float): default timeout for retry tymer(s) if any - Inherited Attributes: - see superclass + Inherited Stubbed Attributes (Memoer): + name (str): unique name for Memoer transport. + Used to manage multiple instances. + opened (bool): True means transport open for use + False otherwise + bc (int|None): count of transport buffers of MaxGramSize + bs (int|None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize + + Inherited Stubbed Methods (Memoer): + send(gram, dst, *, echoic=False) -> int # send gram over transport to dst + receive(self, *, echoic=False) -> (bytes, str|tuple|None) # receive gram + + Inherited Attributes (Memoer): + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. + sources (dict): keyed by mid (memoID) that holds the src for the memo. + This enables reattaching src to fused memo in rxms deque tuple. + counts (dict): keyed by mid (memoID) that holds the gram count from + the first gram for the memo. This enables lookup of the gram count when + fusing its grams. + vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + the memo indexed by its mid (memoID). This enables reattaching + the vid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise vid is None + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is tuple of form: + (memo: str, src: str, vid: str) where: + memo is fused memo, src is source addr, vid is verification ID + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is tuple of form + (memo: str, dst: str, vid: str | None) + memo is memo to be partitioned into gram + dst is dst addr for grams + vid is verification id when gram is to be signed or None otherwise + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when Encodesdatagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) + echos (deque): holding echo receive duples for testing. Each duple of + form: (gram: bytes, dst: str). Attributes: tymeout (float): default timeout for retry tymer(s) if any tymers (dict): keys are tid and values are Tymers for retry tymers for each inflight tx - + Inherited Properties (Tymee): + tyme (float | None): relative cycle time of associated Tymist which is + provided by calling .tymth function wrapper closure which is obtained + from Tymist.tymen(). + None means not assigned yet. + tymth (Callable | None): function wrapper closure returned by + Tymist.tymen() method. When .tymth is called it returns associated + Tymist.tyme. Provides injected dependency on Tymist cycle tyme base. + None means not assigned yet. + + Inherited Properties (Memoer): + code (bytes | None): gram code for gram header when rending for tx + curt (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = over head size + neck size + body size. + other gram size = over head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. + verific (bool): True means any rx grams must be signed. + False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index a2c4137..3e861b2 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -106,8 +106,14 @@ class PeerMemoer(Peer, Memoer): MaxMemoSize. verific (bool): True means any rx grams must be signed. False otherwise - - + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter """ diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index 2ea55e7..426f7dc 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -27,7 +27,9 @@ class PeerMemoer(Peer, Memoer): See Peer Class See Memoer Class - + Inherited Properties: + See Peer Class + See Memoer Class """ def __init__(self, *, bc=4, **kwa): diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 619f62a..fb24000 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -20,15 +20,13 @@ def test_memoer_class(): assert Memoer.Codex == memoing.GramDex assert Memoer.Codes == {'Basic': '__', 'Signed': '_-'} - assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} - - # Codes table with sizes of code (hard) and full primitive material - assert Memoer.Sizes == {'__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), - '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160)} - - - # verify all Codes + assert Memoer.Sizes == \ + { + '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), + '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), + } + # verify Sizes and Codes for code, val in Memoer.Sizes.items(): cs = val.cs ms = val.ms @@ -52,12 +50,16 @@ def test_memoer_class(): if vs: assert ss # ss must not be empty if vs not empty + assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} assert Memoer.Sodex == SGDex + + # Base2 Binary index representation of Text Base64 Char Codes #assert Memoer.Bodes == {b'\xff\xf0': '__', b'\xff\xe0': '_-'} assert Memoer.MaxMemoSize == (2**32-1) # absolute max memo payload size assert Memoer.MaxGramSize == (2**16-1) # absolute max gram size assert Memoer.MaxGramCount == (2**24-1) # absolute max gram count + assert Memoer.BufSize == (2**16-1) # default buffersize """Done Test""" @@ -76,6 +78,7 @@ def test_memoer_basic(): assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize assert not peer.verific + assert not peer.echoic peer.reopen() assert peer.opened == True @@ -100,6 +103,7 @@ def test_memoer_basic(): assert not peer.txgs assert peer.txbs == (b'', None) + # inject sent gram into .echos so it can recieve from its own as mock transport assert not peer.rxgs assert not peer.counts assert not peer.sources @@ -120,7 +124,7 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms - # send and receive via echo + # send and receive via .echos to itself as both sender and receiver memo = "See ya later!" dst = "beta" peer.memoit(memo, dst) @@ -131,7 +135,7 @@ def test_memoer_basic(): assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' - peer.serviceTxGrams(echoic=True) + peer.serviceTxGrams(echoic=True) # send to .echos assert not peer.txgs assert peer.txbs == (b'', None) assert peer.echos @@ -140,7 +144,7 @@ def test_memoer_basic(): assert not peer.rxms assert not peer.counts assert not peer.sources - peer.serviceReceives(echoic=True) + peer.serviceReceives(echoic=True) # receive own echo mid = list(peer.rxgs.keys())[0] assert peer.rxgs[mid][0] == b'See ya later!' assert peer.counts[mid] == 1 @@ -213,6 +217,7 @@ def test_memoer_small_gram_size(): assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == 33 # can't be smaller than head + neck + 1 assert not peer.verific + assert not peer.echoic peer = memoing.Memoer(size=38) assert peer.size == 38 @@ -375,6 +380,7 @@ def test_memoer_multiple(): assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt assert not peer.verific + assert not peer.echoic peer.reopen() assert peer.opened == True @@ -447,6 +453,7 @@ def test_memoer_basic_signed(): assert peer.Sizes[peer.code] == (2, 22, 44, 88, 4, 160) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize assert not peer.verific + assert not peer.echoic peer.reopen() assert peer.opened == True @@ -595,6 +602,7 @@ def test_memoer_multiple_signed(): assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt assert not peer.verific + assert not peer.echoic peer.reopen() assert peer.opened == True @@ -725,6 +733,7 @@ def test_memoer_verific(): assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize assert peer.verific + assert not peer.echoic peer.reopen() assert peer.opened == True From 54d745b06f7405a51135cb5714b750f864371318 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Jan 2026 13:29:08 -0700 Subject: [PATCH 12/27] more unit tests to test serviceAll with signed multiple --- src/hio/core/memo/memoing.py | 13 +- tests/core/memo/test_memoing.py | 243 ++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 4 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index b95524d..b3d17ed 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -111,7 +111,7 @@ class Memoer(hioing.Mixin): Each memo is then segmented into grams (memograms) that respect the size constraints of the underlying datagram transport. These grams are placed in the outgoing gram deque. Each entry in this deque is a duple of form: - (gram: bytes, dst: str). Each duple is pulled off the deque and its + (gram: bytes, dst: str). Each duple is pulled off the self._serviceOneRxMemo()deque and its gram is put in bytearray for transport. memo -> .txms deque -> rend -> grams -> .txgs deque -> send -> .txbs @@ -206,8 +206,10 @@ class Memoer(hioing.Mixin): portion when Encodesdatagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - echos (deque): holding echo receive duples for testing. Each duple of + echos (deque): holds echo receive duples for testing. Each duple of form: (gram: bytes, dst: str). + inbox (deque): holds final received complete memos for testing when not + overridden in subclass to further process otherwise Properties: code (bytes | None): gram code for gram header when rending for tx @@ -358,6 +360,7 @@ def __init__(self, *, self.txbs = txbs if txbs is not None else (bytearray(), None) self.echos = deque() # only used in testing as echoed tx + self.inbox = deque() # holds complete receive memos for testing self.code = code self.curt = curt @@ -911,7 +914,8 @@ def serviceRxMemosOnce(self): Override in subclass to handle result and put it somewhere """ try: - memo, src, vid = self._serviceOneRxMemo() + #memo, src, vid = self._serviceOneRxMemo() + self.inbox.append(self._serviceOneRxMemo()) except IndexError: pass @@ -922,7 +926,8 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + #memo, src, vid = self._serviceOneRxMemo() + self.inbox.append(self._serviceOneRxMemo()) def serviceAllRxOnce(self): diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index fb24000..65ce38b 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -3,6 +3,7 @@ tests.core.test_memoing module """ +from collections import deque from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 @@ -124,6 +125,8 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + # send and receive via .echos to itself as both sender and receiver memo = "See ya later!" dst = "beta" @@ -159,6 +162,9 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 assert peer.curt @@ -198,6 +204,11 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + assert peer.inbox[2] == ('Hello There', 'beta', None) + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -272,6 +283,8 @@ def test_memoer_small_gram_size(): assert not peer.rxms assert not peer.echos + assert peer.inbox[0] == ('Hello There', 'beta', None) + # send and receive via echo memo = "See ya later!" dst = "beta" @@ -310,6 +323,9 @@ def test_memoer_small_gram_size(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 assert peer.curt @@ -363,6 +379,11 @@ def test_memoer_small_gram_size(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + assert peer.inbox[2] == ('See ya later alligator!', 'beta', None) + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -434,11 +455,115 @@ def test_memoer_multiple(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_memoer_multiple_echoic_service_tx_rx(): + """Test Memoer class with small gram size and multiple queued memos + Use .echoic property true so can service all + """ + peer = memoing.Memoer(size=38, echoic=True) + assert peer.size == 38 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.curt + assert not peer.verific + assert peer.echoic + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + + peer.serviceAllTx() + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAllRx() + + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it peer.close() assert peer.opened == False """ End Test """ + +def test_memoer_multiple_echoic_service_all(): + """Test Memoer class with small gram size and multiple queued memos + Use .echoic property true so can service all + """ + peer = memoing.Memoer(size=38, echoic=True) + assert peer.size == 38 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.curt + assert not peer.verific + assert peer.echoic + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + + peer.serviceAll() # services Rx first then Tx so have to serviceAll twice + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # Rx not serviced yet after Tx serviced + + peer.serviceAll() # services Rx first then Tx so have to serviceAll twice + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it + + peer.close() + assert peer.opened == False + """ End Test """ + + def test_memoer_basic_signed(): """Test Memoer class basic signed code """ @@ -586,6 +711,12 @@ def test_memoer_basic_signed(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', vid) + assert peer.inbox[1] == ('See ya later!', 'beta', vid) + assert peer.inbox[2] == ('Hello There', 'beta', vid) + + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -713,6 +844,14 @@ def test_memoer_multiple_signed(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid), + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid) + ]) + peer.close() assert peer.opened == False @@ -774,6 +913,107 @@ def test_memoer_verific(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', vid) + ]) + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_memoer_multiple_signed_verific_echoic_service_all(): + """Test Memoer class with small gram size and multiple queued memos signed + using echos for transport + """ + # verific forces rx memos to be signed or dropped + # to force signed tx then use Signed code + peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, echoic=True) + assert peer.size == 170 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.curt + assert peer.verific + assert peer.echoic + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha", vid) + peer.memoit("How ya doing?", "beta", vid) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid) + ]) + + + # test in base2 mode + peer.curt = True + assert peer.curt + peer.size = 129 + assert peer.size == 129 + + # send and receive multiple via echo + vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha", vid) + peer.memoit("How ya doing?", "beta", vid) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid), + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid), + ]) + + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -1023,9 +1263,12 @@ def test_sure_memoer_doer(): test_memoer_basic() test_memoer_small_gram_size() test_memoer_multiple() + test_memoer_multiple_echoic_service_tx_rx() + test_memoer_multiple_echoic_service_all() test_memoer_basic_signed() test_memoer_multiple_signed() test_memoer_verific() + test_memoer_multiple_signed_verific_echoic_service_all() test_open_memoer() test_memoer_doer() test_sure_memoer_basic() From b135e9650b937592783f803b7b99ea9067280349 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Jan 2026 17:38:00 -0700 Subject: [PATCH 13/27] added .keep property to Memoer for lightweight key managment for testing --- src/hio/core/memo/memoing.py | 47 ++++++- src/hio/core/udp/peermemoing.py | 5 + tests/core/memo/test_memoing.py | 233 +++++++++++++++++++++++--------- 3 files changed, 214 insertions(+), 71 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index b3d17ed..0b8c147 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -31,6 +31,14 @@ # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") +# signature key pair: +# sigkey = private signing key +# verkey = public verifying key +Keyage = namedtuple("Keyage", "sigkey verkey") +# example usage +#keyage = Keyage(sigkey="abc", verkey="xyy") +#keep = dict("ABCXYZ"=keyage) # vid as label, Keyage instance as value + """Sizage: namedtuple for gram header part size entries in Memoer code tables cs is the code part size int number of chars in code part (serialization code) @@ -123,7 +131,7 @@ class Memoer(hioing.Mixin): and placed in the memo deque for consumption by the application or some other higher level protocol. - receive -> (gram, src) -> grams parsed to .rxgs .counts .vids .sources -> + receive -> (gram, src) -> grams parsed to .rxgs .counts .vids .sources -# singing key pair sigkey and verifier key> fuse -> memo .rxms deque When using non-blocking IO, asynchronous datagram transport @@ -227,10 +235,15 @@ class Memoer(hioing.Mixin): transport layer for testing and debugging. False means do not use .echos Each entry in .echos is a duple of form: - (gram: bytes, src: str) + (gram: bytes, src: str)# singing key pair sigkey and verifier key Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter + keep (dict): labels or vids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") Hidden: _code (bytes | None): see size property @@ -238,6 +251,7 @@ class Memoer(hioing.Mixin): _size (int): see size property _verific (bool): see verific property _echoic (bool): see echoic property + _keep (dict): see keep property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex @@ -276,6 +290,7 @@ def __init__(self, *, size=None, verific=False, echoic=False, + keep=None, **kwa ): """Setup instance @@ -345,6 +360,11 @@ def __init__(self, *, Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter + keep (dict|None): labels are vids and values are Keyage instances + that provide current signature key pair for vid + this is a lightweight mechanism that should be + overridden in subclass for real world key management. + """ # initialize attributes @@ -385,6 +405,7 @@ def __init__(self, *, self.bs = self.BufSize self._echoic = True if echoic else False + self._keep = keep if keep is not None else dict() @property def code(self): @@ -493,6 +514,19 @@ def echoic(self): """ return self._echoic + @property + def keep(self): + """Property getter for ._keep + + Returns: + keep (dict): labels or vids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") + """ + return self._keep + def open(self): """Opens transport in nonblocking mode @@ -1480,11 +1514,16 @@ class SureMemoer(Tymee, Memoer): Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter + keep (dict): labels or vids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout - def __init__(self, *, tymeout=None, **kwa): + def __init__(self, *, tymeout=None, code=GramDex.Signed, verific=True, **kwa): """ Initialization method for instance. Inherited Parameters: @@ -1494,7 +1533,7 @@ def __init__(self, *, tymeout=None, **kwa): tymeout (float): default for retry tymer if any """ - super(SureMemoer, self).__init__(**kwa) + super(SureMemoer, self).__init__(code=code, verific=verific, **kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout self.tymers = {} #Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index 3e861b2..8b938a2 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -114,6 +114,11 @@ class PeerMemoer(Peer, Memoer): Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter + keep (dict): labels or vids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") """ diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 65ce38b..28f6063 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -80,6 +80,7 @@ def test_memoer_basic(): assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -229,6 +230,7 @@ def test_memoer_small_gram_size(): assert peer.size == 33 # can't be smaller than head + neck + 1 assert not peer.verific assert not peer.echoic + assert peer.keep == dict() peer = memoing.Memoer(size=38) assert peer.size == 38 @@ -402,6 +404,7 @@ def test_memoer_multiple(): assert not peer.curt assert not peer.verific assert not peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -479,6 +482,7 @@ def test_memoer_multiple_echoic_service_tx_rx(): assert not peer.curt assert not peer.verific assert peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -531,6 +535,7 @@ def test_memoer_multiple_echoic_service_all(): assert not peer.curt assert not peer.verific assert peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -579,6 +584,7 @@ def test_memoer_basic_signed(): assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -734,6 +740,7 @@ def test_memoer_multiple_signed(): assert not peer.curt assert not peer.verific assert not peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -873,6 +880,7 @@ def test_memoer_verific(): assert peer.size == peer.MaxGramSize assert peer.verific assert not peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -939,6 +947,7 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert not peer.curt assert peer.verific assert peer.echoic + assert peer.keep == dict() peer.reopen() assert peer.opened == True @@ -1063,18 +1072,24 @@ def test_memoer_doer(): def test_sure_memoer_basic(): - """Test TymeeMemoer class basic + """Test SureMemoer class basic """ - peer = memoing.SureMemoer() - assert peer.tymeout == 0.0 + peer = memoing.SureMemoer(echoic=True) + assert peer.size == 65535 assert peer.name == "main" assert peer.opened == False - assert peer.code == memoing.GramDex.Basic == '__' + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt + assert peer.verific + assert peer.echoic + assert peer.keep == dict() + # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize - assert not peer.verific + assert peer.tymeout == 0.0 assert peer.tymers == {} peer.reopen() @@ -1083,116 +1098,105 @@ def test_sure_memoer_basic(): assert not peer.txms assert not peer.txgs assert peer.txbs == (b'', None) - peer.service() + peer.service() # alias for .serviceAll assert peer.txbs == (b'', None) memo = "Hello There" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('Hello There', 'beta', None) - peer.serviceTxMemos() + vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit(memo, dst, vid) + assert peer.txms[0] == ('Hello There', 'beta', vid) + + peer.service() # services Rx then Tx so rx of tx not serviced until 2nd pass + assert not peer.txms - m, d = peer.txgs[0] - assert not peer.wiff(m) # base64 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.counts assert not peer.sources assert not peer.rxms - mid = '__ALBI68S1ZIxqwFOSWFF1L2' - gram = (mid + 'AAAA' + 'AAAB' + "Hello There").encode() - echo = (gram, "beta") - peer.echos.append(echo) - peer.serviceReceives(echoic=True) - assert peer.rxgs[mid][0] == bytearray(b'Hello There') - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' - peer.serviceRxGrams() + + peer.service() + + assert not peer.echos assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms - # send and receive via echo + assert peer.inbox == deque( + [ + ('Hello There', 'beta', vid), + ]) + + # send and receive some more memo = "See ya later!" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('See ya later!', 'beta', None) - peer.serviceTxMemos() + peer.memoit(memo, dst, vid) + assert peer.txms[0] == ('See ya later!', 'beta', vid) + + peer.service() + assert not peer.txms - m, d = peer.txgs[0] - assert not peer.wiff(m) # base64 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams(echoic=True) assert not peer.txgs assert peer.txbs == (b'', None) - assert peer.echos + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.rxms assert not peer.counts assert not peer.sources - peer.serviceReceives(echoic=True) - mid = list(peer.rxgs.keys())[0] - assert peer.rxgs[mid][0] == b'See ya later!' - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' + + peer.service() + assert not peer.echos - peer.serviceRxGrams() assert not peer.rxgs assert not peer.counts assert not peer.sources - peer.rxms[0] - assert peer.rxms[0] == ('See ya later!', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', vid), + ('See ya later!', 'beta', vid), + ]) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('Hello There', 'beta', None) - peer.serviceTxMemos() + peer.memoit(memo, dst, vid) + assert peer.txms[0] == ('Hello There', 'beta', vid) + + peer.service() + assert not peer.txms - m, d = peer.txgs[0] - assert peer.wiff(m) # base2 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.counts assert not peer.sources assert not peer.rxms - mid = '__ALBI68S1ZIxqwFOSWFF1L2' - headneck = decodeB64((mid + 'AAAA' + 'AAAB').encode()) - gram = headneck + b"Hello There" - assert peer.wiff(gram) # base2 - echo = (gram, "beta") - peer.echos.append(echo) - peer.serviceReceives(echoic=True) - assert peer.rxgs[mid][0] == bytearray(b'Hello There') - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' - peer.serviceRxGrams() + + peer.service() + assert not peer.echos assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', vid), + ('See ya later!', 'beta', vid), + ('Hello There', 'beta', vid), + ]) + # Test wind tymist = tyming.Tymist(tock=1.0) peer.wind(tymth=tymist.tymen()) @@ -1204,7 +1208,6 @@ def test_sure_memoer_basic(): assert peer.opened == False """ End Test """ - def test_open_sm(): """Test contextmanager decorator openTM for openTymeeMemoer """ @@ -1221,6 +1224,101 @@ def test_open_sm(): """ End Test """ +def test_sure_memoer_multiple_echoic_service_all(): + """Test SureMemoer class with small gram size and multiple queued memos signed + using echos for transport + """ + # verific forces rx memos to be signed or dropped + # to force signed tx then use Signed code + + with memoing.openSM(size=170, echoic=True) as peer: + assert peer.size == 170 + assert peer.name == "test" + assert peer.opened == True + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.curt + assert peer.verific + assert peer.echoic + assert peer.keep == dict() + + # send and receive multiple via echo + vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha", vid) + peer.memoit("How ya doing?", "beta", vid) + assert len(peer.txms) == 2 + + peer.service() # services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid) + ]) + + + # test in base2 mode + peer.curt = True + assert peer.curt + peer.size = 129 + assert peer.size == 129 + + # send and receive multiple via echo + vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha", vid) + peer.memoit("How ya doing?", "beta", vid) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid), + ('Hello there.', 'alpha', vid), + ('How ya doing?', 'beta', vid), + ]) + + peer.inbox = deque() # clear it + + assert peer.opened == False + """ End Test """ + + def test_sure_memoer_doer(): """Test SureMemoerDoer class """ @@ -1273,5 +1371,6 @@ def test_sure_memoer_doer(): test_memoer_doer() test_sure_memoer_basic() test_open_sm() + test_sure_memoer_multiple_echoic_service_all() test_sure_memoer_doer() From 280d2870e557c207840597b0113c63eca27427c4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Jan 2026 17:54:08 -0700 Subject: [PATCH 14/27] added .vid property so has own default vid for signing on tx --- src/hio/core/memo/__init__.py | 2 +- src/hio/core/memo/memoing.py | 31 ++++++++++++++++++++++++++++--- src/hio/core/udp/peermemoing.py | 1 + tests/core/memo/test_memoing.py | 17 ++++++++++++++++- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/hio/core/memo/__init__.py b/src/hio/core/memo/__init__.py index a27fc1d..2dd8835 100644 --- a/src/hio/core/memo/__init__.py +++ b/src/hio/core/memo/__init__.py @@ -4,7 +4,7 @@ """ -from .memoing import (Versionage, Sizage, GramDex, SGDex, +from .memoing import (Versionage, Sizage, GramDex, SGDex, Keyage, openMemoer, Memoer, MemoerDoer, openSM, SureMemoer, SureMemoerDoer) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 0b8c147..96e22b9 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -32,11 +32,11 @@ Versionage = namedtuple("Versionage", "major minor") # signature key pair: -# sigkey = private signing key # verkey = public verifying key -Keyage = namedtuple("Keyage", "sigkey verkey") +# sigkey = private signing key +Keyage = namedtuple("Keyage", "verkey sigkey") # example usage -#keyage = Keyage(sigkey="abc", verkey="xyy") +#keyage = Keyage(verkey="xyy", sigkey="abc", ) #keep = dict("ABCXYZ"=keyage) # vid as label, Keyage instance as value @@ -244,6 +244,7 @@ class Memoer(hioing.Mixin): sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") + vid (str|None): own vid defaults used to lookup keys to sign on tx Hidden: _code (bytes | None): see size property @@ -252,6 +253,7 @@ class Memoer(hioing.Mixin): _verific (bool): see verific property _echoic (bool): see echoic property _keep (dict): see keep property + _vid (str|None): see vid property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex @@ -291,6 +293,7 @@ def __init__(self, *, verific=False, echoic=False, keep=None, + vid=None, **kwa ): """Setup instance @@ -364,6 +367,7 @@ def __init__(self, *, that provide current signature key pair for vid this is a lightweight mechanism that should be overridden in subclass for real world key management. + vid (str|None): own vid defaults used to lookup keys to sign on tx """ @@ -406,6 +410,7 @@ def __init__(self, *, self._echoic = True if echoic else False self._keep = keep if keep is not None else dict() + self.vid = vid if vid else None @property def code(self): @@ -527,6 +532,25 @@ def keep(self): """ return self._keep + @property + def vid(self): + """Property getter for ._vid + + Returns: + vid (str|None): vid used to sign on tx if any + """ + return self._vid + + + @vid.setter + def vid(self, vid): + """Property setter for ._vid + + Parameters: + vid (str|None): value to assign to own vid + """ + self._vid = vid + def open(self): """Opens transport in nonblocking mode @@ -1519,6 +1543,7 @@ class SureMemoer(Tymee, Memoer): sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") + vid (str|None): own vid defaults used to lookup keys to sign on tx """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index 8b938a2..c52c369 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -119,6 +119,7 @@ class PeerMemoer(Peer, Memoer): sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") + vid (str|None): own vid defaults used to lookup keys to sign on tx """ diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 28f6063..f843905 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -12,7 +12,11 @@ from hio.help import helping from hio.base import doing, tyming from hio.core.memo import memoing -from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer +from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer, Keyage + +keyage = Keyage(verkey="ABCD", sigkey="WXYZ") # for testing +Keep = dict(abcdwxyz=keyage) # for testing + def test_memoer_class(): """Test class attributes of Memoer class""" @@ -81,6 +85,7 @@ def test_memoer_basic(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -231,6 +236,7 @@ def test_memoer_small_gram_size(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer = memoing.Memoer(size=38) assert peer.size == 38 @@ -405,6 +411,7 @@ def test_memoer_multiple(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -483,6 +490,7 @@ def test_memoer_multiple_echoic_service_tx_rx(): assert not peer.verific assert peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -536,6 +544,7 @@ def test_memoer_multiple_echoic_service_all(): assert not peer.verific assert peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -585,6 +594,7 @@ def test_memoer_basic_signed(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -741,6 +751,7 @@ def test_memoer_multiple_signed(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -881,6 +892,7 @@ def test_memoer_verific(): assert peer.verific assert not peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -948,6 +960,7 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.verific assert peer.echoic assert peer.keep == dict() + assert peer.vid is None peer.reopen() assert peer.opened == True @@ -1085,6 +1098,7 @@ def test_sure_memoer_basic(): assert peer.verific assert peer.echoic assert peer.keep == dict() + assert peer.vid is None # (code, mid, vid, neck, head, sig) part sizes assert peer.Sizes[peer.code] == Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160) # cs ms vs ss ns hs @@ -1242,6 +1256,7 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.verific assert peer.echoic assert peer.keep == dict() + assert peer.vid is None # send and receive multiple via echo vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' From 00dd7bb9bd779e87cc69a49cb1d628ee606cf0b8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 11 Jan 2026 10:32:22 -0700 Subject: [PATCH 15/27] started to add support for simple key management for signed memos --- tests/core/memo/test_memoing.py | 18 ++++++++++++++++-- tests/core/udp/test_peer_memoing.py | 3 ++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index f843905..6bf4e4c 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -14,8 +14,22 @@ from hio.core.memo import memoing from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer, Keyage -keyage = Keyage(verkey="ABCD", sigkey="WXYZ") # for testing -Keep = dict(abcdwxyz=keyage) # for testing + + +def _setupKeep(): + """Setup Keep for signed memos + + Returns: + keep (dict): labels are vids, values are keyage instances + + """ + keep = {} + + vid = 'abcdwxyz' + keyage = Keyage(verkey="ABCD", sigkey="WXYZ") # for testing + keep[vid] = keyage # for testing + + return keep def test_memoer_class(): diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index 246f683..37271f3 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -205,7 +205,8 @@ def test_memoer_peer_open(): # beta receives beta.serviceReceives() - time.sleep(0.1) + while not beta.rxgs: + time.sleep(0.05) assert not beta.echos assert len(beta.rxgs) == 2 assert len(beta.counts) == 2 From a06dc6a5a214a06fe819a71de3c3e2948744a41b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 11 Jan 2026 10:50:17 -0700 Subject: [PATCH 16/27] added _setupKeep utility to test Memoing with key management --- tests/core/memo/test_memoing.py | 56 ++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 6bf4e4c..f3fda7f 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -9,6 +9,7 @@ import pytest +from hio.hioing import MemoerError from hio.help import helping from hio.base import doing, tyming from hio.core.memo import memoing @@ -16,13 +17,26 @@ -def _setupKeep(): +def _setupKeep(salt=None): """Setup Keep for signed memos Returns: keep (dict): labels are vids, values are keyage instances """ + salt = salt if salt is not None else b"abcdefghijklmnop" + if hasattr(salt, 'encode'): + salt = salt.encode() + + if len(salt) != 16: + raise MemoerError("Invalid provided salt") + + try: + import pysodium + import blake3 + except ImportError: + raise MemoerError("Missing cryptographic module support") + keep = {} vid = 'abcdwxyz' @@ -595,6 +609,12 @@ def test_memoer_multiple_echoic_service_all(): def test_memoer_basic_signed(): """Test Memoer class basic signed code """ + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + peer = memoing.Memoer(code=GramDex.Signed) assert peer.name == "main" assert peer.opened == False @@ -754,6 +774,12 @@ def test_memoer_basic_signed(): def test_memoer_multiple_signed(): """Test Memoer class with small gram size and multiple queued memos signed """ + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + peer = memoing.Memoer(code=GramDex.Signed, size=170) assert peer.size == 170 assert peer.name == "main" @@ -893,6 +919,7 @@ def test_memoer_multiple_signed(): def test_memoer_verific(): """Test Memoer class with verific (signed required) """ + peer = memoing.Memoer(verific=True) assert peer.name == "main" assert peer.opened == False @@ -961,6 +988,12 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): """Test Memoer class with small gram size and multiple queued memos signed using echos for transport """ + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, echoic=True) @@ -1101,6 +1134,12 @@ def test_memoer_doer(): def test_sure_memoer_basic(): """Test SureMemoer class basic """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + + peer = memoing.SureMemoer(echoic=True) assert peer.size == 65535 assert peer.name == "main" @@ -1239,6 +1278,11 @@ def test_sure_memoer_basic(): def test_open_sm(): """Test contextmanager decorator openTM for openTymeeMemoer """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + with (memoing.openSM(name='zeta') as zeta): assert zeta.opened @@ -1256,6 +1300,11 @@ def test_sure_memoer_multiple_echoic_service_all(): """Test SureMemoer class with small gram size and multiple queued memos signed using echos for transport """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code @@ -1351,6 +1400,11 @@ def test_sure_memoer_multiple_echoic_service_all(): def test_sure_memoer_doer(): """Test SureMemoerDoer class """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + tock = 0.03125 ticks = 4 limit = ticks * tock From 49c22b13bdfde7a07e1e2f1cc6ff805922ed399f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 16 Jan 2026 12:31:51 -0700 Subject: [PATCH 17/27] updated doc strings and max memo size --- src/hio/core/memo/memoing.py | 163 +++++++++++++++++++++++++++++++- tests/core/memo/test_memoing.py | 34 +++++-- 2 files changed, 184 insertions(+), 13 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 96e22b9..5324d83 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -39,6 +39,162 @@ #keyage = Keyage(verkey="xyy", sigkey="abc", ) #keep = dict("ABCXYZ"=keyage) # vid as label, Keyage instance as value +"""Design Discusssion of Memo and Gram Sizing and Encoding: + +Each GramCode (tv) Typing/Version code uses a base64 two char code. +Codes always start with `_` and second char in code starts at `_` +and walks backwards through B64 code table for each gram type-version (tv) code. + +tv 0 is '__' +tv 1 is '_-' +tv 3 is '_9' +... + +Each of total head length including code and total neck length must be a +multiple of 4 characters (3 bytes) to align on 24 bit boundary for clean Base64 +round tripping. This ensure each converts to/from B2 without padding. +Head type-version code governs the neck as well. So whenever field lengths or +fields change in any way we need a new gram (tv) code. + +To reiterate, if ever need to change anything need a new gram (tv) code. There are +no versions per type. This is deemed sufficient because we anticipate a very +limited set of possible fields ever needed for memogram transport types. + +All gram head and neck fields are mid-pad B64 primitives. This makes for stable +coding at fron and back of head neck parts. This also means that gram parts can +can be losslessly transformed to/from equivalent CESR primitives merely by +translating the gram tv code to an equivalent entry in the two char cesr +primitive code table. When doing so the neck is always attached and then +stripped when not needed. + +The equivalent CESR codes, albeit different, map one-to-one to the gram tv codes. +This enables representing headers as CESR primitives for management but not +over the wire. One can transform Memogram Grams to CESR primitives to tunnel in +some CESR stream. + +Normally CESR streams are tunneled inside Memoer memograms sent over-the-wire. +In this case, the memogram is the wrapper and each gram uses the gram TV code +defined herein not the equivalent CESR code. +Using so far reserved and unused '_' code which is reserved as the selector +for the CESR op code table for memogram grams makes it more apparent that a gram +belongs to a memogram and not a CESR stream. +Eventually when CESR op codes eventually become defined, it's not likely to be +confusing since the context of CESR op codes is dramatically different from +transport wrappers. + +Morover when a MemoGram payload is a tunneled CESR Stream, the memogram parser +will parse and strip the MemoGram gram wrappers to construct the tunneled, +stream so the wrapper is transarent to the CESR stream and the CESR stream +payload is opaque to the memogram wrapper, i.e. no ambiguity. + +Gram Header Fields: + HeadCode hc = b'__' + HeadCodeSize hcs = 2 + MemoIDSize mis = 22 + GramNumSize gns = 4 + GramCntSize gcs = 4 + GramHeadSize ghs = 28 + GramHeadNeckSize ghns = 28 + 4 = 32 + +Sizing: + +We have for most datagrams, such as UDP, a max datagram size of 65535 = (2 ** 16 -1) +Thix may no longer be true on some OSes for UXD datagrams. On some OSes UXD datagrams +may be bigger. But 65535 seems like a good practical limit for maximum compatibility +across all datagram types. + +Given gram count and gram num fields are 4 Base64 chars (3 bytes) or 24 bits, then: +MaxGramCount = 2**24-1 + +If we want a maximally size memo to be big enough to transport 1 CESR Big frame +with frame counter then the maximum memo size needs to be +(2**30-1) * 4 + 8 = 4294967292 + 8 = 4294967300 + +CESR groups code count in quadlets (4 chars). +The big group count codes have five digits of 6 bits each or 30 bits to count the +size of the following framed chars. +The maximum group count is (2**30-1) == 1073741823 +Cesr group codes count the number +of quadlets (4 chars) in the framed group. The maximum number of bytes in a +CESR big group frame body is therefore: +(2**30-1)*4 = 1073741823 * 4 = 4294967292. +The group code itself is not included in the counted quadlets in the frame body. +The big group codes are 8 chars long. Thus with its group code the total big +frame size is: +(2**30-1)*4 + 8 = 4294967292 + 8 = 4294967300 + +We set +MaxMemoSize (mms) = (2**30-1)*4+8 = 4294967300 + +We can compare UDP and UXD given these constraints. +Notice that gramsize includes head + body and the first gram size is 4 larger +to include gram count field. + +In general +MaxMemoSize = GramBodySize * MaxGramCount + +Because the gram count field is 24 bits (4 chars or 3 bytes), we have: +MaxGramCount = 2**24-1 = 16777215 + +At a MaxGramCount of 16777215 for the MaxMemoSize we +GramBodySize = MaxMemoSize / MaxGramCount = 4294967300/16777215 = 256 remainder 260 +256 = 2**8. So we need GramBodySize to be a little larger than 256 to reach +MaxMemoSize. If we round up to 257 we get +maximum possible memosize = 257 * 16777215 = 4311744255 = ((2**8)+1)*(2**24-1) +Or alternatively if we set GramBodySize to 257 then the max gram count we get is +GramCount = ceil(MaxMemoSize / 257) = (ceil(4294967300/257)) = 16711936 < 16777215 +or 4294967300/257 = 16711935 remainder 5 + +An actual Gram includes a GramHead + GramBody. So as long as the transport MTU +is at least as big as the GramHeadSize + GramBodySize then we can transport the maximum +memo size in MaxGramCount grams. + +The signed header for a gram is 160 bytes for all grams but the zeroth gram. +the zeroth gram includes an extra 4 bytes for the gramcount or 164 bytes. + +UDP IPV4 MTU = 576 Net safe payload = gram size = 548 +MaxGramCount for UDP GramSize of 548. Given signed header, GramBodySize is +548 - 160 = 388 which is larger than 257. So a memo of MaxMemoSize of 4294967300 +can be conveyed in IPV4 datagrams with fewer than MaxGramCount grams. +This provides headroom for the extra 4 bytes in the zeroth gram. + +UXD MTU is typically 65535 depending on the allocated buffer size which is much +greater than 257. + + +MinGramSize = MaxMemoSize // MaxGramCount = ceil(((2**30-1)*4+8 ) / (2**24-1)) = 257 = (2**8+1) +This is the min size that could result in maxMemoSize. Smaller gram sizes than 257 +can not reach MaxMemoSize because MaxGramCount not big enough. +Based on datagram constraints we have +MaxGramSize = (2**16-1) + + +The variables ghs, and ghns are fixed for given transport service type reliable +or unreliable with head and neck fields defined appropriately +The variables gs, gbs,and mms are derived from transport MTU size +The desired gs for a given transport MTU instance may be smaller than the allowed +maximum to accomodate buffers etc. +Given the fixed gram head size for service type, ghs one can calculate the +maximum memo size mms that includes the header overhead given the contraint of +no more than 2**24-1 grams in a given memo due to 24 size of gram count gram num. + +So for a given transport: +gs = min(gs, 2**16-1) (gram size) +gbs = gs - ghs (gram body size = gram size - gram head size) + +mms = min((2**30-1)*4+8, gbs * (2**24-1)) (max memo size) +So compare memo size to mms and if greater raise error and drop memo +otherwise partition (gram) memo into grams with headers and transmit + + +Therefore setting the maxmemosize to be (2**30-1)*4+8 enables the a big group +frame from CESR to be conveyed by a memo. +However a maximally sized memo can not be itself conveyed by a big CESR group frame +because maximally size memo is 8 more than the biggest frame body count. +So translating memos to CESR must check that the memo being translated iself +is limited to a size of (2**30-1)*4 not (2**30-1)*4+8 + +""" """Sizage: namedtuple for gram header part size entries in Memoer code tables cs is the code part size int number of chars in code part (serialization code) @@ -268,10 +424,11 @@ class Memoer(hioing.Mixin): # Base2 Binary index representation of Text Base64 Char Codes #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) - MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size - MaxGramCount = 16777215 # (2**24-1) absolute max gram count + # big enough to hold a CESR big frame with code + MaxMemoSize = 4294967300 # (2**30-1)*4+8 absolute max net memo payload size + MaxGramCount = 16777215 # (2**24-1) absolute max gram count MaxGramSize = 65535 # (2**16-1) absolute max gram size overridden in subclass - BufSize = 65535 # 2 ** 16 - 1 default buffersize + BufSize = 65535 # (2**16-1) default buffersize def __init__(self, *, diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index f3fda7f..1eff190 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -24,6 +24,13 @@ def _setupKeep(salt=None): keep (dict): labels are vids, values are keyage instances """ + try: + import pysodium + import blake3 + except ImportError: + raise MemoerError("Missing cryptographic module support") + + salt = salt if salt is not None else b"abcdefghijklmnop" if hasattr(salt, 'encode'): salt = salt.encode() @@ -31,17 +38,24 @@ def _setupKeep(salt=None): if len(salt) != 16: raise MemoerError("Invalid provided salt") - try: - import pysodium - import blake3 - except ImportError: - raise MemoerError("Missing cryptographic module support") - keep = {} - vid = 'abcdwxyz' - keyage = Keyage(verkey="ABCD", sigkey="WXYZ") # for testing - keep[vid] = keyage # for testing + sigseed0 = pysodium.crypto_pwhash(outlen=32, + passwd="0", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed0) # raw + + + + vid = "abcd" + keyage = Keyage(verkey=verkey, sigkey=sigkey) #raw + keep[vid] = keyage + + return keep @@ -89,7 +103,7 @@ def test_memoer_class(): # Base2 Binary index representation of Text Base64 Char Codes #assert Memoer.Bodes == {b'\xff\xf0': '__', b'\xff\xe0': '_-'} - assert Memoer.MaxMemoSize == (2**32-1) # absolute max memo payload size + assert Memoer.MaxMemoSize == ((2**30-1)*4+8) # absolute max memo payload size assert Memoer.MaxGramSize == (2**16-1) # absolute max gram size assert Memoer.MaxGramCount == (2**24-1) # absolute max gram count assert Memoer.BufSize == (2**16-1) # default buffersize From fd281289aa9cfe4a63500c4e1e5a386165a459b3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 16 Jan 2026 14:20:47 -0700 Subject: [PATCH 18/27] utility classmethod ot create CESR compatible enoded vid. test utilit to set up keep for testing --- src/hio/core/memo/memoing.py | 56 ++++++++++++++++++--- tests/core/memo/test_memoing.py | 89 +++++++++++++++++++++++++++++++-- 2 files changed, 134 insertions(+), 11 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 5324d83..e666261 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -33,10 +33,10 @@ # signature key pair: # verkey = public verifying key -# sigkey = private signing key -Keyage = namedtuple("Keyage", "verkey sigkey") +# sigseed = private signing key seed (ed25519 sigkey = sigseed + verkey) +Keyage = namedtuple("Keyage", "verkey sigseed") # example usage -#keyage = Keyage(verkey="xyy", sigkey="abc", ) +#keyage = Keyage(verkey="xyy", sigseed="abc", ) #keep = dict("ABCXYZ"=keyage) # vid as label, Keyage instance as value """Design Discusssion of Memo and Gram Sizing and Encoding: @@ -430,6 +430,47 @@ class Memoer(hioing.Mixin): MaxGramSize = 65535 # (2**16-1) absolute max gram size overridden in subclass BufSize = 65535 # (2**16-1) default buffersize + @classmethod + def _encodeVid(cls, vid, code='B'): + """Utility method for use with signed headers that encodes raw vid as + CESR compatible fully qualified B64 text domain str using CESR compatible + text code + + Parameters: + vid (bytes): raw vid to be encoded with code + code (str): code for type of vid CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verkey basic derivation + Blake3_256: str = 'E' # Blake3 256 bit digest derivation. + + Returns: + + vidqb64 (str): fully qualified base64 vid + + """ + if code not in ('B', 'D', 'E'): + raise hioing.MemoerError("Invalid vid {code=}") + + cs, ms, vs, ss, ns, hs = cls.Sizes[SGDex.Signed] # cs ms vs ss ns hs + + rvs = len(vid) # raw vid size + if rvs != 32: + raise hioing.MemoerError("Invalid raw vid size {rvs=} not 32") + + ps = (3 - ((rvs) % 3)) % 3 # net pad size for raw vid + if len(code) != ps != 1: + raise hioing.MemoerError("Invalid vid code size={len(code)} not equal" + f" {ps=} not equal 1") + + vidb64 = encodeB64(bytes([0] * ps) + vid)[ps:] # prepad, convert, and prestrip + + vidqb64 = code + vidb64.decode() # fully qualified vid with prefix code + + if len(vidqb64) != vs: + hioing.MemoerError("Invalid vid qb64 size={len(vidqb64) != {vs}}") + + return vidqb64 + def __init__(self, *, name=None, @@ -1209,14 +1250,17 @@ def rend(self, memo, vid=None): memo = bytearray(memo.encode()) # convert and copy to bytearray # self.size is max gram size cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid + if vs and (not vid or len(vid) != vs): raise hioing.MemoerError(f"Invalid {vid=} for size={vs}.") - vid = b"" if vid is None else vid[:vs].encode() + ps = (3 - ((ms) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned - mid = encodeB64(bytes([0] * ps) + uuid.uuid1().bytes)[ps:] # prepad, convert, and strip + mid = encodeB64(bytes([0] * ps) + uuid.uuid1().bytes)[ps:] # prepad, convert, and prestrip + if cs != ps or cs != len(self.code): + raise hioing.MemoerError(f"Invalid code size {cs=} for {ps=} or " + f"code={self.code}") mid = self.code.encode() + mid # fully qualified mid with prefix code ml = len(memo) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 1eff190..cc2b18b 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -16,13 +16,17 @@ from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer, Keyage - def _setupKeep(salt=None): """Setup Keep for signed memos + Parameters: + salt(str|bytes): salt used to generate key pairs and vids for keep + Returns: keep (dict): labels are vids, values are keyage instances + + Ed25519_Seed: str = 'A' # Ed25519 256 bit random seed for private key """ try: import pysodium @@ -40,22 +44,52 @@ def _setupKeep(salt=None): keep = {} - sigseed0 = pysodium.crypto_pwhash(outlen=32, + sigseed = pysodium.crypto_pwhash(outlen=32, passwd="0", salt=salt, opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) # creates signing/verification key pair from seed - verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed0) # raw + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + + vid = Memoer._encodeVid(vid=verkey, code='B') + assert len(vid) == 44 + + keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw + keep[vid] = keyage + sigseed = pysodium.crypto_pwhash(outlen=32, + passwd="1", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + vid = Memoer._encodeVid(vid=verkey, code='D') + assert len(vid) == 44 - vid = "abcd" - keyage = Keyage(verkey=verkey, sigkey=sigkey) #raw + keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw keep[vid] = keyage + sigseed = pysodium.crypto_pwhash(outlen=32, + passwd="2", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + + dig = digest = blake3.blake3(verkey).digest() + vid = Memoer._encodeVid(vid=dig, code='E') + assert len(vid) == 44 + + keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw + keep[vid] = keyage return keep @@ -108,9 +142,53 @@ def test_memoer_class(): assert Memoer.MaxGramCount == (2**24-1) # absolute max gram count assert Memoer.BufSize == (2**16-1) # default buffersize + verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") + vidqb64 = Memoer._encodeVid(vid=verkey) + assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + cs, ms, vs, ss, ns, hs = Memoer.Sizes[SGDex.Signed] # cs ms vs ss ns hs + assert len(vidqb64) == 44 == vs """Done Test""" +def test_setup_keep(): + """Test _setupkeep utility function + """ + try: + keep = _setupKeep() # default salt + except MemoerError as ex: + return + + assert keep == \ + { + 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2': Keyage(verkey=b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v", + sigseed=b"\x9bF\n\xf1\xc2L\xeaBC:\xf7\xe9\xd71\xbc\xd2{\x7f\x81\xae5\x9c\xca\xf9\xdb\xac@`'\x0e\xa4\x10"), + 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH': Keyage(verkey=b'\x96\xf5gJG\xc7~\x8c\x08\xeb\x88\x1ddx\xc8\xfc_" qW8:P&\xa6\xbf\xc7\xc07\xc8\x07', + sigseed=b'|\x97\xe1\xffN\x07:\x8d`\xfef\xd5E8\xc2\xba\xad,\x96\x9e\xba\xbe\xdc\xe6IB\x01i\x92\x8c\xd3W'), + 'EGLDQ97VnSnJS19Dz0j3NcgASGjMgMm4R-DmyrPDRZtN': Keyage(verkey=b'\x0c\x16\x1ev\x1dXY\x1b\x9b\xfa0c\xbc\xab|7\x0eK=\xc2\x8a\x8dO\xb8\xc9=\x0c\r\xe5\xad\xc5Z', + sigseed=b'@L\xb3\x87\xff\x9e\xe0J\x14\xc5\xbf8\x8e\x83\x00\xd8\xc4\x0f\xff\xcc&\t\x9e|\x81Y\x06\x82\xd5\x07\xf3\x8c') + } + + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + + assert keep == \ + { + 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp': Keyage(verkey=b'\x96S\x1c\xd5\x97\xb1\xcb\x93\xf9#\xe8\x90\xfc\xd2x\x19\x12\x86\x91\xe3\xea\x0f\x1bB\xb3\xf4F\x1e\xc8$\xd5)', + sigseed=b'\xc3\xf7y\xe4\\\\VG\xb0\x9b\xcb\xaf\x83=\xcf\x13TD\x12\x85\xe50\xe3T\x94x\xb9\xed4Z\x14\x02'), + 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni': Keyage(verkey=b"c\x91\x04QI{\x96c\xe1=E@zQ\x15'\xb8\xd6\x14.\xab\xc1\xd4,\x03\x16\xc9\xcfB\xd3\xb9\xe2", + sigseed=b'\xec\xf5I\xcf\xa3|H\xe65!\x16\xb7O\xa0\xe3\xe7 \xb2\x17\x1d\x01\xf9/\x11\x19FJK\x9eG\xc7\xa4'), + 'EJ9RvVS6j6-stXpJEyaTi-MOJ1lZUQYkv9-Dc5GGHCVK': Keyage(verkey=b'L>zx\xc6s\xea;\xfdK7\xe0\x86N\xfaP\xdb\xa8Q\xb58!\x80\x83\xc7\xb5\xfe\x1dk_\xa7&', + sigseed=b'\xd9\x8f\xb7\xc9\xcd#\xd6 X\x07+*]\x7f\xa3\x99W\xbfSt-\xee\xea\x8e\xa4q\x9d\xdb\x88\xdf\xef ') + } + + """Done""" + + + + def test_memoer_basic(): """Test Memoer class basic """ @@ -1455,6 +1533,7 @@ def test_sure_memoer_doer(): if __name__ == "__main__": test_memoer_class() + test_setup_keep() test_memoer_basic() test_memoer_small_gram_size() test_memoer_multiple() From 18d50464b3dc93983d1455b4c2d2170caccf9586 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 17 Jan 2026 17:31:10 -0700 Subject: [PATCH 19/27] more support for signing of memo grams --- src/hio/core/memo/memoing.py | 46 +++++++--- tests/core/memo/test_memoing.py | 151 +++++++++++++++++++------------- 2 files changed, 123 insertions(+), 74 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index e666261..c0dd436 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -1223,13 +1223,27 @@ def sign(self, ser, vid): Parameters: ser (bytes): signed portion of gram is delivered format - vid (str | bytes): qualified base64 qb64 of verifier ID + vid (bytes): qb64 or qb2 if .curt of vid of signer + assumes vid of correct length """ - if hasattr(vid, 'encode'): - vid = vid.encode() + + if vid not in self.keep: + return b'' # return of empty signature should raise error in caller + + cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs - return b'A' * ss + + sig = b'A' * ss + + if self.curt: + hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 + ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 + vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 + ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + sig = decodeB64(sig) # make b2 + + return sig def rend(self, memo, vid=None): @@ -1251,9 +1265,10 @@ def rend(self, memo, vid=None): # self.size is max gram size cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + vid = vid if vid is not None else self.vid + if vs and (not vid or len(vid) != vs): - raise hioing.MemoerError(f"Invalid {vid=} for size={vs}.") - vid = b"" if vid is None else vid[:vs].encode() + raise hioing.MemoerError(f"Missing or invalid {vid=} for {vs=}") ps = (3 - ((ms) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned @@ -1270,7 +1285,8 @@ def rend(self, memo, vid=None): vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 mid = decodeB64(mid) - vid = decodeB64(vid) + + bs = (self.size - hs) # max standard gram body size without neck # compute gram count based on overhead note added neck overhead in first gram @@ -1299,7 +1315,14 @@ def rend(self, memo, vid=None): else: num = helping.intToB64b(gn, l=ns) # num size must always be neck size - head = mid + vid + num + if vs: # need vid part, but can't mod here may need below to sign + if self.curt: + vidp = decodeB64(vid.encode()) # vid part b2 + else: + vidp = vid.encode() # vid part b64 bytes + head = mid + vidp + num + else: + head = mid + num if gn == 0: gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end @@ -1308,10 +1331,11 @@ def rend(self, memo, vid=None): gram = head + memo[:bs] # copy slice past end just copies to end del memo[:bs] # del slice past end just deletes to end - if ss: # sign + if ss: # signed so sign sig = self.sign(gram, vid) - if self.curt: - sig = decodeB64(sig) + if not sig or len(sig) != ss: + raise hioing.MemoerError(f"Signed but unable to sign or " + f"invalid signature {sig=}") gram = gram + sig grams.append(gram) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index cc2b18b..400e291 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -707,7 +707,10 @@ def test_memoer_basic_signed(): except MemoerError as ex: return - peer = memoing.Memoer(code=GramDex.Signed) + vid = list(keep.keys())[0] + assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + peer = memoing.Memoer(code=GramDex.Signed, keep=keep, vid=vid) assert peer.name == "main" assert peer.opened == False assert peer.bc is None @@ -719,8 +722,8 @@ def test_memoer_basic_signed(): assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic - assert peer.keep == dict() - assert peer.vid is None + assert peer.keep == keep + assert peer.vid == vid peer.reopen() assert peer.opened == True @@ -733,7 +736,8 @@ def test_memoer_basic_signed(): memo = "Hello There" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + + vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid peer.memoit(memo, dst, vid) assert peer.txms[0] == ('Hello There', 'beta', vid) peer.serviceTxMemos() @@ -773,7 +777,7 @@ def test_memoer_basic_signed(): # send and receive via echo memo = "See ya later!" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid peer.memoit(memo, dst, vid) assert peer.txms[0] == ('See ya later!', 'beta', vid) peer.serviceTxMemos() @@ -812,7 +816,7 @@ def test_memoer_basic_signed(): peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid peer.memoit(memo, dst, vid) assert peer.txms[0] == ('Hello There', 'beta', vid) peer.serviceTxMemos() @@ -832,7 +836,7 @@ def test_memoer_basic_signed(): assert not peer.sources assert not peer.rxms mid = '_-ALBI68S1ZIxqwFOSWFF1L2' - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid sig = ('A' * 88) head = decodeB64((mid + vid + 'AAAA' + 'AAAB').encode()) tail = decodeB64(sig.encode()) @@ -872,7 +876,13 @@ def test_memoer_multiple_signed(): except MemoerError as ex: return - peer = memoing.Memoer(code=GramDex.Signed, size=170) + vid = list(keep.keys())[0] + assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + vidBeta = list(keep.keys())[1] + assert vidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' + + peer = memoing.Memoer(code=GramDex.Signed, size=170, keep=keep, vid=vid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False @@ -882,16 +892,15 @@ def test_memoer_multiple_signed(): assert not peer.curt assert not peer.verific assert not peer.echoic - assert peer.keep == dict() - assert peer.vid is None + assert peer.keep == keep + assert peer.vid == vid peer.reopen() assert peer.opened == True # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + peer.memoit("Hello there.", "alpha") # use default for vidAlpha + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -934,7 +943,7 @@ def test_memoer_multiple_signed(): assert not peer.sources assert len(peer.rxms) == 2 assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vid) + assert peer.rxms[1] == ('How ya doing?', 'beta', vidBeta) peer.serviceRxMemos() assert not peer.rxms @@ -944,10 +953,10 @@ def test_memoer_multiple_signed(): peer.size = 129 assert peer.size == 129 - # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + # send and receive multiple via echo in base2 .curt = True mode + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha") # use default for vidAlpha + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -990,16 +999,16 @@ def test_memoer_multiple_signed(): assert not peer.sources assert len(peer.rxms) == 2 assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vid) + assert peer.rxms[1] == ('How ya doing?', 'beta', vidBeta) peer.serviceRxMemos() assert not peer.rxms assert peer.inbox == deque( [ ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid), + ('How ya doing?', 'beta', vidBeta), ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid) + ('How ya doing?', 'beta', vidBeta) ]) @@ -1011,7 +1020,6 @@ def test_memoer_multiple_signed(): def test_memoer_verific(): """Test Memoer class with verific (signed required) """ - peer = memoing.Memoer(verific=True) assert peer.name == "main" assert peer.opened == False @@ -1086,9 +1094,16 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): except MemoerError as ex: return + vid = list(keep.keys())[0] + assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + vidBeta = list(keep.keys())[1] + assert vidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' + # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code - peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, echoic=True) + peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, + echoic=True, keep=keep, vid=vid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False @@ -1098,16 +1113,16 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert not peer.curt assert peer.verific assert peer.echoic - assert peer.keep == dict() - assert peer.vid is None + assert peer.keep == keep + assert peer.vid == vid peer.reopen() assert peer.opened == True # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.serviceAll() # servicAll services Rx first then Tx so have to repeat @@ -1131,10 +1146,9 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.inbox == deque( [ ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid) + ('How ya doing?', 'beta', vidBeta) ]) - # test in base2 mode peer.curt = True assert peer.curt @@ -1142,9 +1156,9 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.size == 129 # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.serviceAll() # servicAll services Rx first then Tx so have to repeat @@ -1168,9 +1182,9 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.inbox == deque( [ ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid), + ('How ya doing?', 'beta', vidBeta), ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid), + ('How ya doing?', 'beta', vidBeta), ]) peer.inbox = deque() # clear it @@ -1231,8 +1245,13 @@ def test_sure_memoer_basic(): except MemoerError as ex: return + vid = list(keep.keys())[0] + assert vid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + + vidBeta = list(keep.keys())[1] + assert vidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' - peer = memoing.SureMemoer(echoic=True) + peer = memoing.SureMemoer(echoic=True, keep=keep, vid=vid) assert peer.size == 65535 assert peer.name == "main" assert peer.opened == False @@ -1242,8 +1261,8 @@ def test_sure_memoer_basic(): assert not peer.curt assert peer.verific assert peer.echoic - assert peer.keep == dict() - assert peer.vid is None + assert peer.keep == keep + assert peer.vid == vid # (code, mid, vid, neck, head, sig) part sizes assert peer.Sizes[peer.code] == Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160) # cs ms vs ss ns hs @@ -1262,9 +1281,9 @@ def test_sure_memoer_basic(): memo = "Hello There" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit(memo, dst, vidBeta) + assert peer.txms[0] == ('Hello There', 'beta', vidBeta) peer.service() # services Rx then Tx so rx of tx not serviced until 2nd pass @@ -1288,14 +1307,14 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vid), + ('Hello There', 'beta', vidBeta), ]) # send and receive some more memo = "See ya later!" dst = "beta" - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('See ya later!', 'beta', vid) + peer.memoit(memo, dst, vidBeta) + assert peer.txms[0] == ('See ya later!', 'beta', vidBeta) peer.service() @@ -1319,16 +1338,16 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vid), - ('See ya later!', 'beta', vid), + ('Hello There', 'beta', vidBeta), + ('See ya later!', 'beta', vidBeta), ]) # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + peer.memoit(memo, dst, vidBeta) + assert peer.txms[0] == ('Hello There', 'beta', vidBeta) peer.service() @@ -1351,9 +1370,9 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vid), - ('See ya later!', 'beta', vid), - ('Hello There', 'beta', vid), + ('Hello There', 'beta', vidBeta), + ('See ya later!', 'beta', vidBeta), + ('Hello There', 'beta', vidBeta), ]) # Test wind @@ -1397,10 +1416,16 @@ def test_sure_memoer_multiple_echoic_service_all(): except MemoerError as ex: return + vid = list(keep.keys())[0] + assert vid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + + vidBeta = list(keep.keys())[1] + assert vidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' + # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code - with memoing.openSM(size=170, echoic=True) as peer: + with memoing.openSM(size=170, echoic=True, keep=keep, vid=vid) as peer: assert peer.size == 170 assert peer.name == "test" assert peer.opened == True @@ -1410,13 +1435,13 @@ def test_sure_memoer_multiple_echoic_service_all(): assert not peer.curt assert peer.verific assert peer.echoic - assert peer.keep == dict() - assert peer.vid is None + assert peer.keep == keep + assert peer.vid is vid # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.service() # services Rx first then Tx so have to repeat @@ -1441,7 +1466,7 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.inbox == deque( [ ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid) + ('How ya doing?', 'beta', vidBeta) ]) @@ -1452,9 +1477,9 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.size == 129 # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 peer.serviceAll() # servicAll services Rx first then Tx so have to repeat @@ -1478,9 +1503,9 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.inbox == deque( [ ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid), + ('How ya doing?', 'beta', vidBeta), ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vid), + ('How ya doing?', 'beta', vidBeta), ]) peer.inbox = deque() # clear it From d38f8cae4fc487d69976a1bc02baee48c1b87339 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 17 Jan 2026 21:04:15 -0700 Subject: [PATCH 20/27] refactor keep setup so that verkey and sigseed are CESR equiv fully qualified base64 not raw --- src/hio/core/memo/memoing.py | 107 +++++++++++++++++++++++++------- tests/core/memo/test_memoing.py | 54 ++++++++-------- 2 files changed, 112 insertions(+), 49 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index c0dd436..30b9143 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -32,12 +32,12 @@ Versionage = namedtuple("Versionage", "major minor") # signature key pair: -# verkey = public verifying key -# sigseed = private signing key seed (ed25519 sigkey = sigseed + verkey) -Keyage = namedtuple("Keyage", "verkey sigseed") +# qvk = qualified base64 public verifying key +# qss = qualified base64 private signing key seed (ed25519 sigkey = sigseed + verkey) +Keyage = namedtuple("Keyage", "qvk qss") # example usage -#keyage = Keyage(verkey="xyy", sigseed="abc", ) -#keep = dict("ABCXYZ"=keyage) # vid as label, Keyage instance as value +#keyage = Keyage(qvk="xyy", qss="abc", ) +#keep = dict("ABCXYZ"=keyage) # qualified vid as label, Keyage instance as value """Design Discusssion of Memo and Gram Sizing and Encoding: @@ -431,45 +431,108 @@ class Memoer(hioing.Mixin): BufSize = 65535 # (2**16-1) default buffersize @classmethod - def _encodeVid(cls, vid, code='B'): + def _encodeVID(cls, raw, code='B'): """Utility method for use with signed headers that encodes raw vid as CESR compatible fully qualified B64 text domain str using CESR compatible text code Parameters: - vid (bytes): raw vid to be encoded with code + raw (bytes): vid to be encoded with code code (str): code for type of vid CESR compatible Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. Ed25519: str = 'D' # Ed25519 verkey basic derivation Blake3_256: str = 'E' # Blake3 256 bit digest derivation. Returns: - - vidqb64 (str): fully qualified base64 vid - + qb64 (str): fully qualified base64 vid """ if code not in ('B', 'D', 'E'): - raise hioing.MemoerError("Invalid vid {code=}") + raise hioing.MemoerError(f"Invalid vid {code=}") + + rs = len(raw) # raw vid size + if rs != 32: + raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") + + ps = (3 - ((rs) % 3)) % 3 # net pad size for raw vid + if len(code) != ps != 1: + raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" + f" {ps=} not equal 1") + + b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified base64 vid with prefix code cs, ms, vs, ss, ns, hs = cls.Sizes[SGDex.Signed] # cs ms vs ss ns hs + if len(qb64) != vs: + hioing.MemoerError(f"Invalid vid qb64 size={len(qb64) != {vs}}") + + return qb64 # fully qualified base64 vid with prefix code + + + @classmethod + def _encodeQVK(cls, raw, code='B'): + """Utility method for use with signed headers that encodes raw vid as + CESR compatible fully qualified B64 text domain str using CESR compatible + text code + + Parameters: + raw (bytes): verkey to be encoded with code + code (str): code for type of vid CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + + Returns: + qb64 (str): fully qualified base64 verkey + """ + if code not in ('B'): + raise hioing.MemoerError(f"Invalid qvk {code=}") - rvs = len(vid) # raw vid size - if rvs != 32: - raise hioing.MemoerError("Invalid raw vid size {rvs=} not 32") + rs = len(raw) # raw size + if rs != 32: + raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") - ps = (3 - ((rvs) % 3)) % 3 # net pad size for raw vid + ps = (3 - ((rs) % 3)) % 3 # net pad size for raw verkey if len(code) != ps != 1: - raise hioing.MemoerError("Invalid vid code size={len(code)} not equal" - f" {ps=} not equal 1") + raise hioing.MemoerError(f"Invalid code size={len(code)} " + f"not equal {ps=} not equal 1") + + b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified verkey with prefix code + + return qb64 # qualified base64 verkey + + + @classmethod + def _encodeQSS(cls, raw, code='A'): + """Utility method for use with signed headers that encodes raw vid as + CESR compatible fully qualified B64 text domain str using CESR compatible + text code + + Parameters: + raw (bytes): sigseed to be encoded with code + code (str): code for type of vid CESR compatible + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key - vidb64 = encodeB64(bytes([0] * ps) + vid)[ps:] # prepad, convert, and prestrip + Returns: + qb64 (str): fully qualified base64 sigseed + """ + if code not in ('A'): + raise hioing.MemoerError(f"Invalid qss {code=}") + + rs = len(raw) # raw size + if rs != 32: + raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") + + ps = (3 - ((rs) % 3)) % 3 # net pad size for raw sigseed + if len(code) != ps != 1: + raise hioing.MemoerError(f"Invalid code size={len(code)} " + f"not equal {ps=} not equal 1") - vidqb64 = code + vidb64.decode() # fully qualified vid with prefix code + b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip - if len(vidqb64) != vs: - hioing.MemoerError("Invalid vid qb64 size={len(vidqb64) != {vs}}") + qb64 = code + b64.decode() # fully qualified with prefix code - return vidqb64 + return qb64 # qualified base64 sigseed def __init__(self, *, diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 400e291..2e11be1 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -53,10 +53,11 @@ def _setupKeep(salt=None): # creates signing/verification key pair from seed verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw - vid = Memoer._encodeVid(vid=verkey, code='B') - assert len(vid) == 44 - - keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw + # non transferable public key as vid + vid = Memoer._encodeVID(raw=verkey, code='B') # make fully qualified + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw keep[vid] = keyage sigseed = pysodium.crypto_pwhash(outlen=32, @@ -68,10 +69,10 @@ def _setupKeep(salt=None): # creates signing/verification key pair from seed verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw - vid = Memoer._encodeVid(vid=verkey, code='D') - assert len(vid) == 44 - - keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw + vid = Memoer._encodeVID(raw=verkey, code='D') # make fully qualified + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw keep[vid] = keyage sigseed = pysodium.crypto_pwhash(outlen=32, @@ -82,13 +83,12 @@ def _setupKeep(salt=None): alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) # creates signing/verification key pair from seed verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw - dig = digest = blake3.blake3(verkey).digest() - vid = Memoer._encodeVid(vid=dig, code='E') - assert len(vid) == 44 - - keyage = Keyage(verkey=verkey, sigseed=sigseed) # raw + vid = Memoer._encodeVID(raw=dig, code='E') + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw keep[vid] = keyage return keep @@ -143,7 +143,7 @@ def test_memoer_class(): assert Memoer.BufSize == (2**16-1) # default buffersize verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") - vidqb64 = Memoer._encodeVid(vid=verkey) + vidqb64 = Memoer._encodeVID(raw=verkey) assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' cs, ms, vs, ss, ns, hs = Memoer.Sizes[SGDex.Signed] # cs ms vs ss ns hs assert len(vidqb64) == 44 == vs @@ -160,14 +160,15 @@ def test_setup_keep(): assert keep == \ { - 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2': Keyage(verkey=b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v", - sigseed=b"\x9bF\n\xf1\xc2L\xeaBC:\xf7\xe9\xd71\xbc\xd2{\x7f\x81\xae5\x9c\xca\xf9\xdb\xac@`'\x0e\xa4\x10"), - 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH': Keyage(verkey=b'\x96\xf5gJG\xc7~\x8c\x08\xeb\x88\x1ddx\xc8\xfc_" qW8:P&\xa6\xbf\xc7\xc07\xc8\x07', - sigseed=b'|\x97\xe1\xffN\x07:\x8d`\xfef\xd5E8\xc2\xba\xad,\x96\x9e\xba\xbe\xdc\xe6IB\x01i\x92\x8c\xd3W'), - 'EGLDQ97VnSnJS19Dz0j3NcgASGjMgMm4R-DmyrPDRZtN': Keyage(verkey=b'\x0c\x16\x1ev\x1dXY\x1b\x9b\xfa0c\xbc\xab|7\x0eK=\xc2\x8a\x8dO\xb8\xc9=\x0c\r\xe5\xad\xc5Z', - sigseed=b'@L\xb3\x87\xff\x9e\xe0J\x14\xc5\xbf8\x8e\x83\x00\xd8\xc4\x0f\xff\xcc&\t\x9e|\x81Y\x06\x82\xd5\x07\xf3\x8c') + 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2': Keyage(qvk='BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2', + qss='AJtGCvHCTOpCQzr36dcxvNJ7f4GuNZzK-dusQGAnDqQQ'), + 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH': Keyage(qvk='BJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH', + qss='AHyX4f9OBzqNYP5m1UU4wrqtLJaeur7c5klCAWmSjNNX'), + 'EGLDQ97VnSnJS19Dz0j3NcgASGjMgMm4R-DmyrPDRZtN': Keyage(qvk='BAwWHnYdWFkbm_owY7yrfDcOSz3Cio1PuMk9DA3lrcVa', + qss='AEBMs4f_nuBKFMW_OI6DANjED__MJgmefIFZBoLVB_OM') } + salt = b"ABCDEFGHIJKLMNOP" try: keep = _setupKeep(salt=salt) @@ -176,14 +177,13 @@ def test_setup_keep(): assert keep == \ { - 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp': Keyage(verkey=b'\x96S\x1c\xd5\x97\xb1\xcb\x93\xf9#\xe8\x90\xfc\xd2x\x19\x12\x86\x91\xe3\xea\x0f\x1bB\xb3\xf4F\x1e\xc8$\xd5)', - sigseed=b'\xc3\xf7y\xe4\\\\VG\xb0\x9b\xcb\xaf\x83=\xcf\x13TD\x12\x85\xe50\xe3T\x94x\xb9\xed4Z\x14\x02'), - 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni': Keyage(verkey=b"c\x91\x04QI{\x96c\xe1=E@zQ\x15'\xb8\xd6\x14.\xab\xc1\xd4,\x03\x16\xc9\xcfB\xd3\xb9\xe2", - sigseed=b'\xec\xf5I\xcf\xa3|H\xe65!\x16\xb7O\xa0\xe3\xe7 \xb2\x17\x1d\x01\xf9/\x11\x19FJK\x9eG\xc7\xa4'), - 'EJ9RvVS6j6-stXpJEyaTi-MOJ1lZUQYkv9-Dc5GGHCVK': Keyage(verkey=b'L>zx\xc6s\xea;\xfdK7\xe0\x86N\xfaP\xdb\xa8Q\xb58!\x80\x83\xc7\xb5\xfe\x1dk_\xa7&', - sigseed=b'\xd9\x8f\xb7\xc9\xcd#\xd6 X\x07+*]\x7f\xa3\x99W\xbfSt-\xee\xea\x8e\xa4q\x9d\xdb\x88\xdf\xef ') + 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp': Keyage(qvk='BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp', + qss='AMP3eeRcXFZHsJvLr4M9zxNURBKF5TDjVJR4ue00WhQC'), + 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni': Keyage(qvk='BGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni', + qss='AOz1Sc-jfEjmNSEWt0-g4-cgshcdAfkvERlGSkueR8ek'), + 'EJ9RvVS6j6-stXpJEyaTi-MOJ1lZUQYkv9-Dc5GGHCVK': Keyage(qvk='BEw-enjGc-o7_Us34IZO-lDbqFG1OCGAg8e1_h1rX6cm', + qss='ANmPt8nNI9YgWAcrKl1_o5lXv1N0Le7qjqRxnduI3-8g') } - """Done""" From 470f729bb13f386ec660a2128078bd57799b4964 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 18 Jan 2026 09:29:41 -0700 Subject: [PATCH 21/27] refactor Sizage to us field names less likely to conflict or confuse part of changing vid to oid --- src/hio/core/memo/memoing.py | 198 ++++++++++++++-------------- tests/core/memo/test_memoing.py | 62 ++++----- tests/core/udp/test_peer_memoing.py | 8 +- tests/core/uxd/test_peer_memoing.py | 6 +- 4 files changed, 133 insertions(+), 141 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 30b9143..efef520 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -197,23 +197,23 @@ """ """Sizage: namedtuple for gram header part size entries in Memoer code tables -cs is the code part size int number of chars in code part (serialization code) -ms is the mid part size int number of chars in the mid part (memoID) -vs is the vid part size int number of chars in the vid part (verification ID) -ns is the neck part size int number of chars for the gram number in all grams +cz is the serialization code part size int number of chars in code part +mz is the mid (memo ID) part size int number of chars in the mid part +oz is the oid (origin ID) part size int number of chars in the oid part +nz is the neck part size int number of chars for the gram number in all grams it is also the size of the gram count that only appears in the zeroth gram - The zeroth gram has a long neck, two ns sized fields all other grams have a - short neck (1 ns sized field) -ss is the signature part size int number of chars in the signature part. + The zeroth gram has a long neck consiting of two nz sized fields + All other grams have ashort neck consisting of 1 nz sized field +sz is the signature part size int number of chars in the signature part the signature part is discontinuously attached after the body part but its size is included in the overhead size computation for the body size -hs is the head size int number of chars - short next - hs = cs + ms + vs + ns + ss - zeroth long neck - hs = cs + ms + vs + ns + ns + ss +hz is the head (header) size int number of chars + head with short neck + hz = cz + mz + oz + nz + sz + zeroth head with long neck is calculated from short neck head + zhz = hz + nz """ -Sizage = namedtuple("Sizage", "cs ms vs ss ns hs") +Sizage = namedtuple("Sizage", "cz mz oz nz sz hz") @dataclass(frozen=True) class GramCodex: @@ -418,8 +418,8 @@ class Memoer(hioing.Mixin): Sodex = SGDex # signed gram codex # dict of gram header part sizes keyed by gram codes: cs ms vs ss ns hs Sizes = { - '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), - '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), + '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), + '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), } # Base2 Binary index representation of Text Base64 Char Codes @@ -449,22 +449,22 @@ def _encodeVID(cls, raw, code='B'): if code not in ('B', 'D', 'E'): raise hioing.MemoerError(f"Invalid vid {code=}") - rs = len(raw) # raw vid size - if rs != 32: - raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") + rz = len(raw) # raw vid size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") - ps = (3 - ((rs) % 3)) % 3 # net pad size for raw vid - if len(code) != ps != 1: + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw vid + if len(code) != pz != 1: raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" - f" {ps=} not equal 1") + f" {pz=} not equal 1") - b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip qb64 = code + b64.decode() # fully qualified base64 vid with prefix code - cs, ms, vs, ss, ns, hs = cls.Sizes[SGDex.Signed] # cs ms vs ss ns hs - if len(qb64) != vs: - hioing.MemoerError(f"Invalid vid qb64 size={len(qb64) != {vs}}") + _, _, oz, _, _, _ = cls.Sizes[SGDex.Signed] # cz mz oz nz sz hz + if len(qb64) != oz: + hioing.MemoerError(f"Invalid vid qb64 size={len(qb64) != {oz}}") return qb64 # fully qualified base64 vid with prefix code @@ -745,14 +745,14 @@ def size(self, size): size (int | None): gram size for rending memo """ - _, _, _, _, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + _, _, _, nz, _, hz = self.Sizes[self.code] # cz mz oz nz sz hz size = size if size is not None else self.MaxGramSize if self.curt: # minimum header smaller when in base2 curt - hs = 3 * hs // 4 - ns = 3 * ns // 4 + hz = 3 * hz // 4 + nz = 3 * nz // 4 # mininum size must be big enough for first gram header and 1 body byte - self._size = max(min(size, self.MaxGramSize), hs + ns + 1) + self._size = max(min(size, self.MaxGramSize), hz + nz + 1) @property @@ -944,37 +944,37 @@ def pick(self, gram): raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - cms = 3 * (cs + ms) // 4 # cs + ms are aligned on 24 bit boundary - hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid + cms = 3 * (cz + mz) // 4 # cz + mz are aligned on 24 bit boundary + hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 + sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 - if len(gram) < (hs + 1): # not big enough for non-first gram + if len(gram) < (hz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" - f" < {hs + 1}.") + f" < {hz + 1}.") mid = encodeB64(gram[:cms]).decode() # fully qualified with prefix code - vid = encodeB64(gram[cms:cms+vs]).decode() # must be on 24 bit boundary - gn = int.from_bytes(gram[cms+vs:cms+vs+ns]) + vid = encodeB64(gram[cms:cms+oz]).decode() # must be on 24 bit boundary + gn = int.from_bytes(gram[cms+oz:cms+oz+nz]) if gn == 0: # first (zeroth) gram so long neck - if len(gram) < hs + ns + 1: + if len(gram) < hz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b2 " - f"gram < {hs + ns + 1}.") - neck = gram[cms+vs+ns:cms+vs+2*ns] # slice takes a copy + f"gram < {hz + nz + 1}.") + neck = gram[cms+oz+nz:cms+oz+2*nz] # slice takes a copy gc = int.from_bytes(neck) # convert to int - sig = encodeB64(gram[-ss if ss else len(gram):]) # last ss bytes are signature - del gram[-ss if ss else len(gram):] # strip sig + sig = encodeB64(gram[-sz if sz else len(gram):]) # last ss bytes are signature + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss+ns] # strip of fore head leaving body in gram + del gram[:hz-sz+nz] # strip of fore head leaving body in gram else: # non-zeroth gram so short neck gc = None - sig = encodeB64(gram[-ss if ss else len(gram):]) - del gram[-ss if ss else len(gram):] # strip sig + sig = encodeB64(gram[-sz if sz else len(gram):]) + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss] # strip of fore head leaving body in gram + del gram[:hz-sz] # strip of fore head leaving body in gram else: # base64 text encoding if len(gram) < 2: # assumes len(code) must be 2 @@ -984,31 +984,31 @@ def pick(self, gram): if self.verific and code not in self.Sodex: # must be signed raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs + cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz - if len(gram) < (hs + 1): # not big enough for non-first gram + if len(gram) < (hz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b64 gram" - f" < {hs + 1}.") + f" < {hz + 1}.") - mid = bytes(gram[:cs+ms]).decode() # fully qualified with prefix code - vid = bytes(gram[cs+ms:cs+ms+vs]).decode() # must be on 24 bit boundary - gn = helping.b64ToInt(gram[cs+ms+vs:cs+ms+vs+ns]) + mid = bytes(gram[:cz+mz]).decode() # fully qualified with prefix code + vid = bytes(gram[cz+mz:cz+mz+oz]).decode() # must be on 24 bit boundary + gn = helping.b64ToInt(gram[cz+mz+oz:cz+mz+oz+nz]) if gn == 0: # first (zeroth) gram so long neck - if len(gram) < hs + ns + 1: + if len(gram) < hz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b64 " - f"gram < {hs + ns + 1}.") - neck = gram[cs+ms+vs+ns:cs+ms+vs+2*ns] # slice takes a copy + f"gram < {hz + nz + 1}.") + neck = gram[cz+mz+oz+nz:cz+mz+oz+2*nz] # slice takes a copy gc = helping.b64ToInt(neck) # convert to int - sig = bytes(gram[-ss if ss else len(gram):]) # last ss bytes are signature - del gram[-ss if ss else len(gram):] # strip sig + sig = bytes(gram[-sz if sz else len(gram):]) # last sz bytes signature + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss+ns] # strip of fore head leaving body in gram + del gram[:hz-sz+nz] # strip of fore head leaving body in gram else: # non-zeroth gram short neck gc = None - sig = bytes(gram[-ss if ss else len(gram):]) - del gram[-ss if ss else len(gram):] # strip sig + sig = bytes(gram[-sz if sz else len(gram):]) + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss] # strip of fore head leaving body in gram + del gram[:hz-sz] # strip of fore head leaving body in gram if sig: # signature not empty if not self.verify(sig, signed, vid): @@ -1295,15 +1295,15 @@ def sign(self, ser, vid): return b'' # return of empty signature should raise error in caller - cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + _, _, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz - sig = b'A' * ss + sig = b'A' * sz if self.curt: - hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 + sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 sig = decodeB64(sig) # make b2 return sig @@ -1320,38 +1320,38 @@ def rend(self, memo, vid=None): vid (str | None): verification ID when gram is to be signed. None means not signable - Note first gram has head + neck overhead, hs + ns so bs is smaller by ns - non-first grams have just head overhead hs so bs is bigger by ns + Note zeroth gram has head + neck overhead, zhz = hz + nz + so bz that fits is smaller by nz relative to non-zeroth + non-zeroth grams just head overhead hz + so bz that fits is bigger by nz relative to zeroth """ grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray # self.size is max gram size - cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + cz, mz, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz vid = vid if vid is not None else self.vid - if vs and (not vid or len(vid) != vs): - raise hioing.MemoerError(f"Missing or invalid {vid=} for {vs=}") + if oz and (not vid or len(vid) != oz): + raise hioing.MemoerError(f"Missing or invalid {vid=} for {oz=}") - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned - mid = encodeB64(bytes([0] * ps) + uuid.uuid1().bytes)[ps:] # prepad, convert, and prestrip - if cs != ps or cs != len(self.code): - raise hioing.MemoerError(f"Invalid code size {cs=} for {ps=} or " + mid = encodeB64(bytes([0] * pz) + uuid.uuid1().bytes)[pz:] #pzrepad, convert, and prestrip + if cz != pz or cz != len(self.code): + raise hioing.MemoerError(f"Invalid code size {cz=} for {pz=} or " f"code={self.code}") mid = self.code.encode() + mid # fully qualified mid with prefix code ml = len(memo) if self.curt: # rend header parts in base2 instead of base64 - hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 + sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 mid = decodeB64(mid) - - - bs = (self.size - hs) # max standard gram body size without neck + bz = (self.size - hz) # max standard gram body size without neck # compute gram count based on overhead note added neck overhead in first gram # first gram is special its header is longer by ns than the other grams # which means its payload body is shorter by ns than the other gram bodies @@ -1360,25 +1360,25 @@ def rend(self, memo, vid=None): # (ceil) to get cnt of other grams and add 1 for the first gram to get # total gram cnt. # gc = ceil((ml-(bs-ns))/bs + 1) = ceil((ml-bs+ns)/bs + 1) - gc = math.ceil((ml-bs+ns)/bs+1) # includes added neck ns overhead - mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload + gc = math.ceil((ml-bz+nz)/bz+1) # includes added neck ns overhead + mms = min(self.MaxMemoSize, (bz * self.MaxGramCount) - nz) # max memo payload if ml > mms: raise hioing.MemoerError(f"Memo length={ml} exceeds max={mms}.") if self.curt: - neck = gc.to_bytes(ns) + neck = gc.to_bytes(nz) else: - neck = helping.intToB64b(gc, l=ns) + neck = helping.intToB64b(gc, l=nz) gn = 0 while memo: if self.curt: - num = gn.to_bytes(ns) # num size must always be neck size + num = gn.to_bytes(nz) # num size must always be neck size else: - num = helping.intToB64b(gn, l=ns) # num size must always be neck size + num = helping.intToB64b(gn, l=nz) # num size must always be neck size - if vs: # need vid part, but can't mod here may need below to sign + if oz: # need vid part, but can't mod here may need below to sign if self.curt: vidp = decodeB64(vid.encode()) # vid part b2 else: @@ -1388,15 +1388,15 @@ def rend(self, memo, vid=None): head = mid + num if gn == 0: - gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end - del memo[:bs-ns] # del slice past end just deletes to end + gram = head + neck + memo[:bz-nz] # copy slice past end just copies to end + del memo[:bz-nz] # del slice past end just deletes to end else: - gram = head + memo[:bs] # copy slice past end just copies to end - del memo[:bs] # del slice past end just deletes to end + gram = head + memo[:bz] # copy slice past end just copies to end + del memo[:bz] # del slice past end just deletes to end - if ss: # signed so sign + if sz: # signed so sign sig = self.sign(gram, vid) - if not sig or len(sig) != ss: + if not sig or len(sig) != sz: raise hioing.MemoerError(f"Signed but unable to sign or " f"invalid signature {sig=}") gram = gram + sig diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 2e11be1..55daa27 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -104,32 +104,32 @@ def test_memoer_class(): # Codes table with sizes of code (hard) and full primitive material assert Memoer.Sizes == \ { - '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), - '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), + '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), + '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), } # verify Sizes and Codes for code, val in Memoer.Sizes.items(): - cs = val.cs - ms = val.ms - vs = val.vs - ss = val.ss - ns = val.ns - hs = val.hs - - assert len(code) == cs == 2 + cz = val.cz + mz = val.mz + oz = val.oz + nz = val.nz + sz = val.sz + hz = val.hz + + assert len(code) == cz == 2 assert code[0] == '_' code[1] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_' - assert hs > 0 - assert hs == cs + ms + vs + ss + ns - assert ms # ms must not be empty - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - assert ps == (cs % 4) # combined code + mid size must lie on 24 bit boundary - assert not vs % 4 # vid size must be on 24 bit boundary - assert not ss % 4 # sig size must be on 24 bit boundary - assert ns and not ns % 4 # neck (num or cnt) size must be on 24 bit boundary - assert hs and not hs % 4 # head size must be on 24 bit boundary - if vs: - assert ss # ss must not be empty if vs not empty + assert hz > 0 + assert hz == cz + mz + oz + sz + nz + assert mz # ms must not be empty + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid + assert pz == (cz % 4) # combined code + mid size must lie on 24 bit boundary + assert not oz % 4 # vid size must be on 24 bit boundary + assert not sz % 4 # sig size must be on 24 bit boundary + assert nz and not nz % 4 # neck (num or cnt) size must be on 24 bit boundary + assert hz and not hz % 4 # head size must be on 24 bit boundary + if oz: + assert sz # ss must not be empty if vs not empty assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} assert Memoer.Sodex == SGDex @@ -145,8 +145,8 @@ def test_memoer_class(): verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") vidqb64 = Memoer._encodeVID(raw=verkey) assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' - cs, ms, vs, ss, ns, hs = Memoer.Sizes[SGDex.Signed] # cs ms vs ss ns hs - assert len(vidqb64) == 44 == vs + _, _, oz, _, _, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz + assert len(vidqb64) == 44 == oz """Done Test""" @@ -199,8 +199,7 @@ def test_memoer_basic(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic @@ -350,8 +349,7 @@ def test_memoer_small_gram_size(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == 33 # can't be smaller than head + neck + 1 assert not peer.verific assert not peer.echoic @@ -717,8 +715,7 @@ def test_memoer_basic_signed(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 44, 88, 4, 160) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic @@ -1027,8 +1024,7 @@ def test_memoer_verific(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert peer.verific assert not peer.echoic @@ -1264,8 +1260,8 @@ def test_sure_memoer_basic(): assert peer.keep == keep assert peer.vid == vid - # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160) + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert peer.tymeout == 0.0 assert peer.tymers == {} diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index 37271f3..787be75 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -25,8 +25,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 1240 # default MaxGramSize for udp assert alpha.bc == 1024 assert not alpha.opened @@ -39,8 +38,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == size assert alpha.bc == 1024 assert not alpha.opened @@ -174,7 +172,7 @@ def test_memoer_peer_open(): assert alpha.code == GramDex.Basic assert not alpha.curt # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == size assert alpha.bc == 1024 diff --git a/tests/core/uxd/test_peer_memoing.py b/tests/core/uxd/test_peer_memoing.py index 13ce8af..1cf4dfd 100644 --- a/tests/core/uxd/test_peer_memoing.py +++ b/tests/core/uxd/test_peer_memoing.py @@ -22,8 +22,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 assert alpha.bc == 4 @@ -152,8 +151,7 @@ def test_memoer_peer_open(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 assert alpha.bc == 4 From ac4d56bfccb144ff304489cf2eee0788ea8fb8d8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 18 Jan 2026 10:08:23 -0700 Subject: [PATCH 22/27] refactored Memoer to use oid not vid for origin id (instead of verification id) this will avoid any future confusion with SPAC/TSP use of the term vid. These are not the same since Memoer is transport for SPAC/TSP --- src/hio/base/multidoing.py | 4 +- src/hio/core/memo/memoing.py | 256 ++++++++++++++-------------- src/hio/core/udp/peermemoing.py | 18 +- tests/core/memo/test_memoing.py | 206 +++++++++++----------- tests/core/udp/test_peer_memoing.py | 1 - 5 files changed, 242 insertions(+), 243 deletions(-) diff --git a/src/hio/base/multidoing.py b/src/hio/base/multidoing.py index 0863583..f96e568 100644 --- a/src/hio/base/multidoing.py +++ b/src/hio/base/multidoing.py @@ -568,7 +568,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + memo, src, oid = self._serviceOneRxMemo() self.logger.debug("Boss Peer RX: name=%s rx from src=%s memo=%s.", self.name, src, memo) @@ -797,7 +797,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + memo, src, oid = self._serviceOneRxMemo() self.logger.debug("Hand Peer RX: name=%s rx from src=%s memo=%s.", self.name, src, memo) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index efef520..e7d5cfa 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -37,7 +37,7 @@ Keyage = namedtuple("Keyage", "qvk qss") # example usage #keyage = Keyage(qvk="xyy", qss="abc", ) -#keep = dict("ABCXYZ"=keyage) # qualified vid as label, Keyage instance as value +#keep = dict("ABCXYZ"=keyage) # qualified oid as label, Keyage instance as value """Design Discusssion of Memo and Gram Sizing and Encoding: @@ -287,7 +287,8 @@ class Memoer(hioing.Mixin): and placed in the memo deque for consumption by the application or some other higher level protocol. - receive -> (gram, src) -> grams parsed to .rxgs .counts .vids .sources -# singing key pair sigkey and verifier key> + receive -> (gram, src) -> grams parsed to .rxgs .counts .oids .sources -> + signing key pair sigkey and verifier key -> fuse -> memo .rxms deque When using non-blocking IO, asynchronous datagram transport @@ -349,20 +350,20 @@ class Memoer(hioing.Mixin): counts (dict): keyed by mid (memoID) that holds the gram count from the first gram for the memo. This enables lookup of the gram count when fusing its grams. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + the oid to memo when placing fused memo in rxms deque. + oid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is origin id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: @@ -395,12 +396,12 @@ class Memoer(hioing.Mixin): Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter - keep (dict): labels or vids, values are Keyage instances + keep (dict): labels are oids, values are Keyage instances named tuple of signature key pair: sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") - vid (str|None): own vid defaults used to lookup keys to sign on tx + oid (str|None): own oid defaults used to lookup keys to sign on tx Hidden: _code (bytes | None): see size property @@ -409,14 +410,14 @@ class Memoer(hioing.Mixin): _verific (bool): see verific property _echoic (bool): see echoic property _keep (dict): see keep property - _vid (str|None): see vid property + _oid (str|None): see oid property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex Codes = asdict(Codex) # map code name to code Names = {val : key for key, val in Codes.items()} # invert map code to code name Sodex = SGDex # signed gram codex - # dict of gram header part sizes keyed by gram codes: cs ms vs ss ns hs + # dict of gram header part sizes keyed by gram codes: cz mz oz nz sz hz Sizes = { '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), @@ -431,53 +432,53 @@ class Memoer(hioing.Mixin): BufSize = 65535 # (2**16-1) default buffersize @classmethod - def _encodeVID(cls, raw, code='B'): - """Utility method for use with signed headers that encodes raw vid as - CESR compatible fully qualified B64 text domain str using CESR compatible - text code + def _encodeOID(cls, raw, code='B'): + """Utility method for use with signed headers that encodes raw oid + (origin ID) as CESR compatible fully qualified B64 text domain str + using CESR compatible text code Parameters: - raw (bytes): vid to be encoded with code - code (str): code for type of vid CESR compatible + raw (bytes): oid to be encoded with code + code (str): code for type of raw oid CESR compatible Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. Ed25519: str = 'D' # Ed25519 verkey basic derivation Blake3_256: str = 'E' # Blake3 256 bit digest derivation. Returns: - qb64 (str): fully qualified base64 vid + qb64 (str): fully qualified base64 oid """ if code not in ('B', 'D', 'E'): - raise hioing.MemoerError(f"Invalid vid {code=}") + raise hioing.MemoerError(f"Invalid oid {code=}") - rz = len(raw) # raw vid size + rz = len(raw) # raw size if rz != 32: raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") - pz = (3 - ((rz) % 3)) % 3 # net pad size for raw vid + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw oid if len(code) != pz != 1: raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" f" {pz=} not equal 1") b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip - qb64 = code + b64.decode() # fully qualified base64 vid with prefix code + qb64 = code + b64.decode() # fully qualified base64 oid with prefix code _, _, oz, _, _, _ = cls.Sizes[SGDex.Signed] # cz mz oz nz sz hz if len(qb64) != oz: - hioing.MemoerError(f"Invalid vid qb64 size={len(qb64) != {oz}}") + hioing.MemoerError(f"Invalid oid qb64 size={len(qb64) != {oz}}") - return qb64 # fully qualified base64 vid with prefix code + return qb64 # fully qualified base64 oid with prefix code @classmethod def _encodeQVK(cls, raw, code='B'): - """Utility method for use with signed headers that encodes raw vid as + """Utility method for use with signed headers that encodes raw verkey as CESR compatible fully qualified B64 text domain str using CESR compatible text code Parameters: raw (bytes): verkey to be encoded with code - code (str): code for type of vid CESR compatible + code (str): code for type of raw verkey CESR compatible Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. Returns: @@ -486,16 +487,16 @@ def _encodeQVK(cls, raw, code='B'): if code not in ('B'): raise hioing.MemoerError(f"Invalid qvk {code=}") - rs = len(raw) # raw size - if rs != 32: - raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") + rz = len(raw) # raw size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") - ps = (3 - ((rs) % 3)) % 3 # net pad size for raw verkey - if len(code) != ps != 1: + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw verkey + if len(code) != pz != 1: raise hioing.MemoerError(f"Invalid code size={len(code)} " - f"not equal {ps=} not equal 1") + f"not equal {pz=} not equal 1") - b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip qb64 = code + b64.decode() # fully qualified verkey with prefix code @@ -504,13 +505,13 @@ def _encodeQVK(cls, raw, code='B'): @classmethod def _encodeQSS(cls, raw, code='A'): - """Utility method for use with signed headers that encodes raw vid as + """Utility method for use with signed headers that encodes raw sigseed as CESR compatible fully qualified B64 text domain str using CESR compatible text code Parameters: raw (bytes): sigseed to be encoded with code - code (str): code for type of vid CESR compatible + code (str): code for type of raw sigseed CESR compatible Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key Returns: @@ -519,16 +520,16 @@ def _encodeQSS(cls, raw, code='A'): if code not in ('A'): raise hioing.MemoerError(f"Invalid qss {code=}") - rs = len(raw) # raw size - if rs != 32: - raise hioing.MemoerError(f"Invalid raw size {rs=} not 32") + rz = len(raw) # raw size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") - ps = (3 - ((rs) % 3)) % 3 # net pad size for raw sigseed - if len(code) != ps != 1: + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw sigseed + if len(code) != pz != 1: raise hioing.MemoerError(f"Invalid code size={len(code)} " - f"not equal {ps=} not equal 1") + f"not equal {pz=} not equal 1") - b64 = encodeB64(bytes([0] * ps) + raw)[ps:] # prepad, convert, and prestrip + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip qb64 = code + b64.decode() # fully qualified with prefix code @@ -543,7 +544,7 @@ def __init__(self, *, rxgs=None, sources=None, counts=None, - vids=None, + oids=None, rxms=None, txms=None, txgs=None, @@ -554,7 +555,7 @@ def __init__(self, *, verific=False, echoic=False, keep=None, - vid=None, + oid=None, **kwa ): """Setup instance @@ -584,20 +585,21 @@ def __init__(self, *, memo indexed by its mid (memoID). This enables lookup of the gram count for a given memo to know when it has received all its constituent grams in order to fuse back into the memo. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for - the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID + str for the memo indexed by its mid (memoID). + This enables reattaching the oid to memo when placing fused memo + in rxms deque. + oid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is origin id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). Grams include gram headers. txbs (tuple): current transmisstion duple of form: @@ -624,11 +626,11 @@ def __init__(self, *, Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter - keep (dict|None): labels are vids and values are Keyage instances - that provide current signature key pair for vid + keep (dict|None): labels are oids and values are Keyage instances + that provide current signature key pair for oid this is a lightweight mechanism that should be overridden in subclass for real world key management. - vid (str|None): own vid defaults used to lookup keys to sign on tx + oid (str|None): own oid defaults used to lookup keys to sign on tx """ @@ -637,7 +639,7 @@ def __init__(self, *, self.rxgs = rxgs if rxgs is not None else dict() self.sources = sources if sources is not None else dict() self.counts = counts if counts is not None else dict() - self.vids = vids if vids is not None else dict() + self.oids = oids if oids is not None else dict() self.rxms = rxms if rxms is not None else deque() self.txms = txms if txms is not None else deque() @@ -671,7 +673,7 @@ def __init__(self, *, self._echoic = True if echoic else False self._keep = keep if keep is not None else dict() - self.vid = vid if vid else None + self.oid = oid if oid else None @property def code(self): @@ -785,7 +787,7 @@ def keep(self): """Property getter for ._keep Returns: - keep (dict): labels or vids, values are Keyage instances + keep (dict): labels are oids, values are Keyage instances named tuple of signature key pair: sigkey = private signing key verkey = public verifying key @@ -794,23 +796,23 @@ def keep(self): return self._keep @property - def vid(self): - """Property getter for ._vid + def oid(self): + """Property getter for ._oid Returns: - vid (str|None): vid used to sign on tx if any + oid (str|None): oid used to sign on tx if any """ - return self._vid + return self._oid - @vid.setter - def vid(self, vid): - """Property setter for ._vid + @oid.setter + def oid(self, oid): + """Property setter for ._oid Parameters: - vid (str|None): value to assign to own vid + oid (str|None): value to assign to own oid """ - self._vid = vid + self._oid = oid def open(self): @@ -887,8 +889,9 @@ def wiff(self, gram): raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") - def verify(self, sig, ser, vid): - """Verify signature sig on signed part of gram, ser, using key from vid. + def verify(self, sig, ser, oid): + """Verify signature sig on signed part of gram, ser, using current verkey + for oid. Must be overriden in subclass to perform real signature verification. This is a stub. @@ -899,13 +902,13 @@ def verify(self, sig, ser, vid): Parameters: sig (bytes | str): qualified base64 qb64b ser (bytes): signed portion of gram in delivered format - vid (bytes | str): qualified base64 qb64b of verifier ID + oid (bytes | str): qualified base64 qb64b of origin ID """ if hasattr(sig, 'encode'): sig = sig.encode() - if hasattr(vid, 'encode'): - vid = vid.encode() + if hasattr(oid, 'encode'): + oid = oid.encode() return True @@ -916,14 +919,14 @@ def pick(self, gram): Returns: result (tuple): tuple of form: - (mid: str, vid: str, gn: int, gc: int | None) where: + (mid: str, oid: str, gn: int, gc: int | None) where: mid is fully qualified memoID, - vid is verID, + oid is origin ID used to look up signature verification key, gn is gram number, gc is gram count. - When first gram (zeroth) returns (mid, vid, 0, gc). - When other gram returns (mid, vid, gn, None) - When code has empty vid then vid is None + When first gram (zeroth) returns (mid, oid, 0, gc). + When other gram returns (mid, oid, gn, None) + When code has empty oid then oid is None Otherwise raises MemoerError error. When valid recognized header, strips header bytes from front of gram @@ -957,7 +960,7 @@ def pick(self, gram): f" < {hz + 1}.") mid = encodeB64(gram[:cms]).decode() # fully qualified with prefix code - vid = encodeB64(gram[cms:cms+oz]).decode() # must be on 24 bit boundary + oid = encodeB64(gram[cms:cms+oz]).decode() # must be on 24 bit boundary gn = int.from_bytes(gram[cms+oz:cms+oz+nz]) if gn == 0: # first (zeroth) gram so long neck if len(gram) < hz + nz + 1: @@ -991,7 +994,7 @@ def pick(self, gram): f" < {hz + 1}.") mid = bytes(gram[:cz+mz]).decode() # fully qualified with prefix code - vid = bytes(gram[cz+mz:cz+mz+oz]).decode() # must be on 24 bit boundary + oid = bytes(gram[cz+mz:cz+mz+oz]).decode() # must be on 24 bit boundary gn = helping.b64ToInt(gram[cz+mz+oz:cz+mz+oz+nz]) if gn == 0: # first (zeroth) gram so long neck if len(gram) < hz + nz + 1: @@ -1010,12 +1013,12 @@ def pick(self, gram): signed = bytes(gram[:]) # copy signed portion of gram del gram[:hz-sz] # strip of fore head leaving body in gram - if sig: # signature not empty - if not self.verify(sig, signed, vid): + if sig: # signature not emptyoid + if not self.verify(sig, signed, oid): raise hioing.MemoerError(f"Invalid signature on gram from " - f"verifier {vid=}.") + f"verifier {oid=}.") - return (mid, vid if vid else None, gn, gc) + return (mid, oid if oid else None, gn, gc) def receive(self, *, echoic=False) -> (bytes, str|tuple|None): @@ -1100,7 +1103,7 @@ def _serviceOneReceived(self, *, echoic=False): gram = bytearray(gram) # make copy bytearray so can strip off header try: - mid, vid, gn, gc = self.pick(gram) # parse and strip off head leaving body + mid, oid, gn, gc = self.pick(gram) # parse and strip off head leaving body except hioing.MemoerError as ex: # invalid gram so drop logger.error("Unrecognized Memoer gram from %s.\n %s.", src, ex) return True # did receive data so can try again now @@ -1116,8 +1119,8 @@ def _serviceOneReceived(self, *, echoic=False): if mid not in self.counts: # make idempotent first only no replay self.counts[mid] = gc # save gram count for mid - if mid not in self.vids: - self.vids[mid] = vid + if mid not in self.oids: + self.oids[mid] = oid # assumes unique mid across all possible sources. No replay by different # source only first source for a given mid is ever recognized @@ -1192,11 +1195,11 @@ def _serviceOnceRxGrams(self): continue memo = self.fuse(self.rxgs[mid], self.counts[mid]) if memo is not None: # allows for empty "" memo for some src - self.rxms.append((memo, self.sources[mid], self.vids[mid])) + self.rxms.append((memo, self.sources[mid], self.oids[mid])) del self.rxgs[mid] del self.counts[mid] del self.sources[mid] - del self.vids[mid] + del self.oids[mid] def serviceRxGramsOnce(self): @@ -1233,7 +1236,7 @@ def serviceRxMemosOnce(self): Override in subclass to handle result and put it somewhere """ try: - #memo, src, vid = self._serviceOneRxMemo() + #memo, src, oid = self._serviceOneRxMemo() self.inbox.append(self._serviceOneRxMemo()) except IndexError: pass @@ -1245,7 +1248,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - #memo, src, vid = self._serviceOneRxMemo() + #memo, src, oid = self._serviceOneRxMemo() self.inbox.append(self._serviceOneRxMemo()) @@ -1265,20 +1268,20 @@ def serviceAllRx(self): self.serviceRxMemos() - def memoit(self, memo, dst, vid=None): - """Append (memo, dst, vid) tuple to .txms deque + def memoit(self, memo, dst, oid=None): + """Append (memo, dst, oid) tuple to .txms deque Parameters: memo (str): to be segmented and packed into gram(s) dst (str): address of remote destination of memo - vid (str | None): verifiable ID for signing grams + oid (str | None): origin ID for verifying signature on grams """ - self.txms.append((memo, dst, vid)) + self.txms.append((memo, dst, oid)) - def sign(self, ser, vid): - """Sign serialization ser using private key for verifier ID vid - Must be overriden in subclass to fetch private key for vid and sign. + def sign(self, ser, oid): + """Sign serialization ser using private sigkey for origin ID oid + Must be overriden in subclass to fetch private key for oid and sign. This is a stub. Returns: @@ -1286,12 +1289,12 @@ def sign(self, ser, vid): Parameters: ser (bytes): signed portion of gram is delivered format - vid (bytes): qb64 or qb2 if .curt of vid of signer - assumes vid of correct length + oid (bytes): qb64 or qb2 if .curt of oid of signer + assumes oid of correct length """ - if vid not in self.keep: + if oid not in self.keep: return b'' # return of empty signature should raise error in caller @@ -1309,7 +1312,7 @@ def sign(self, ser, vid): return sig - def rend(self, memo, vid=None): + def rend(self, memo, oid=None): """Partition memo into packed grams with headers. Returns: @@ -1317,7 +1320,8 @@ def rend(self, memo, vid=None): Parameters: memo (str): to be partitioned into grams with headers - vid (str | None): verification ID when gram is to be signed. + oid (str | None): origin ID when gram is to be signed, used to + lookup sigkey to sign. None means not signable Note zeroth gram has head + neck overhead, zhz = hz + nz @@ -1330,10 +1334,10 @@ def rend(self, memo, vid=None): # self.size is max gram size cz, mz, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz - vid = vid if vid is not None else self.vid + oid = oid if oid is not None else self.oid - if oz and (not vid or len(vid) != oz): - raise hioing.MemoerError(f"Missing or invalid {vid=} for {oz=}") + if oz and (not oid or len(oid) != oz): + raise hioing.MemoerError(f"Missing or invalid {oid=} for {oz=}") pz = (3 - ((mz) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned @@ -1378,12 +1382,12 @@ def rend(self, memo, vid=None): else: num = helping.intToB64b(gn, l=nz) # num size must always be neck size - if oz: # need vid part, but can't mod here may need below to sign + if oz: # need oid part, but can't mod here may need below to sign if self.curt: - vidp = decodeB64(vid.encode()) # vid part b2 + oidp = decodeB64(oid.encode()) # oid part b2 else: - vidp = vid.encode() # vid part b64 bytes - head = mid + vidp + num + oidp = oid.encode() # oid part b64 bytes + head = mid + oidp + num else: head = mid + num @@ -1395,7 +1399,7 @@ def rend(self, memo, vid=None): del memo[:bz] # del slice past end just deletes to end if sz: # signed so sign - sig = self.sign(gram, vid) + sig = self.sign(gram, oid) if not sig or len(sig) != sz: raise hioing.MemoerError(f"Signed but unable to sign or " f"invalid signature {sig=}") @@ -1446,11 +1450,11 @@ def send(self, gram, dst, *, echoic=False) -> int: def _serviceOneTxMemo(self): - """Service one (memo, dst, vid) tuple from .txms deque where tuple is - of form: (memo: str, dst: str, vid: str) where: + """Service one (memo, dst, oid) tuple from .txms deque where tuple is + of form: (memo: str, dst: str, oid: str) where: memo is outgoing memo dst is destination address - vid is verification ID + oid is origin ID used to lookup sigkey to sign Calls .rend method to process the partitioning and packing as appropriate to convert memo into grams with headers and sign when @@ -1458,9 +1462,9 @@ def _serviceOneTxMemo(self): Appends (gram, dst) duple to .txgs deque. """ - memo, dst, vid = self.txms.popleft() # raises IndexError if empty deque + memo, dst, oid = self.txms.popleft() # raises IndexError if empty deque - for gram in self.rend(memo, vid): # partition memo into gram parts with head + for gram in self.rend(memo, oid): # partition memo into gram parts with head self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) @@ -1767,20 +1771,20 @@ class SureMemoer(Tymee, Memoer): counts (dict): keyed by mid (memoID) that holds the gram count from the first gram for the memo. This enables lookup of the gram count when fusing its grams. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + the oid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is origin id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: @@ -1826,12 +1830,12 @@ class SureMemoer(Tymee, Memoer): Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter - keep (dict): labels or vids, values are Keyage instances + keep (dict): labels are oids, values are Keyage instances named tuple of signature key pair: sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") - vid (str|None): own vid defaults used to lookup keys to sign on tx + oid (str|None): own oid defaults used to lookup keys to sign on tx """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py index c52c369..63dfabc 100644 --- a/src/hio/core/udp/peermemoing.py +++ b/src/hio/core/udp/peermemoing.py @@ -63,20 +63,20 @@ class PeerMemoer(Peer, Memoer): counts (dict): keyed by mid (memoID) that holds the gram count from the first gram for the memo. This enables lookup of the gram count when fusing its grams. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + the oid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is verification id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: @@ -114,12 +114,12 @@ class PeerMemoer(Peer, Memoer): Default echo is duple that indicates nothing to receive of form (b'', None) When False may be overridden by a method parameter - keep (dict): labels or vids, values are Keyage instances + keep (dict): labels are oids, values are Keyage instances named tuple of signature key pair: sigkey = private signing key verkey = public verifying key Keyage = namedtuple("Keyage", "sigkey verkey") - vid (str|None): own vid defaults used to lookup keys to sign on tx + oid (str|None): own oid defaults used to lookup keys to sign on tx """ diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 55daa27..7be1ddb 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -20,10 +20,10 @@ def _setupKeep(salt=None): """Setup Keep for signed memos Parameters: - salt(str|bytes): salt used to generate key pairs and vids for keep + salt(str|bytes): salt used to generate key pairs and oids for keep Returns: - keep (dict): labels are vids, values are keyage instances + keep (dict): labels are oids, values are keyage instances Ed25519_Seed: str = 'A' # Ed25519 256 bit random seed for private key @@ -44,6 +44,7 @@ def _setupKeep(salt=None): keep = {} + # non transferable verkey as oid sigseed = pysodium.crypto_pwhash(outlen=32, passwd="0", salt=salt, @@ -53,13 +54,13 @@ def _setupKeep(salt=None): # creates signing/verification key pair from seed verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw - # non transferable public key as vid - vid = Memoer._encodeVID(raw=verkey, code='B') # make fully qualified + oid = Memoer._encodeOID(raw=verkey, code='B') # make fully qualified qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified keyage = Keyage(qvk=qvk, qss=qss) # raw - keep[vid] = keyage + keep[oid] = keyage + # transferable verkey as oid sigseed = pysodium.crypto_pwhash(outlen=32, passwd="1", salt=salt, @@ -69,12 +70,13 @@ def _setupKeep(salt=None): # creates signing/verification key pair from seed verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw - vid = Memoer._encodeVID(raw=verkey, code='D') # make fully qualified + oid = Memoer._encodeOID(raw=verkey, code='D') # make fully qualified qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified keyage = Keyage(qvk=qvk, qss=qss) # raw - keep[vid] = keyage + keep[oid] = keyage + # digest of verkey as oid sigseed = pysodium.crypto_pwhash(outlen=32, passwd="2", salt=salt, @@ -85,11 +87,11 @@ def _setupKeep(salt=None): verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw dig = digest = blake3.blake3(verkey).digest() - vid = Memoer._encodeVID(raw=dig, code='E') + oid = Memoer._encodeOID(raw=dig, code='E') qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified keyage = Keyage(qvk=qvk, qss=qss) # raw - keep[vid] = keyage + keep[oid] = keyage return keep @@ -124,12 +126,12 @@ def test_memoer_class(): assert mz # ms must not be empty pz = (3 - ((mz) % 3)) % 3 # net pad size for mid assert pz == (cz % 4) # combined code + mid size must lie on 24 bit boundary - assert not oz % 4 # vid size must be on 24 bit boundary + assert not oz % 4 # oid size must be on 24 bit boundary assert not sz % 4 # sig size must be on 24 bit boundary assert nz and not nz % 4 # neck (num or cnt) size must be on 24 bit boundary assert hz and not hz % 4 # head size must be on 24 bit boundary if oz: - assert sz # ss must not be empty if vs not empty + assert sz # sz must not be empty if oz not empty assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} assert Memoer.Sodex == SGDex @@ -143,7 +145,7 @@ def test_memoer_class(): assert Memoer.BufSize == (2**16-1) # default buffersize verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") - vidqb64 = Memoer._encodeVID(raw=verkey) + vidqb64 = Memoer._encodeOID(raw=verkey) assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' _, _, oz, _, _, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz assert len(vidqb64) == 44 == oz @@ -204,7 +206,7 @@ def test_memoer_basic(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -354,7 +356,7 @@ def test_memoer_small_gram_size(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer = memoing.Memoer(size=38) assert peer.size == 38 @@ -529,7 +531,7 @@ def test_memoer_multiple(): assert not peer.verific assert not peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -608,7 +610,7 @@ def test_memoer_multiple_echoic_service_tx_rx(): assert not peer.verific assert peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -662,7 +664,7 @@ def test_memoer_multiple_echoic_service_all(): assert not peer.verific assert peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -705,10 +707,10 @@ def test_memoer_basic_signed(): except MemoerError as ex: return - vid = list(keep.keys())[0] - assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' - peer = memoing.Memoer(code=GramDex.Signed, keep=keep, vid=vid) + peer = memoing.Memoer(code=GramDex.Signed, keep=keep, oid=oid) assert peer.name == "main" assert peer.opened == False assert peer.bc is None @@ -720,7 +722,7 @@ def test_memoer_basic_signed(): assert not peer.verific assert not peer.echoic assert peer.keep == keep - assert peer.vid == vid + assert peer.oid == oid peer.reopen() assert peer.opened == True @@ -734,9 +736,9 @@ def test_memoer_basic_signed(): memo = "Hello There" dst = "beta" - vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('Hello There', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -756,7 +758,7 @@ def test_memoer_basic_signed(): mid = '_-ALBI68S1ZIxqwFOSWFF1L2' sig = ('A' * 88) - gram = (mid + vid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() + gram = (mid + oid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -767,16 +769,16 @@ def test_memoer_basic_signed(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms # send and receive via echo memo = "See ya later!" dst = "beta" - vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('See ya later!', 'beta', vid) + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('See ya later!', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -805,7 +807,7 @@ def test_memoer_basic_signed(): assert not peer.counts assert not peer.sources peer.rxms[0] - assert peer.rxms[0] == ('See ya later!', 'beta', vid) + assert peer.rxms[0] == ('See ya later!', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms @@ -813,9 +815,9 @@ def test_memoer_basic_signed(): peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('Hello There', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -833,9 +835,9 @@ def test_memoer_basic_signed(): assert not peer.sources assert not peer.rxms mid = '_-ALBI68S1ZIxqwFOSWFF1L2' - vid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .vid + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid sig = ('A' * 88) - head = decodeB64((mid + vid + 'AAAA' + 'AAAB').encode()) + head = decodeB64((mid + oid + 'AAAA' + 'AAAB').encode()) tail = decodeB64(sig.encode()) gram = head + memo.encode() + tail assert peer.wiff(gram) # base2 @@ -850,13 +852,13 @@ def test_memoer_basic_signed(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms - assert peer.inbox[0] == ('Hello There', 'beta', vid) - assert peer.inbox[1] == ('See ya later!', 'beta', vid) - assert peer.inbox[2] == ('Hello There', 'beta', vid) + assert peer.inbox[0] == ('Hello There', 'beta', oid) + assert peer.inbox[1] == ('See ya later!', 'beta', oid) + assert peer.inbox[2] == ('Hello There', 'beta', oid) peer.inbox = deque() # clear it @@ -873,13 +875,13 @@ def test_memoer_multiple_signed(): except MemoerError as ex: return - vid = list(keep.keys())[0] - assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' - vidBeta = list(keep.keys())[1] - assert vidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' - peer = memoing.Memoer(code=GramDex.Signed, size=170, keep=keep, vid=vid) + peer = memoing.Memoer(code=GramDex.Signed, size=170, keep=keep, oid=oid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False @@ -890,14 +892,14 @@ def test_memoer_multiple_signed(): assert not peer.verific assert not peer.echoic assert peer.keep == keep - assert peer.vid == vid + assert peer.oid == oid peer.reopen() assert peer.opened == True # send and receive multiple via echo peer.memoit("Hello there.", "alpha") # use default for vidAlpha - peer.memoit("How ya doing?", "beta", vidBeta) + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -939,8 +941,8 @@ def test_memoer_multiple_signed(): assert not peer.counts assert not peer.sources assert len(peer.rxms) == 2 - assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vidBeta) + assert peer.rxms[0] == ('Hello there.', 'alpha', oid) + assert peer.rxms[1] == ('How ya doing?', 'beta', oidBeta) peer.serviceRxMemos() assert not peer.rxms @@ -951,9 +953,8 @@ def test_memoer_multiple_signed(): assert peer.size == 129 # send and receive multiple via echo in base2 .curt = True mode - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' peer.memoit("Hello there.", "alpha") # use default for vidAlpha - peer.memoit("How ya doing?", "beta", vidBeta) + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -995,17 +996,17 @@ def test_memoer_multiple_signed(): assert not peer.counts assert not peer.sources assert len(peer.rxms) == 2 - assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vidBeta) + assert peer.rxms[0] == ('Hello there.', 'alpha', oid) + assert peer.rxms[1] == ('How ya doing?', 'beta', oidBeta) peer.serviceRxMemos() assert not peer.rxms assert peer.inbox == deque( [ - ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vidBeta), - ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vidBeta) + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta) ]) @@ -1029,7 +1030,7 @@ def test_memoer_verific(): assert peer.verific assert not peer.echoic assert peer.keep == dict() - assert peer.vid is None + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -1052,9 +1053,9 @@ def test_memoer_verific(): assert not peer.sources assert not peer.rxms mid = '_-ALBI68S1ZIxqwFOSWFF1L2' - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + oid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sig = ('A' * 88) - gram = (mid + vid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() + gram = (mid + oid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -1066,13 +1067,13 @@ def test_memoer_verific(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms assert peer.inbox == deque( [ - ('Hello There', 'beta', vid) + ('Hello There', 'beta', oid) ]) peer.close() @@ -1090,8 +1091,8 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): except MemoerError as ex: return - vid = list(keep.keys())[0] - assert vid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' vidBeta = list(keep.keys())[1] assert vidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' @@ -1099,7 +1100,7 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, - echoic=True, keep=keep, vid=vid) + echoic=True, keep=keep, oid=oid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False @@ -1110,13 +1111,12 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.verific assert peer.echoic assert peer.keep == keep - assert peer.vid == vid + assert peer.oid == oid peer.reopen() assert peer.opened == True # send and receive multiple via echo - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' peer.memoit("Hello there.", "alpha") peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 @@ -1141,7 +1141,7 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.inbox == deque( [ - ('Hello there.', 'alpha', vid), + ('Hello there.', 'alpha', oid), ('How ya doing?', 'beta', vidBeta) ]) @@ -1152,7 +1152,6 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.size == 129 # send and receive multiple via echo - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' peer.memoit("Hello there.", "alpha") peer.memoit("How ya doing?", "beta", vidBeta) assert len(peer.txms) == 2 @@ -1177,9 +1176,9 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): assert peer.inbox == deque( [ - ('Hello there.', 'alpha', vid), + ('Hello there.', 'alpha', oid), ('How ya doing?', 'beta', vidBeta), - ('Hello there.', 'alpha', vid), + ('Hello there.', 'alpha', oid), ('How ya doing?', 'beta', vidBeta), ]) @@ -1241,13 +1240,13 @@ def test_sure_memoer_basic(): except MemoerError as ex: return - vid = list(keep.keys())[0] - assert vid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + oid = list(keep.keys())[0] + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' - vidBeta = list(keep.keys())[1] - assert vidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' - peer = memoing.SureMemoer(echoic=True, keep=keep, vid=vid) + peer = memoing.SureMemoer(echoic=True, keep=keep, oid=oid) assert peer.size == 65535 assert peer.name == "main" assert peer.opened == False @@ -1258,7 +1257,7 @@ def test_sure_memoer_basic(): assert peer.verific assert peer.echoic assert peer.keep == keep - assert peer.vid == vid + assert peer.oid == oid assert peer.Sizes[peer.code] == Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160) assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz @@ -1277,9 +1276,8 @@ def test_sure_memoer_basic(): memo = "Hello There" dst = "beta" - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit(memo, dst, vidBeta) - assert peer.txms[0] == ('Hello There', 'beta', vidBeta) + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('Hello There', 'beta', oidBeta) peer.service() # services Rx then Tx so rx of tx not serviced until 2nd pass @@ -1303,14 +1301,14 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vidBeta), + ('Hello There', 'beta', oidBeta), ]) # send and receive some more memo = "See ya later!" dst = "beta" - peer.memoit(memo, dst, vidBeta) - assert peer.txms[0] == ('See ya later!', 'beta', vidBeta) + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('See ya later!', 'beta', oidBeta) peer.service() @@ -1334,16 +1332,16 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vidBeta), - ('See ya later!', 'beta', vidBeta), + ('Hello There', 'beta', oidBeta), + ('See ya later!', 'beta', oidBeta), ]) # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - peer.memoit(memo, dst, vidBeta) - assert peer.txms[0] == ('Hello There', 'beta', vidBeta) + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('Hello There', 'beta', oidBeta) peer.service() @@ -1366,9 +1364,9 @@ def test_sure_memoer_basic(): assert peer.inbox == deque( [ - ('Hello There', 'beta', vidBeta), - ('See ya later!', 'beta', vidBeta), - ('Hello There', 'beta', vidBeta), + ('Hello There', 'beta', oidBeta), + ('See ya later!', 'beta', oidBeta), + ('Hello There', 'beta', oidBeta), ]) # Test wind @@ -1412,16 +1410,16 @@ def test_sure_memoer_multiple_echoic_service_all(): except MemoerError as ex: return - vid = list(keep.keys())[0] - assert vid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + oid = list(keep.keys())[0] + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' - vidBeta = list(keep.keys())[1] - assert vidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code - with memoing.openSM(size=170, echoic=True, keep=keep, vid=vid) as peer: + with memoing.openSM(size=170, echoic=True, keep=keep, oid=oid) as peer: assert peer.size == 170 assert peer.name == "test" assert peer.opened == True @@ -1432,12 +1430,11 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.verific assert peer.echoic assert peer.keep == keep - assert peer.vid is vid + assert peer.oid is oid # send and receive multiple via echo - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' peer.memoit("Hello there.", "alpha") - peer.memoit("How ya doing?", "beta", vidBeta) + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.service() # services Rx first then Tx so have to repeat @@ -1461,8 +1458,8 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.inbox == deque( [ - ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vidBeta) + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta) ]) @@ -1473,9 +1470,8 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.size == 129 # send and receive multiple via echo - #vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' peer.memoit("Hello there.", "alpha") - peer.memoit("How ya doing?", "beta", vidBeta) + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.serviceAll() # servicAll services Rx first then Tx so have to repeat @@ -1498,10 +1494,10 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.inbox == deque( [ - ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vidBeta), - ('Hello there.', 'alpha', vid), - ('How ya doing?', 'beta', vidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), ]) peer.inbox = deque() # clear it diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index 787be75..1997dc2 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -171,7 +171,6 @@ def test_memoer_peer_open(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == size assert alpha.bc == 1024 From b5dbc49aee09dbea41f9f055a697c577058af83a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 18 Jan 2026 10:57:39 -0700 Subject: [PATCH 23/27] preliminary work to support actual signing of memo grams --- src/hio/core/memo/memoing.py | 46 ++++++++++++++++++++++++++++++++- tests/core/memo/test_memoing.py | 6 +++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index e7d5cfa..ea5d25c 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -536,6 +536,45 @@ def _encodeQSS(cls, raw, code='A'): return qb64 # qualified base64 sigseed + @classmethod + def _decodeQSS(cls, qss, code='A'): + """Utility method for use with signed headers that decodes qualified + base64 sigseed to raw domain bytes from CESR compatible text code + + Parameters: + qss (bytes): qualified base64 sigseed to be deccoded with code + code (str): code for type of raw sigseed CESR compatible + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key + + Returns: + sigseed (bytes): raw sigseed suitable for signing + """ + if code not in ('A'): + raise hioing.MemoerError(f"Invalid qss {code=}") + + qz = len(qss) # text size + if qz != 44: + raise hioing.MemoerError(f"Invalid qss text size {qz=} not 44") + + cz = len(code) + pz = cz % 4 # net pad size given cz + if cz != pz != 1: # special case here for now we only accept cz=1 + raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + + base = pz * b'A' + qss[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(ps)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {qss}") + + return raw # qualified base64 sigseed + + def __init__(self, *, name=None, bc=None, @@ -1292,11 +1331,16 @@ def sign(self, ser, oid): oid (bytes): qb64 or qb2 if .curt of oid of signer assumes oid of correct length - """ + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key + """ if oid not in self.keep: return b'' # return of empty signature should raise error in caller + qvk, qss = self.keep[oid] + + sigseed = Memoer._decodeQSS(qss) + _, _, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 7be1ddb..966c70a 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -149,6 +149,12 @@ def test_memoer_class(): assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' _, _, oz, _, _, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz assert len(vidqb64) == 44 == oz + + sigseed = (b"\x9bF\n\xf1\xc2L\xeaBC:\xf7\xe9\xd71\xbc\xd2{\x7f\x81\xae5\x9c\xca\xf9\xdb\xac@`'\x0e\xa4\x10") + qss = Memoer._encodeQSS(raw=sigseed) + assert qss == 'AJtGCvHCTOpCQzr36dcxvNJ7f4GuNZzK-dusQGAnDqQQ' + raw = Memoer._decodeQSS(qss) + assert raw == sigseed """Done Test""" From 87a9a70250e6d3f489a086f9e02cfc54194ad776 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Jan 2026 18:01:38 -0700 Subject: [PATCH 24/27] fixed deprecated utcnow() more refinement on memoing --- src/hio/core/coring.py | 4 +- src/hio/core/http/httping.py | 2 +- src/hio/core/http/serving.py | 4 +- src/hio/core/memo/memoing.py | 180 +++++++++++++++++++++++++++----- tests/base/test_multidoing.py | 4 +- tests/core/memo/test_memoing.py | 28 +++-- 6 files changed, 184 insertions(+), 38 deletions(-) diff --git a/src/hio/core/coring.py b/src/hio/core/coring.py index 7264782..7a54caf 100644 --- a/src/hio/core/coring.py +++ b/src/hio/core/coring.py @@ -83,7 +83,7 @@ def arpCreate(ether, host, interface="en0", temp=True): """ temp = "temp" if temp else "" console.terse("{0}: Creating {1} arp entry for {2} at {3} on {4}\n".format( - datetime.datetime.utcnow().isoformat(), + datetime.datetime.now(datetime.UTC).isoformat(), temp, ether, host, @@ -103,7 +103,7 @@ def arpCreate(ether, host, interface="en0", temp=True): check=True) except subprocess.SubprocessError as ex: console.terse("{0}: Failed Creation of {1} arp entry for {2} at {3} on {4}\n".format( - datetime.datetime.utcnow().isoformat(), + datetime.datetime.now(datetime.UTC).isoformat(), temp, ether, host, diff --git a/src/hio/core/http/httping.py b/src/hio/core/http/httping.py index 644e503..a37d45b 100644 --- a/src/hio/core/http/httping.py +++ b/src/hio/core/http/httping.py @@ -277,7 +277,7 @@ def httpDate1123(dt): The supplied date must be in UTC. import datetime - httpDate1123(datetime.datetime.utcnow()) + httpDate1123(datetime.datetime.now(datetime.UTC)) 'Wed, 30 Sep 2015 14:29:18 GMT' """ weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()] diff --git a/src/hio/core/http/serving.py b/src/hio/core/http/serving.py index d8f514c..ab76416 100644 --- a/src/hio/core/http/serving.py +++ b/src/hio/core/http/serving.py @@ -353,7 +353,7 @@ def build(self): self.headers[u'server'] = "Ioflo WSGI Server" if u'date' not in self.headers: # create Date header - self.headers[u'date'] = httping.httpDate1123(datetime.datetime.utcnow()) + self.headers[u'date'] = httping.httpDate1123(datetime.datetime.now(datetime.UTC)) if self.chunkable and ('transfer-encoding' not in self.headers or self.headers['transfer-encoding'] == 'chunked'): @@ -949,7 +949,7 @@ def build(self, self.headers[u'server'] = "Ioflo Server" if u'date' not in self.headers: # create Date header - self.headers[u'date'] = httping.httpDate1123(datetime.datetime.utcnow()) + self.headers[u'date'] = httping.httpDate1123(datetime.datetime.now(datetime.UTC)) if self.data is not None: body = json.dumps(self.data, separators=(',', ':')).encode("utf-8") diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index ea5d25c..dfe4845 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -470,6 +470,51 @@ def _encodeOID(cls, raw, code='B'): return qb64 # fully qualified base64 oid with prefix code + @classmethod + def _decodeOID(cls, qb64): + """Utility method for use with signed headers that decodes qualified + base64 oid to raw domain bytes from CESR compatible text code + + Parameters: + qb64 (str): qualified base64 oid to be decoded with code + code (str): code for type of oid CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verkey basic derivation + Blake3_256: str = 'E' # Blake3 256 bit digest derivation. + + Returns: + tuple(raw, code) where: + raw (bytes): oid suitable for crypto operations + code (str): CESR compatible code from qb64 + """ + cz = 1 # only support qb64 length 44 + code = qb64[:cz] + if code not in ('B', 'D', 'E'): + raise hioing.MemoerError(f"Invalid oid {code=}") + + qz = len(qb64) # text size + if qz != 44: + raise hioing.MemoerError(f"Invalid oid text size {qz=} not 44") + + cz = len(code) + pz = cz % 4 # net pad size given cz + if cz != pz != 1: # special case here for now we only accept cz=1 + raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {oid}") + + return raw, code + + @classmethod def _encodeQVK(cls, raw, code='B'): """Utility method for use with signed headers that encodes raw verkey as @@ -529,7 +574,7 @@ def _encodeQSS(cls, raw, code='A'): raise hioing.MemoerError(f"Invalid code size={len(code)} " f"not equal {pz=} not equal 1") - b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, aqb64prestrip qb64 = code + b64.decode() # fully qualified with prefix code @@ -537,22 +582,26 @@ def _encodeQSS(cls, raw, code='A'): @classmethod - def _decodeQSS(cls, qss, code='A'): + def _decodeQSS(cls, qb64): """Utility method for use with signed headers that decodes qualified base64 sigseed to raw domain bytes from CESR compatible text code Parameters: - qss (bytes): qualified base64 sigseed to be deccoded with code + qb64 (str): qualified base64 sigseed to be decoded with code code (str): code for type of raw sigseed CESR compatible Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key Returns: - sigseed (bytes): raw sigseed suitable for signing + tuple(raw, code) where: + raw (bytes): sigseed suitable for crypto operations + code (str): CESR compatible code from qb64 """ + cz = 1 # only support qb64 length 44 + code = qb64[:cz] if code not in ('A'): raise hioing.MemoerError(f"Invalid qss {code=}") - qz = len(qss) # text size + qz = len(qb64) # text size if qz != 44: raise hioing.MemoerError(f"Invalid qss text size {qz=} not 44") @@ -561,18 +610,97 @@ def _decodeQSS(cls, qss, code='A'): if cz != pz != 1: # special case here for now we only accept cz=1 raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") - base = pz * b'A' + qss[cz:].encode() # strip code from b64 and prepad pz 'A's + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's paw = decodeB64(base) # now should have pz leading sextexts of zeros raw = paw[pz:] # remove prepad midpad bytes to invert back to raw # ensure midpad bytes are zero pi = int.from_bytes(paw[:pz], "big") if pi != 0: - raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(ps)*2}x}.") + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") if len(raw) != ((qz - cz) * 3 // 4): # exact lengths raise hioing.MemoerError(f"Improperly qualified material = {qss}") - return raw # qualified base64 sigseed + return raw, code + + + @classmethod + def _encodeSig(cls, raw, code='0B'): + """Utility method for use with signed headers that encodes raw signature + as CESR compatible fully qualified B64 text domain str using CESR + compatible text code + + Parameters: + raw (bytes): sig to be encoded with code + code (str): code for type of raw sig CESR compatible + Ed25519_Sig: str = '0B' # Ed25519 signature. not indexed + + Returns: + qb64 (str): fully qualified base64 sig + """ + if code not in ('0B'): + raise hioing.MemoerError(f"Invalid sig {code=}") + + rz = len(raw) # raw size + if rz != 64: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 64") + + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw + if len(code) != pz != 2: + raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" + f" {pz=} not equal 2") + + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified base64 with prefixqb64de + + _, _, _, _, sz, _ = cls.Sizes[SGDex.Signed] # cz mz oz nz sz hz + if len(qb64) != sz: + hioing.MemoerError(f"Invalid sig qb64 size={len(qb64) != {sz}}") + + return qb64 # fully qualified base64 oid with prefix code + + + @classmethod + def _decodeSig(cls, qb64): + """Utility method for use with signed headers that decodes qualified + base64 sig to raw domain bytes from CESR compatible text code + + Parameters: + qb64 (str): qualified base64 sig to be decoded with code + code (str): code for type of raw sig CESR compatible + Ed25519_Sig: str = '0B' # Ed25519 signature. not indexed + + Returns: + tuple(raw, code) where: + raw (bytes): signature suitable for crypto operations + code (str): CESR compatible code from qb64 + """ + cz = 2 # only support qb64 length 88 + code = qb64[:cz] + if code not in ('0B'): + raise hioing.MemoerError(f"Invalid sig {code=}") + + qz = len(qb64) # text size + if qz != 88: + raise hioing.MemoerError(f"Invalid sig text size {qz=} not 88") + + pz = cz % 4 # net pad size given cz + if pz != cz: + raise hioing.MemoerError(f"Invalid {pz=} not equal {cz=}") + + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {sig}") + + return raw, code # qualified base64 sigseed def __init__(self, *, @@ -1324,7 +1452,10 @@ def sign(self, ser, oid): This is a stub. Returns: - sig(bytes): qb64b qualified base64 representation of signature + sig (bytes): qb64b or qb2 when .curt + if not .curt then qualified base64 of signature + else if .curt then qualified base2 of signature + raises MemoerError if problem signing Parameters: ser (bytes): signed portion of gram is delivered format @@ -1335,23 +1466,25 @@ def sign(self, ser, oid): """ if oid not in self.keep: - return b'' # return of empty signature should raise error in caller + raise hioing.MemoerError("Invalid {oid=} for signing") qvk, qss = self.keep[oid] + sigseed, code = self._decodeQSS(qss) # raises MemoerError if problem + if code not in ('A'): + raise hioing.MemoerError(f"Invalid sigseed algorithm type {code=}") - sigseed = Memoer._decodeQSS(qss) - - - _, _, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz + try: + import pysodium + except ImportError as ex: + raise hioing.MemoerError("Missing pysodium lib for signing") from ex - sig = b'A' * sz + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) + raw = pysodium.crypto_sign_detached(ser, sigkey) # raw sig + sig = self._encodeSig(raw) # raise MemoerError if problem + sig = sig.encode() # make bytes if self.curt: - hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 - nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 - oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 - sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 - sig = decodeB64(sig) # make b2 + sig = decodeB64(sig) # make qb2 return sig @@ -1442,11 +1575,8 @@ def rend(self, memo, oid=None): gram = head + memo[:bz] # copy slice past end just copies to end del memo[:bz] # del slice past end just deletes to end - if sz: # signed so sign - sig = self.sign(gram, oid) - if not sig or len(sig) != sz: - raise hioing.MemoerError(f"Signed but unable to sign or " - f"invalid signature {sig=}") + if sz: # signed gram, .sign returns proper sig format when .curt + sig = self.sign(gram, oid) # raises MemoerError if invalid gram = gram + sig grams.append(gram) diff --git a/tests/base/test_multidoing.py b/tests/base/test_multidoing.py index 3c617ef..d319d57 100644 --- a/tests/base/test_multidoing.py +++ b/tests/base/test_multidoing.py @@ -350,11 +350,11 @@ def test_boss_crew_basic_multi(): assert not doer.opened doers = [doer] - doist = doing.Doist(tock=0.01, real=True, limit=0.50, doers=doers, temp=True) + doist = doing.Doist(tock=0.01, real=True, limit=1.0, doers=doers, temp=True) assert doist.tyme == 0.0 # on next cycle assert doist.tock == 0.01 assert doist.real == True - assert doist.limit == 0.50 + assert doist.limit == 1.0 assert doist.doers == [doer] doist.do() diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 966c70a..effd817 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -31,8 +31,8 @@ def _setupKeep(salt=None): try: import pysodium import blake3 - except ImportError: - raise MemoerError("Missing cryptographic module support") + except ImportError as ex: + raise MemoerError("Missing cryptographic module support") from ex salt = salt if salt is not None else b"abcdefghijklmnop" @@ -145,16 +145,32 @@ def test_memoer_class(): assert Memoer.BufSize == (2**16-1) # default buffersize verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") - vidqb64 = Memoer._encodeOID(raw=verkey) - assert vidqb64 == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + oid = Memoer._encodeOID(raw=verkey) + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + raw, code = Memoer._decodeOID(oid) + assert raw == verkey + assert code == 'B' _, _, oz, _, _, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz - assert len(vidqb64) == 44 == oz + assert len(oid) == 44 == oz sigseed = (b"\x9bF\n\xf1\xc2L\xeaBC:\xf7\xe9\xd71\xbc\xd2{\x7f\x81\xae5\x9c\xca\xf9\xdb\xac@`'\x0e\xa4\x10") qss = Memoer._encodeQSS(raw=sigseed) assert qss == 'AJtGCvHCTOpCQzr36dcxvNJ7f4GuNZzK-dusQGAnDqQQ' - raw = Memoer._decodeQSS(qss) + raw, code = Memoer._decodeQSS(qss) assert raw == sigseed + assert code == 'A' + + signature = (b'\xb0\xc0\xd5\t\xa0\xd3Q0\xfa8\x93B\x0c\x83\xb5.\xfbH\xa5\xde\xbf}6{' + b'\xcf|\xa3\x0el"\x84f\xd3sbHC\xb9\xb9\x85\xb0\xd2v\xed\x07\xcf|c\xa4\xd6\xdcE' + b'\xbe\x8a{w5=\xbf\x84_\x9e\xb3\x04') + sig = Memoer._encodeSig(raw=signature) + assert sig == '0BCwwNUJoNNRMPo4k0IMg7Uu-0il3r99NnvPfKMObCKEZtNzYkhDubmFsNJ27QfPfGOk1txFvop7dzU9v4RfnrME' + raw, code = Memoer._decodeSig(sig) + assert raw == signature + assert code == '0B' + _, _, _, _, sz, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz + assert len(sig) == 88 == sz + """Done Test""" From 6555e290a72c3d23f08c036f345b30f2cd8d615e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Jan 2026 18:30:54 -0700 Subject: [PATCH 25/27] added more sleep to fix github test runner --- tests/core/udp/test_udping.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/udp/test_udping.py b/tests/core/udp/test_udping.py index 04a4f01..1de827a 100644 --- a/tests/core/udp/test_udping.py +++ b/tests/core/udp/test_udping.py @@ -236,28 +236,28 @@ def test_open_peer(): msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) - time.sleep(0.05) + time.sleep(0.1) msgIn, src = beta.receive() assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"alpha sends to alpha" alpha.send(msgOut, alpha.ha) - time.sleep(0.05) + time.sleep(0.1) msgIn, src = alpha.receive() assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"beta sends to alpha" beta.send(msgOut, alpha.ha) - time.sleep(0.05) + time.sleep(0.1) msgIn, src = alpha.receive() assert msgOut == msgIn assert src[1] == beta.port # ports equal msgOut = b"beta sends to beta" beta.send(msgOut, beta.ha) - time.sleep(0.05) + time.sleep(0.1) msgIn, src = beta.receive() assert msgOut == msgIn assert src[1] == beta.port # ports equal From 603d10baf854f843aaf9f02c25afe980c9f62dda Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Jan 2026 18:40:46 -0700 Subject: [PATCH 26/27] added more sleep delay to udp tests since failing on github runner --- tests/core/udp/test_peer_memoing.py | 7 ++++--- tests/core/udp/test_udping.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index 1997dc2..2744786 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -201,8 +201,8 @@ def test_memoer_peer_open(): assert not alpha.sources # beta receives - beta.serviceReceives() while not beta.rxgs: + beta.serviceReceives() time.sleep(0.05) assert not beta.echos assert len(beta.rxgs) == 2 @@ -248,8 +248,9 @@ def test_memoer_peer_open(): assert not beta.sources # alpha receives - alpha.serviceReceives() - time.sleep(0.05) + while not alpha.rxgs: + alpha.serviceReceives() + time.sleep(0.05) assert not alpha.echos assert len(alpha.rxgs) == 2 assert len(alpha.counts) == 2 diff --git a/tests/core/udp/test_udping.py b/tests/core/udp/test_udping.py index 1de827a..6284631 100644 --- a/tests/core/udp/test_udping.py +++ b/tests/core/udp/test_udping.py @@ -245,6 +245,10 @@ def test_open_peer(): alpha.send(msgOut, alpha.ha) time.sleep(0.1) msgIn, src = alpha.receive() + if not msgIn: + time.sleep(0.1) + msgIn, src = alpha.receive() + assert msgOut == msgIn assert src[1] == alpha.port # ports equal From f77c492e366b9d60b3a28b4442672db3e5987270 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Jan 2026 18:58:53 -0700 Subject: [PATCH 27/27] more adjustment to udp tests to pass on github --- tests/core/udp/test_udping.py | 55 +++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/core/udp/test_udping.py b/tests/core/udp/test_udping.py index 6284631..dc365a4 100644 --- a/tests/core/udp/test_udping.py +++ b/tests/core/udp/test_udping.py @@ -131,7 +131,10 @@ def test_udp(): assert alpha.reopen() assert alpha.opened assert alpha.ha == alphaHa - assert alpha.actualBufSizes() == (alpha.bs, alpha.bs) == (1269760, 1269760) + #assert alpha.actualBufSizes() == (alpha.bs, alpha.bs) == (1269760, 1269760) + sizes = alpha.actualBufSizes() + assert sizes[0] > 16383 + assert sizes[1] > 16383 bs = 2 ** 15 - 1 assert bs == 32767 @@ -151,28 +154,40 @@ def test_udp(): msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) time.sleep(0.05) - msgIn, src = beta.receive() + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"alpha sends to alpha" alpha.send(msgOut, alpha.ha) time.sleep(0.05) - msgIn, src = alpha.receive() + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"beta sends to alpha" beta.send(msgOut, alpha.ha) time.sleep(0.05) - msgIn, src = alpha.receive() + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal msgOut = b"beta sends to beta" beta.send(msgOut, beta.ha) time.sleep(0.05) - msgIn, src = beta.receive() + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal @@ -236,33 +251,41 @@ def test_open_peer(): msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) - time.sleep(0.1) - msgIn, src = beta.receive() + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"alpha sends to alpha" alpha.send(msgOut, alpha.ha) - time.sleep(0.1) - msgIn, src = alpha.receive() - if not msgIn: - time.sleep(0.1) + time.sleep(0.05) + msgIn = b"" + while not msgIn: msgIn, src = alpha.receive() - + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"beta sends to alpha" beta.send(msgOut, alpha.ha) - time.sleep(0.1) - msgIn, src = alpha.receive() + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal msgOut = b"beta sends to beta" beta.send(msgOut, beta.ha) - time.sleep(0.1) - msgIn, src = beta.receive() + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal