diff --git a/src/hio/base/multidoing.py b/src/hio/base/multidoing.py index 08635831..f96e5686 100644 --- a/src/hio/base/multidoing.py +++ b/src/hio/base/multidoing.py @@ -568,7 +568,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + memo, src, oid = self._serviceOneRxMemo() self.logger.debug("Boss Peer RX: name=%s rx from src=%s memo=%s.", self.name, src, memo) @@ -797,7 +797,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + memo, src, oid = self._serviceOneRxMemo() self.logger.debug("Hand Peer RX: name=%s rx from src=%s memo=%s.", self.name, src, memo) diff --git a/src/hio/base/tyming.py b/src/hio/base/tyming.py index 990f30ef..c3e92bea 100644 --- a/src/hio/base/tyming.py +++ b/src/hio/base/tyming.py @@ -104,14 +104,16 @@ class Tymee(hioing.Mixin): Tymee has .tyme property that returns the artificial or simulated or cycle time from its referenced Tymist instance ._tymist. + Class Attributes: + Attributes: Properties: - .tyme (float | None): relative cycle time of associated Tymist which is + tyme (float | None): relative cycle time of associated Tymist which is provided by calling .tymth function wrapper closure which is obtained from Tymist.tymen(). None means not assigned yet. - .tymth (Callable | None): function wrapper closure returned by + tymth (Callable | None): function wrapper closure returned by Tymist.tymen() method. When .tymth is called it returns associated Tymist.tyme. Provides injected dependency on Tymist cycle tyme base. None means not assigned yet. @@ -120,7 +122,7 @@ class Tymee(hioing.Mixin): .wind injects ._tymth dependency from associated Tymist to get its .tyme Hidden: - ._tymth is injected function wrapper closure returned by .tymen() of + _tymth is injected function wrapper closure returned by .tymen() of associated Tymist instance that returns Tymist .tyme. when called. """ @@ -184,17 +186,17 @@ class Tymer(Tymee): Attributes: Inherited Properties: - .tyme is float relative cycle time of associated Tymist .tyme obtained + tyme is float relative cycle time of associated Tymist .tyme obtained via injected .tymth function wrapper closure. - .tymth is function wrapper closure returned by Tymist .tymeth() method. + tymth is function wrapper closure returned by Tymist .tymeth() method. When .tymth is called it returns associated Tymist .tyme. .tymth provides injected dependency on Tymist tyme base. Properties: - .duration = tyme duration of tymer in seconds from ._start to ._stop - .elaspsed = tyme elasped in seconds since ._start - .remaining = tyme remaining in seconds until ._stop - .expired = True if expired, False otherwise, i.e. .tyme >= ._stop + duration (float): tyme duration in seconds from ._start to ._stop + elaspsed (float): tyme elasped in seconds since ._start + remaining (float): tyme remaining in seconds until ._stop + expired (bool): True if expired, False otherwise, i.e. .tyme >= ._stop Inherited Methods: .wind is injects ._tymth dependency @@ -204,10 +206,10 @@ class Tymer(Tymee): .restart() = restart tymer at last ._stop so no time lost Hidden: - ._tymth is injected function wrapper closure returned by .tymen() of + _tymth (closure): injected function wrapper closure returned by .tymen() of associated Tymist instance that returns Tymist .tyme. when called. - ._start is start tyme in seconds - ._stop is stop tyme in seconds + _start (float): start tyme in seconds + _stop (float): stop tyme in seconds """ Duration = 0.0 # default duration when not provided diff --git a/src/hio/core/coring.py b/src/hio/core/coring.py index 72647825..7a54caf1 100644 --- a/src/hio/core/coring.py +++ b/src/hio/core/coring.py @@ -83,7 +83,7 @@ def arpCreate(ether, host, interface="en0", temp=True): """ temp = "temp" if temp else "" console.terse("{0}: Creating {1} arp entry for {2} at {3} on {4}\n".format( - datetime.datetime.utcnow().isoformat(), + datetime.datetime.now(datetime.UTC).isoformat(), temp, ether, host, @@ -103,7 +103,7 @@ def arpCreate(ether, host, interface="en0", temp=True): check=True) except subprocess.SubprocessError as ex: console.terse("{0}: Failed Creation of {1} arp entry for {2} at {3} on {4}\n".format( - datetime.datetime.utcnow().isoformat(), + datetime.datetime.now(datetime.UTC).isoformat(), temp, ether, host, diff --git a/src/hio/core/http/httping.py b/src/hio/core/http/httping.py index 644e503a..a37d45b1 100644 --- a/src/hio/core/http/httping.py +++ b/src/hio/core/http/httping.py @@ -277,7 +277,7 @@ def httpDate1123(dt): The supplied date must be in UTC. import datetime - httpDate1123(datetime.datetime.utcnow()) + httpDate1123(datetime.datetime.now(datetime.UTC)) 'Wed, 30 Sep 2015 14:29:18 GMT' """ weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()] diff --git a/src/hio/core/http/serving.py b/src/hio/core/http/serving.py index d8f514c0..ab764169 100644 --- a/src/hio/core/http/serving.py +++ b/src/hio/core/http/serving.py @@ -353,7 +353,7 @@ def build(self): self.headers[u'server'] = "Ioflo WSGI Server" if u'date' not in self.headers: # create Date header - self.headers[u'date'] = httping.httpDate1123(datetime.datetime.utcnow()) + self.headers[u'date'] = httping.httpDate1123(datetime.datetime.now(datetime.UTC)) if self.chunkable and ('transfer-encoding' not in self.headers or self.headers['transfer-encoding'] == 'chunked'): @@ -949,7 +949,7 @@ def build(self, self.headers[u'server'] = "Ioflo Server" if u'date' not in self.headers: # create Date header - self.headers[u'date'] = httping.httpDate1123(datetime.datetime.utcnow()) + self.headers[u'date'] = httping.httpDate1123(datetime.datetime.now(datetime.UTC)) if self.data is not None: body = json.dumps(self.data, separators=(',', ':')).encode("utf-8") diff --git a/src/hio/core/memo/__init__.py b/src/hio/core/memo/__init__.py index 6924ae2d..2dd88354 100644 --- a/src/hio/core/memo/__init__.py +++ b/src/hio/core/memo/__init__.py @@ -4,7 +4,7 @@ """ -from .memoing import (Versionage, Sizage, GramDex, SGDex, +from .memoing import (Versionage, Sizage, GramDex, SGDex, Keyage, openMemoer, Memoer, MemoerDoer, - openTM, TymeeMemoer, TymeeMemoerDoer) + openSM, SureMemoer, SureMemoerDoer) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 474f3dce..dfe4845a 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -3,9 +3,10 @@ hio.core.memoing Module Mixin Base Classes that add support for memograms over datagram based transports. -In this context, a memogram is a larger memogram that is partitioned into -smaller parts as transport specific datagrams. This allows larger messages to -be transported over datagram transports than the underlying transport can support. +In this context, a memo made is a larger message that is partitioned into +smaller memograms parts that fit into transport specific datagrams. +This allows larger messages (memos) to be transported over datagram transports +which messages are larger than a single datagram in the underlying transport. """ import socket @@ -22,6 +23,7 @@ from ... import hioing, help from ...base import tyming, doing +from ...base.tyming import Tymer, Tymee from ...help import helping logger = help.ogler.getLogger() @@ -29,21 +31,189 @@ # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") +# signature key pair: +# qvk = qualified base64 public verifying key +# qss = qualified base64 private signing key seed (ed25519 sigkey = sigseed + verkey) +Keyage = namedtuple("Keyage", "qvk qss") +# example usage +#keyage = Keyage(qvk="xyy", qss="abc", ) +#keep = dict("ABCXYZ"=keyage) # qualified oid as label, Keyage instance as value + +"""Design Discusssion of Memo and Gram Sizing and Encoding: + +Each GramCode (tv) Typing/Version code uses a base64 two char code. +Codes always start with `_` and second char in code starts at `_` +and walks backwards through B64 code table for each gram type-version (tv) code. + +tv 0 is '__' +tv 1 is '_-' +tv 3 is '_9' +... + +Each of total head length including code and total neck length must be a +multiple of 4 characters (3 bytes) to align on 24 bit boundary for clean Base64 +round tripping. This ensure each converts to/from B2 without padding. +Head type-version code governs the neck as well. So whenever field lengths or +fields change in any way we need a new gram (tv) code. + +To reiterate, if ever need to change anything need a new gram (tv) code. There are +no versions per type. This is deemed sufficient because we anticipate a very +limited set of possible fields ever needed for memogram transport types. + +All gram head and neck fields are mid-pad B64 primitives. This makes for stable +coding at fron and back of head neck parts. This also means that gram parts can +can be losslessly transformed to/from equivalent CESR primitives merely by +translating the gram tv code to an equivalent entry in the two char cesr +primitive code table. When doing so the neck is always attached and then +stripped when not needed. + +The equivalent CESR codes, albeit different, map one-to-one to the gram tv codes. +This enables representing headers as CESR primitives for management but not +over the wire. One can transform Memogram Grams to CESR primitives to tunnel in +some CESR stream. + +Normally CESR streams are tunneled inside Memoer memograms sent over-the-wire. +In this case, the memogram is the wrapper and each gram uses the gram TV code +defined herein not the equivalent CESR code. +Using so far reserved and unused '_' code which is reserved as the selector +for the CESR op code table for memogram grams makes it more apparent that a gram +belongs to a memogram and not a CESR stream. +Eventually when CESR op codes eventually become defined, it's not likely to be +confusing since the context of CESR op codes is dramatically different from +transport wrappers. + +Morover when a MemoGram payload is a tunneled CESR Stream, the memogram parser +will parse and strip the MemoGram gram wrappers to construct the tunneled, +stream so the wrapper is transarent to the CESR stream and the CESR stream +payload is opaque to the memogram wrapper, i.e. no ambiguity. + +Gram Header Fields: + HeadCode hc = b'__' + HeadCodeSize hcs = 2 + MemoIDSize mis = 22 + GramNumSize gns = 4 + GramCntSize gcs = 4 + GramHeadSize ghs = 28 + GramHeadNeckSize ghns = 28 + 4 = 32 + +Sizing: + +We have for most datagrams, such as UDP, a max datagram size of 65535 = (2 ** 16 -1) +Thix may no longer be true on some OSes for UXD datagrams. On some OSes UXD datagrams +may be bigger. But 65535 seems like a good practical limit for maximum compatibility +across all datagram types. + +Given gram count and gram num fields are 4 Base64 chars (3 bytes) or 24 bits, then: +MaxGramCount = 2**24-1 + +If we want a maximally size memo to be big enough to transport 1 CESR Big frame +with frame counter then the maximum memo size needs to be +(2**30-1) * 4 + 8 = 4294967292 + 8 = 4294967300 + +CESR groups code count in quadlets (4 chars). +The big group count codes have five digits of 6 bits each or 30 bits to count the +size of the following framed chars. +The maximum group count is (2**30-1) == 1073741823 +Cesr group codes count the number +of quadlets (4 chars) in the framed group. The maximum number of bytes in a +CESR big group frame body is therefore: +(2**30-1)*4 = 1073741823 * 4 = 4294967292. +The group code itself is not included in the counted quadlets in the frame body. +The big group codes are 8 chars long. Thus with its group code the total big +frame size is: +(2**30-1)*4 + 8 = 4294967292 + 8 = 4294967300 + +We set +MaxMemoSize (mms) = (2**30-1)*4+8 = 4294967300 + +We can compare UDP and UXD given these constraints. +Notice that gramsize includes head + body and the first gram size is 4 larger +to include gram count field. + +In general +MaxMemoSize = GramBodySize * MaxGramCount + +Because the gram count field is 24 bits (4 chars or 3 bytes), we have: +MaxGramCount = 2**24-1 = 16777215 + +At a MaxGramCount of 16777215 for the MaxMemoSize we +GramBodySize = MaxMemoSize / MaxGramCount = 4294967300/16777215 = 256 remainder 260 +256 = 2**8. So we need GramBodySize to be a little larger than 256 to reach +MaxMemoSize. If we round up to 257 we get +maximum possible memosize = 257 * 16777215 = 4311744255 = ((2**8)+1)*(2**24-1) +Or alternatively if we set GramBodySize to 257 then the max gram count we get is +GramCount = ceil(MaxMemoSize / 257) = (ceil(4294967300/257)) = 16711936 < 16777215 +or 4294967300/257 = 16711935 remainder 5 + +An actual Gram includes a GramHead + GramBody. So as long as the transport MTU +is at least as big as the GramHeadSize + GramBodySize then we can transport the maximum +memo size in MaxGramCount grams. + +The signed header for a gram is 160 bytes for all grams but the zeroth gram. +the zeroth gram includes an extra 4 bytes for the gramcount or 164 bytes. + +UDP IPV4 MTU = 576 Net safe payload = gram size = 548 +MaxGramCount for UDP GramSize of 548. Given signed header, GramBodySize is +548 - 160 = 388 which is larger than 257. So a memo of MaxMemoSize of 4294967300 +can be conveyed in IPV4 datagrams with fewer than MaxGramCount grams. +This provides headroom for the extra 4 bytes in the zeroth gram. + +UXD MTU is typically 65535 depending on the allocated buffer size which is much +greater than 257. + + +MinGramSize = MaxMemoSize // MaxGramCount = ceil(((2**30-1)*4+8 ) / (2**24-1)) = 257 = (2**8+1) +This is the min size that could result in maxMemoSize. Smaller gram sizes than 257 +can not reach MaxMemoSize because MaxGramCount not big enough. +Based on datagram constraints we have +MaxGramSize = (2**16-1) + + +The variables ghs, and ghns are fixed for given transport service type reliable +or unreliable with head and neck fields defined appropriately +The variables gs, gbs,and mms are derived from transport MTU size +The desired gs for a given transport MTU instance may be smaller than the allowed +maximum to accomodate buffers etc. +Given the fixed gram head size for service type, ghs one can calculate the +maximum memo size mms that includes the header overhead given the contraint of +no more than 2**24-1 grams in a given memo due to 24 size of gram count gram num. + +So for a given transport: +gs = min(gs, 2**16-1) (gram size) +gbs = gs - ghs (gram body size = gram size - gram head size) + +mms = min((2**30-1)*4+8, gbs * (2**24-1)) (max memo size) +So compare memo size to mms and if greater raise error and drop memo +otherwise partition (gram) memo into grams with headers and transmit + + +Therefore setting the maxmemosize to be (2**30-1)*4+8 enables the a big group +frame from CESR to be conveyed by a memo. +However a maximally sized memo can not be itself conveyed by a big CESR group frame +because maximally size memo is 8 more than the biggest frame body count. +So translating memos to CESR must check that the memo being translated iself +is limited to a size of (2**30-1)*4 not (2**30-1)*4+8 + +""" """Sizage: namedtuple for gram header part size entries in Memoer code tables -cs is the code part size int number of chars in code part -ms is the mid part size int number of chars in the mid part (memoID) -vs is the vid part size int number of chars in the vid part (verification ID) -ns is the neck part size int number of chars for the gram number in all grams - and the additional neck that also appears in first gram for gram count -ss is the signature part size int number of chars in the signature part. +cz is the serialization code part size int number of chars in code part +mz is the mid (memo ID) part size int number of chars in the mid part +oz is the oid (origin ID) part size int number of chars in the oid part +nz is the neck part size int number of chars for the gram number in all grams + it is also the size of the gram count that only appears in the zeroth gram + The zeroth gram has a long neck consiting of two nz sized fields + All other grams have ashort neck consisting of 1 nz sized field +sz is the signature part size int number of chars in the signature part the signature part is discontinuously attached after the body part but - its size is included in over head computation for the body size -hs is the head size int number of chars for other grams no-neck. - hs = cs + ms + vs + ns + ss. The size for the first gram with neck is - hs + ns + its size is included in the overhead size computation for the body size +hz is the head (header) size int number of chars + head with short neck + hz = cz + mz + oz + nz + sz + zeroth head with long neck is calculated from short neck head + zhz = hz + nz """ -Sizage = namedtuple("Sizage", "cs ms vs ss ns hs") +Sizage = namedtuple("Sizage", "cz mz oz nz sz hz") @dataclass(frozen=True) class GramCodex: @@ -78,41 +248,49 @@ def __iter__(self): class Memoer(hioing.Mixin): - """ - Memoer mixin base class to adds memogram support to a transport class. - Memoer supports asynchronous memograms. Provides common methods for subclasses. + """Memoer mixin base class to adds memogram support to a transport class. + Memoer supports memos composed of asynchronous memograms. + Provides common methods for subclasses. - A memogram is a higher level construct that sits on top of a datagram. + A memogram is a higher level construct that sits on top of a datagram transport. A memogram supports the segmentation and desegmentation of memos to respect the underlying datagram size, buffer, and fragmentation behavior. - Layering a reliable transaction protocol on top of a memogram enables reliable - asynchronous messaging over unreliable datagram transport protocols. + Layering a reliable transaction protocol for memos on top of a memogram + transport enables reliable asynchronous messaging over unreliable datagram + transport protocols. - When the datagram protocol is already reliable, then a memogram enables + When the datagram protocol is already reliable, then a memogram transport enables larger memos (messages) than that natively supported by the datagram. Usage: Do not instantiate directly but use as a mixin with a transport class in order to create a new subclass that adds memogram support to the - transport class. For example MemoerUdp or MemoerUxd + datagram transport class. For example MemoerUdp or MemoerUxd Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. - On the receive side each complete datagram (gram) is put in a gram receive - deque as a segment of a memo. These deques are indexed by the sender's - source addr. The grams in the gram recieve deque are then desegmented into a memo - and placed in the memo deque for consumption by the application or some other - higher level protocol. - - On the transmit side memos are placed in a deque (double ended queue). Each - memo is then segmented into grams (datagrams) that respect the size constraints + On the transmit side memos are placed in a memo deque (double ended queue). + Each memo is then segmented into grams (memograms) that respect the size constraints of the underlying datagram transport. These grams are placed in the outgoing gram deque. Each entry in this deque is a duple of form: - (gram: bytes, dst: str). Each duple is pulled off the deque and its + (gram: bytes, dst: str). Each duple is pulled off the self._serviceOneRxMemo()deque and its gram is put in bytearray for transport. + memo -> .txms deque -> rend -> grams -> .txgs deque -> send -> .txbs + + On the receive side each complete memogram (gram) is put in a gram receive + deque as a memogram (datagram sized) segment of a memo. + These deques are indexed by the sender's source addr. + The grams in the gram recieve deque are then desegmented into a memo + and placed in the memo deque for consumption by the application or some other + higher level protocol. + + receive -> (gram, src) -> grams parsed to .rxgs .counts .oids .sources -> + signing key pair sigkey and verifier key -> + fuse -> memo .rxms deque + When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation of the sent datagram which means the whole datagram is not sent at once via @@ -129,7 +307,8 @@ class Memoer(hioing.Mixin): Memo segmentation/desegmentation information is embedded in the grams. Inherited Class Attributes: - MaxGramSize (int): absolute max gram size on tx with overhead + MaxGramSize (int): absolute max gram size on tx with overhead, override + in subclass Class Attributes: Version (Versionage): default version consisting of namedtuple of form @@ -141,43 +320,50 @@ class Memoer(hioing.Mixin): Sizes (dict): gram head part sizes Sizage instances keyed by gram codes MaxMemoSize (int): absolute max memo size MaxGramCount (int): absolute max gram count + BufSize (int): used to set default buffer size for transport datagram buffers - Inherited Attributes: - name (str): unique name for Memoer transport. Used to manage. + Stubbed Attributes: + name (str): unique name for Memoer transport. + Used to manage multiple instances. opened (bool): True means transport open for use False otherwise - bc (int | None): count of transport buffers of MaxGramSize + bc (int|None): count of transport buffers of MaxGramSize + bs (int|None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize + + Stubbed Methods: + send(gram, dst, *, echoic=False) -> int # send gram over transport to dst + receive(self, *, echoic=False) -> (bytes, str|tuple|None) # receive gram Attributes: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - rxgs (dict): holding rx (receive) (data) gram deques of grams. - Each item in dict has key=src and val=deque of grames received - rxgs (dict): keyed by mid (memoID) with value of dict where each deque - holds incomplete memo grams for that mid. - The mid appears in every gram from the same memo. - The value dict is keyed by the gram number with value - that is the gram bytes. + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. sources (dict): keyed by mid (memoID) that holds the src for the memo. This enables reattaching src to fused memo in rxms deque tuple. counts (dict): keyed by mid (memoID) that holds the gram count from the first gram for the memo. This enables lookup of the gram count when fusing its grams. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + the oid to memo when placing fused memo in rxms deque. + oid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is origin id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: @@ -185,17 +371,10 @@ class Memoer(hioing.Mixin): portion when Encodesdatagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - echos (deque): holding echo receive duples for testing. Each duple of + echos (deque): holds echo receive duples for testing. Each duple of form: (gram: bytes, dst: str). - - - Hidden: - _code (bytes | None): see size property - _curt (bool): see curt property - _size (int): see size property - _verific (bool): see verific property - - + inbox (deque): holds final received complete memos for testing when not + overridden in subclass to further process otherwise Properties: code (bytes | None): gram code for gram header when rending for tx @@ -209,33 +388,330 @@ class Memoer(hioing.Mixin): MaxMemoSize. verific (bool): True means any rx grams must be signed. False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str)# singing key pair sigkey and verifier key + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter + keep (dict): labels are oids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") + oid (str|None): own oid defaults used to lookup keys to sign on tx - + Hidden: + _code (bytes | None): see size property + _curt (bool): see curt property + _size (int): see size property + _verific (bool): see verific property + _echoic (bool): see echoic property + _keep (dict): see keep property + _oid (str|None): see oid property """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex Codes = asdict(Codex) # map code name to code Names = {val : key for key, val in Codes.items()} # invert map code to code name Sodex = SGDex # signed gram codex - # dict of gram header part sizes keyed by gram codes: cs ms vs ss ns hs + # dict of gram header part sizes keyed by gram codes: cz mz oz nz sz hz Sizes = { - '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), - '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), + '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), + '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), } + + # Base2 Binary index representation of Text Base64 Char Codes #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) - MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size - MaxGramCount = 16777215 # (2**24-1) absolute max gram count - MaxGramSize = 65535 # (2**16-1) Overridden in subclass + # big enough to hold a CESR big frame with code + MaxMemoSize = 4294967300 # (2**30-1)*4+8 absolute max net memo payload size + MaxGramCount = 16777215 # (2**24-1) absolute max gram count + MaxGramSize = 65535 # (2**16-1) absolute max gram size overridden in subclass + BufSize = 65535 # (2**16-1) default buffersize + + @classmethod + def _encodeOID(cls, raw, code='B'): + """Utility method for use with signed headers that encodes raw oid + (origin ID) as CESR compatible fully qualified B64 text domain str + using CESR compatible text code + + Parameters: + raw (bytes): oid to be encoded with code + code (str): code for type of raw oid CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verkey basic derivation + Blake3_256: str = 'E' # Blake3 256 bit digest derivation. + + Returns: + qb64 (str): fully qualified base64 oid + """ + if code not in ('B', 'D', 'E'): + raise hioing.MemoerError(f"Invalid oid {code=}") + + rz = len(raw) # raw size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") + + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw oid + if len(code) != pz != 1: + raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" + f" {pz=} not equal 1") + + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified base64 oid with prefix code + + _, _, oz, _, _, _ = cls.Sizes[SGDex.Signed] # cz mz oz nz sz hz + if len(qb64) != oz: + hioing.MemoerError(f"Invalid oid qb64 size={len(qb64) != {oz}}") + + return qb64 # fully qualified base64 oid with prefix code + + + @classmethod + def _decodeOID(cls, qb64): + """Utility method for use with signed headers that decodes qualified + base64 oid to raw domain bytes from CESR compatible text code + + Parameters: + qb64 (str): qualified base64 oid to be decoded with code + code (str): code for type of oid CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verkey basic derivation + Blake3_256: str = 'E' # Blake3 256 bit digest derivation. + + Returns: + tuple(raw, code) where: + raw (bytes): oid suitable for crypto operations + code (str): CESR compatible code from qb64 + """ + cz = 1 # only support qb64 length 44 + code = qb64[:cz] + if code not in ('B', 'D', 'E'): + raise hioing.MemoerError(f"Invalid oid {code=}") + + qz = len(qb64) # text size + if qz != 44: + raise hioing.MemoerError(f"Invalid oid text size {qz=} not 44") + + cz = len(code) + pz = cz % 4 # net pad size given cz + if cz != pz != 1: # special case here for now we only accept cz=1 + raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {oid}") + + return raw, code + + + @classmethod + def _encodeQVK(cls, raw, code='B'): + """Utility method for use with signed headers that encodes raw verkey as + CESR compatible fully qualified B64 text domain str using CESR compatible + text code + + Parameters: + raw (bytes): verkey to be encoded with code + code (str): code for type of raw verkey CESR compatible + Ed25519N: str = 'B' # Ed25519 verkey non-transferable, basic derivation. + + Returns: + qb64 (str): fully qualified base64 verkey + """ + if code not in ('B'): + raise hioing.MemoerError(f"Invalid qvk {code=}") + + rz = len(raw) # raw size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") + + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw verkey + if len(code) != pz != 1: + raise hioing.MemoerError(f"Invalid code size={len(code)} " + f"not equal {pz=} not equal 1") + + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified verkey with prefix code + + return qb64 # qualified base64 verkey + + + @classmethod + def _encodeQSS(cls, raw, code='A'): + """Utility method for use with signed headers that encodes raw sigseed as + CESR compatible fully qualified B64 text domain str using CESR compatible + text code + + Parameters: + raw (bytes): sigseed to be encoded with code + code (str): code for type of raw sigseed CESR compatible + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key + + Returns: + qb64 (str): fully qualified base64 sigseed + """ + if code not in ('A'): + raise hioing.MemoerError(f"Invalid qss {code=}") + + rz = len(raw) # raw size + if rz != 32: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 32") + + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw sigseed + if len(code) != pz != 1: + raise hioing.MemoerError(f"Invalid code size={len(code)} " + f"not equal {pz=} not equal 1") + + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, aqb64prestrip + + qb64 = code + b64.decode() # fully qualified with prefix code + + return qb64 # qualified base64 sigseed + + + @classmethod + def _decodeQSS(cls, qb64): + """Utility method for use with signed headers that decodes qualified + base64 sigseed to raw domain bytes from CESR compatible text code + + Parameters: + qb64 (str): qualified base64 sigseed to be decoded with code + code (str): code for type of raw sigseed CESR compatible + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key + + Returns: + tuple(raw, code) where: + raw (bytes): sigseed suitable for crypto operations + code (str): CESR compatible code from qb64 + """ + cz = 1 # only support qb64 length 44 + code = qb64[:cz] + if code not in ('A'): + raise hioing.MemoerError(f"Invalid qss {code=}") + + qz = len(qb64) # text size + if qz != 44: + raise hioing.MemoerError(f"Invalid qss text size {qz=} not 44") + + cz = len(code) + pz = cz % 4 # net pad size given cz + if cz != pz != 1: # special case here for now we only accept cz=1 + raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {qss}") + + return raw, code + + + @classmethod + def _encodeSig(cls, raw, code='0B'): + """Utility method for use with signed headers that encodes raw signature + as CESR compatible fully qualified B64 text domain str using CESR + compatible text code + + Parameters: + raw (bytes): sig to be encoded with code + code (str): code for type of raw sig CESR compatible + Ed25519_Sig: str = '0B' # Ed25519 signature. not indexed + + Returns: + qb64 (str): fully qualified base64 sig + """ + if code not in ('0B'): + raise hioing.MemoerError(f"Invalid sig {code=}") + + rz = len(raw) # raw size + if rz != 64: + raise hioing.MemoerError(f"Invalid raw size {rz=} not 64") + + pz = (3 - ((rz) % 3)) % 3 # net pad size for raw + if len(code) != pz != 2: + raise hioing.MemoerError(f"Invalid code size={len(code)} not equal" + f" {pz=} not equal 2") + + b64 = encodeB64(bytes([0] * pz) + raw)[pz:] # prepad, convert, and prestrip + + qb64 = code + b64.decode() # fully qualified base64 with prefixqb64de + + _, _, _, _, sz, _ = cls.Sizes[SGDex.Signed] # cz mz oz nz sz hz + if len(qb64) != sz: + hioing.MemoerError(f"Invalid sig qb64 size={len(qb64) != {sz}}") + + return qb64 # fully qualified base64 oid with prefix code + + + @classmethod + def _decodeSig(cls, qb64): + """Utility method for use with signed headers that decodes qualified + base64 sig to raw domain bytes from CESR compatible text code + + Parameters: + qb64 (str): qualified base64 sig to be decoded with code + code (str): code for type of raw sig CESR compatible + Ed25519_Sig: str = '0B' # Ed25519 signature. not indexed + + Returns: + tuple(raw, code) where: + raw (bytes): signature suitable for crypto operations + code (str): CESR compatible code from qb64 + """ + cz = 2 # only support qb64 length 88 + code = qb64[:cz] + if code not in ('0B'): + raise hioing.MemoerError(f"Invalid sig {code=}") + + qz = len(qb64) # text size + if qz != 88: + raise hioing.MemoerError(f"Invalid sig text size {qz=} not 88") + + pz = cz % 4 # net pad size given cz + if pz != cz: + raise hioing.MemoerError(f"Invalid {pz=} not equal {cz=}") + + base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + paw = decodeB64(base) # now should have pz leading sextexts of zeros + raw = paw[pz:] # remove prepad midpad bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:pz], "big") + if pi != 0: + raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") + + if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + raise hioing.MemoerError(f"Improperly qualified material = {sig}") + + return raw, code # qualified base64 sigseed def __init__(self, *, name=None, bc=None, + bs=None, version=None, rxgs=None, sources=None, counts=None, - vids=None, + oids=None, rxms=None, txms=None, txgs=None, @@ -244,6 +720,9 @@ def __init__(self, *, curt=False, size=None, verific=False, + echoic=False, + keep=None, + oid=None, **kwa ): """Setup instance @@ -253,6 +732,10 @@ def __init__(self, *, opened (bool): True means opened for transport False otherwise bc (int | None): count of transport buffers of MaxGramSize + bs (int | None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize Parameters: version (Versionage): version for this memoir instance consisting of @@ -269,20 +752,21 @@ def __init__(self, *, memo indexed by its mid (memoID). This enables lookup of the gram count for a given memo to know when it has received all its constituent grams in order to fuse back into the memo. - vids (dict[mid: (vid | None)]): keyed by mid that holds the verification ID str for - the memo indexed by its mid (memoID). This enables reattaching - the vid to memo when placing fused memo in rxms deque. - Vid is only present when signed header otherwise vid is None + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID + str for the memo indexed by its mid (memoID). + This enables reattaching the oid to memo when placing fused memo + in rxms deque. + oid is only present when signed header otherwise oid is None rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams each entry in deque is tuple of form: - (memo: str, src: str, vid: str) where: - memo is fused memo, src is source addr, vid is verification ID + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is tuple of form - (memo: str, dst: str, vid: str | None) + (memo: str, dst: str, oid: str | None) memo is memo to be partitioned into gram dst is dst addr for grams - vid is verification id when gram is to be signed or None otherwise + oid is origin id when gram is to be signed or None otherwise txgs (deque): grams to transmit, each entry is duple of form: (gram: bytes, dst: str). Grams include gram headers. txbs (tuple): current transmisstion duple of form: @@ -301,6 +785,20 @@ def __init__(self, *, MaxMemoSize. verific (bool): True means any rx grams must be signed. False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter + keep (dict|None): labels are oids and values are Keyage instances + that provide current signature key pair for oid + this is a lightweight mechanism that should be + overridden in subclass for real world key management. + oid (str|None): own oid defaults used to lookup keys to sign on tx + """ # initialize attributes @@ -308,7 +806,7 @@ def __init__(self, *, self.rxgs = rxgs if rxgs is not None else dict() self.sources = sources if sources is not None else dict() self.counts = counts if counts is not None else dict() - self.vids = vids if vids is not None else dict() + self.oids = oids if oids is not None else dict() self.rxms = rxms if rxms is not None else deque() self.txms = txms if txms is not None else deque() @@ -316,13 +814,14 @@ def __init__(self, *, self.txbs = txbs if txbs is not None else (bytearray(), None) self.echos = deque() # only used in testing as echoed tx + self.inbox = deque() # holds complete receive memos for testing self.code = code self.curt = curt self.size = size # property sets size given .code and constraints self._verific = True if verific else False - super(Memoer, self).__init__(name=name, bc=bc, **kwa) + super(Memoer, self).__init__(name=name, bc=bc, bs=bs, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. self.name = name if name is not None else "main" # mixed with subclass should provide this. @@ -331,8 +830,17 @@ def __init__(self, *, self.opened = False # mixed with subclass should provide this. if not hasattr(self, "bc"): # stub so mixin works in isolation. - self.bc = bc if bc is not None else 4 # mixed with subclass should provide this. + self.bc = bc # mixin subclass should provide this. + + if not hasattr(self, "bs"): # stub so mixin works in isolation. + self.bs = bs # mixin subclass should provide this. + + if self.bs is None: # default if not provided by mixin + self.bs = self.BufSize + self._echoic = True if echoic else False + self._keep = keep if keep is not None else dict() + self.oid = oid if oid else None @property def code(self): @@ -406,14 +914,14 @@ def size(self, size): size (int | None): gram size for rending memo """ - _, _, _, _, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + _, _, _, nz, _, hz = self.Sizes[self.code] # cz mz oz nz sz hz size = size if size is not None else self.MaxGramSize if self.curt: # minimum header smaller when in base2 curt - hs = 3 * hs // 4 - ns = 3 * ns // 4 + hz = 3 * hz // 4 + nz = 3 * nz // 4 # mininum size must be big enough for first gram header and 1 body byte - self._size = max(min(size, self.MaxGramSize), hs + ns + 1) + self._size = max(min(size, self.MaxGramSize), hz + nz + 1) @property @@ -426,6 +934,53 @@ def verific(self): """ return self._verific + @property + def echoic(self): + """Property getter for ._echoic + + Returns: + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + """ + return self._echoic + + @property + def keep(self): + """Property getter for ._keep + + Returns: + keep (dict): labels are oids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") + """ + return self._keep + + @property + def oid(self): + """Property getter for ._oid + + Returns: + oid (str|None): oid used to sign on tx if any + """ + return self._oid + + + @oid.setter + def oid(self, oid): + """Property setter for ._oid + + Parameters: + oid (str|None): value to assign to own oid + """ + self._oid = oid + def open(self): """Opens transport in nonblocking mode @@ -457,7 +1012,6 @@ def wiff(self, gram): """Determines encoding of gram bytes header when parsing grams. The encoding maybe either base2 or base64. - Returns: curt (bool): True means base2 encoding False means base64 encoding @@ -489,7 +1043,7 @@ def wiff(self, gram): the count from 0 to 63. There is one collision, however, between Base2 and Base 64 for invalid gram headers. This is because 0o27 is the base2 code for ascii 'V'. This means that Base2 encodings - that begin 0o27 witch is the Base2 encoding of the ascii 'V, would be + that begin with 0o27 wich is the Base2 encoding of the ascii 'V, would be incorrectly detected as text not binary. """ @@ -502,8 +1056,9 @@ def wiff(self, gram): raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") - def verify(self, sig, ser, vid): - """Verify signature sig on signed part of gram, ser, using key from vid. + def verify(self, sig, ser, oid): + """Verify signature sig on signed part of gram, ser, using current verkey + for oid. Must be overriden in subclass to perform real signature verification. This is a stub. @@ -514,13 +1069,13 @@ def verify(self, sig, ser, vid): Parameters: sig (bytes | str): qualified base64 qb64b ser (bytes): signed portion of gram in delivered format - vid (bytes | str): qualified base64 qb64b of verifier ID + oid (bytes | str): qualified base64 qb64b of origin ID """ if hasattr(sig, 'encode'): sig = sig.encode() - if hasattr(vid, 'encode'): - vid = vid.encode() + if hasattr(oid, 'encode'): + oid = oid.encode() return True @@ -531,14 +1086,14 @@ def pick(self, gram): Returns: result (tuple): tuple of form: - (mid: str, vid: str, gn: int, gc: int | None) where: + (mid: str, oid: str, gn: int, gc: int | None) where: mid is fully qualified memoID, - vid is verID, + oid is origin ID used to look up signature verification key, gn is gram number, gc is gram count. - When first gram (zeroth) returns (mid, vid, 0, gc). - When other gram returns (mid, vid, gn, None) - When code has empty vid then vid is None + When first gram (zeroth) returns (mid, oid, 0, gc). + When other gram returns (mid, oid, gn, None) + When code has empty oid then oid is None Otherwise raises MemoerError error. When valid recognized header, strips header bytes from front of gram @@ -559,38 +1114,37 @@ def pick(self, gram): raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - cms = 3 * (cs + ms) // 4 # cs + ms are aligned on 24 bit boundary - hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid + cms = 3 * (cz + mz) // 4 # cz + mz are aligned on 24 bit boundary + hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 + sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 - if len(gram) < (hs + 1): # not big enough for non-first gram + if len(gram) < (hz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" - f" < {hs + 1}.") + f" < {hz + 1}.") - #mid = encodeB64(bytes([0] * ps) + gram[cs:cms])[ps:].decode() # prepad, convert, strip mid = encodeB64(gram[:cms]).decode() # fully qualified with prefix code - vid = encodeB64(gram[cms:cms+vs]).decode() # must be on 24 bit boundary - gn = int.from_bytes(gram[cms+vs:cms+vs+ns]) - if gn == 0: # first (zeroth) gram so get neck - if len(gram) < hs + ns + 1: + oid = encodeB64(gram[cms:cms+oz]).decode() # must be on 24 bit boundary + gn = int.from_bytes(gram[cms+oz:cms+oz+nz]) + if gn == 0: # first (zeroth) gram so long neck + if len(gram) < hz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b2 " - f"gram < {hs + ns + 1}.") - neck = gram[cms+vs+ns:cms+vs+2*ns] # slice takes a copy + f"gram < {hz + nz + 1}.") + neck = gram[cms+oz+nz:cms+oz+2*nz] # slice takes a copy gc = int.from_bytes(neck) # convert to int - sig = encodeB64(gram[-ss if ss else len(gram):]) # last ss bytes are signature - del gram[-ss if ss else len(gram):] # strip sig + sig = encodeB64(gram[-sz if sz else len(gram):]) # last ss bytes are signature + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss+ns] # strip of fore head leaving body in gram - else: # non-first gram no neck + del gram[:hz-sz+nz] # strip of fore head leaving body in gram + else: # non-zeroth gram so short neck gc = None - sig = encodeB64(gram[-ss if ss else len(gram):]) - del gram[-ss if ss else len(gram):] # strip sig + sig = encodeB64(gram[-sz if sz else len(gram):]) + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss] # strip of fore head leaving body in gram + del gram[:hz-sz] # strip of fore head leaving body in gram else: # base64 text encoding if len(gram) < 2: # assumes len(code) must be 2 @@ -600,58 +1154,69 @@ def pick(self, gram): if self.verific and code not in self.Sodex: # must be signed raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs + cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz - if len(gram) < (hs + 1): # not big enough for non-first gram + if len(gram) < (hz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b64 gram" - f" < {hs + 1}.") + f" < {hz + 1}.") - mid = bytes(gram[:cs+ms]).decode() # fully qualified with prefix code - vid = bytes(gram[cs+ms:cs+ms+vs]).decode() # must be on 24 bit boundary - gn = helping.b64ToInt(gram[cs+ms+vs:cs+ms+vs+ns]) - if gn == 0: # first (zeroth) gram so get neck - if len(gram) < hs + ns + 1: + mid = bytes(gram[:cz+mz]).decode() # fully qualified with prefix code + oid = bytes(gram[cz+mz:cz+mz+oz]).decode() # must be on 24 bit boundary + gn = helping.b64ToInt(gram[cz+mz+oz:cz+mz+oz+nz]) + if gn == 0: # first (zeroth) gram so long neck + if len(gram) < hz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b64 " - f"gram < {hs + ns + 1}.") - neck = gram[cs+ms+vs+ns:cs+ms+vs+2*ns] # slice takes a copy + f"gram < {hz + nz + 1}.") + neck = gram[cz+mz+oz+nz:cz+mz+oz+2*nz] # slice takes a copy gc = helping.b64ToInt(neck) # convert to int - sig = bytes(gram[-ss if ss else len(gram):]) # last ss bytes are signature - del gram[-ss if ss else len(gram):] # strip sig + sig = bytes(gram[-sz if sz else len(gram):]) # last sz bytes signature + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss+ns] # strip of fore head leaving body in gram - else: # non-first gram no neck + del gram[:hz-sz+nz] # strip of fore head leaving body in gram + else: # non-zeroth gram short neck gc = None - sig = bytes(gram[-ss if ss else len(gram):]) - del gram[-ss if ss else len(gram):] # strip sig + sig = bytes(gram[-sz if sz else len(gram):]) + del gram[-sz if sz else len(gram):] # strip sig signed = bytes(gram[:]) # copy signed portion of gram - del gram[:hs-ss] # strip of fore head leaving body in gram + del gram[:hz-sz] # strip of fore head leaving body in gram - if sig: # signature not empty - if not self.verify(sig, signed, vid): + if sig: # signature not emptyoid + if not self.verify(sig, signed, oid): raise hioing.MemoerError(f"Invalid signature on gram from " - f"verifier {vid=}.") + f"verifier {oid=}.") - return (mid, vid if vid else None, gn, gc) + return (mid, oid if oid else None, gn, gc) - def receive(self, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. + def receive(self, *, echoic=False) -> (bytes, str|tuple|None): + """Attemps to receive bytes from remote source. + + May use echoic=True and .echos to mock a transport layer for testing + Puts sent gram into .echos so that .recieve can extract it when using + same Memoer to send and recieve to itself via its own .echos + When using different Memoers each for send and recieve then must + manually copy from sender Memoer .echos to Receiver Memoer .echos + Must be overridden in subclass. This is a stub to define mixin interface. Parameters: - echoic (bool): True means use .echos in .receive debugging purposes - where echo is duple of form: (gram: bytes, src: str) - False means do not use .echos default is duple that + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that indicates nothing to receive of form (b'', None) + Returns: - duple (tuple): of form (data: bytes, src: str) where data is the + duple (tuple): of form (data: bytes, src: str|tuple|None) where data is the bytes of received data and src is the source address. When no data the duple is (b'', None) unless echoic is True then pop off echo from .echos """ - + echoic = echoic or self.echoic # use parm when True else default .echoic if echoic: try: result = self.echos.popleft() @@ -702,10 +1267,10 @@ def _serviceOneReceived(self, *, echoic=False): if not gram: # no received data return False # so try again later - gram = bytearray(gram)# make copy bytearray so can strip off header + gram = bytearray(gram) # make copy bytearray so can strip off header try: - mid, vid, gn, gc = self.pick(gram) # parse and strip off head leaving body + mid, oid, gn, gc = self.pick(gram) # parse and strip off head leaving body except hioing.MemoerError as ex: # invalid gram so drop logger.error("Unrecognized Memoer gram from %s.\n %s.", src, ex) return True # did receive data so can try again now @@ -721,8 +1286,8 @@ def _serviceOneReceived(self, *, echoic=False): if mid not in self.counts: # make idempotent first only no replay self.counts[mid] = gc # save gram count for mid - if mid not in self.vids: - self.vids[mid] = vid + if mid not in self.oids: + self.oids[mid] = oid # assumes unique mid across all possible sources. No replay by different # source only first source for a given mid is ever recognized @@ -797,11 +1362,11 @@ def _serviceOnceRxGrams(self): continue memo = self.fuse(self.rxgs[mid], self.counts[mid]) if memo is not None: # allows for empty "" memo for some src - self.rxms.append((memo, self.sources[mid], self.vids[mid])) + self.rxms.append((memo, self.sources[mid], self.oids[mid])) del self.rxgs[mid] del self.counts[mid] del self.sources[mid] - del self.vids[mid] + del self.oids[mid] def serviceRxGramsOnce(self): @@ -838,7 +1403,8 @@ def serviceRxMemosOnce(self): Override in subclass to handle result and put it somewhere """ try: - memo, src, vid = self._serviceOneRxMemo() + #memo, src, oid = self._serviceOneRxMemo() + self.inbox.append(self._serviceOneRxMemo()) except IndexError: pass @@ -849,7 +1415,8 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - memo, src, vid = self._serviceOneRxMemo() + #memo, src, oid = self._serviceOneRxMemo() + self.inbox.append(self._serviceOneRxMemo()) def serviceAllRxOnce(self): @@ -868,57 +1435,61 @@ def serviceAllRx(self): self.serviceRxMemos() - def send(self, txbs, dst, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. - Must be overridden in subclass. - This is a stub to define mixin interface. - - Returns: - count (int): bytes actually sent - - Parameters: - txbs (bytes | bytearray): of bytes to send - dst (str): remote destination address - echoic (bool): True means echo sends into receives via. echos - False measn do not echo - """ - if echoic: - self.echos.append((bytes(txbs), dst)) # make copy - return len(txbs) # all sent - - - - def memoit(self, memo, dst, vid=None): - """Append (memo, dst, vid) tuple to .txms deque + def memoit(self, memo, dst, oid=None): + """Append (memo, dst, oid) tuple to .txms deque Parameters: memo (str): to be segmented and packed into gram(s) dst (str): address of remote destination of memo - vid (str | None): verifiable ID for signing grams + oid (str | None): origin ID for verifying signature on grams """ - self.txms.append((memo, dst, vid)) + self.txms.append((memo, dst, oid)) - def sign(self, ser, vid): - """Sign serialization ser using private key for verifier ID vid - Must be overriden in subclass to fetch private key for vid and sign. + def sign(self, ser, oid): + """Sign serialization ser using private sigkey for origin ID oid + Must be overriden in subclass to fetch private key for oid and sign. This is a stub. Returns: - sig(bytes): qb64b qualified base64 representation of signature + sig (bytes): qb64b or qb2 when .curt + if not .curt then qualified base64 of signature + else if .curt then qualified base2 of signature + raises MemoerError if problem signing Parameters: ser (bytes): signed portion of gram is delivered format - vid (str | bytes): qualified base64 qb64 of verifier ID + oid (bytes): qb64 or qb2 if .curt of oid of signer + assumes oid of correct length + + Ed25519_Seed:str = 'A' # Ed25519 256 bit random seed for private key """ - if hasattr(vid, 'encode'): - vid = vid.encode() - cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs - return b'A' * ss + if oid not in self.keep: + raise hioing.MemoerError("Invalid {oid=} for signing") + + qvk, qss = self.keep[oid] + sigseed, code = self._decodeQSS(qss) # raises MemoerError if problem + if code not in ('A'): + raise hioing.MemoerError(f"Invalid sigseed algorithm type {code=}") + + try: + import pysodium + except ImportError as ex: + raise hioing.MemoerError("Missing pysodium lib for signing") from ex + + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) + raw = pysodium.crypto_sign_detached(ser, sigkey) # raw sig + sig = self._encodeSig(raw) # raise MemoerError if problem + sig = sig.encode() # make bytes + + if self.curt: + sig = decodeB64(sig) # make qb2 + return sig - def rend(self, memo, vid=None): + + def rend(self, memo, oid=None): """Partition memo into packed grams with headers. Returns: @@ -926,36 +1497,42 @@ def rend(self, memo, vid=None): Parameters: memo (str): to be partitioned into grams with headers - vid (str | None): verification ID when gram is to be signed. + oid (str | None): origin ID when gram is to be signed, used to + lookup sigkey to sign. None means not signable - Note first gram has head + neck overhead, hs + ns so bs is smaller by ns - non-first grams have just head overhead hs so bs is bigger by ns + Note zeroth gram has head + neck overhead, zhz = hz + nz + so bz that fits is smaller by nz relative to non-zeroth + non-zeroth grams just head overhead hz + so bz that fits is bigger by nz relative to zeroth """ grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray # self.size is max gram size - cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - if vs and (not vid or len(vid) != vs): - raise hioing.MemoerError(f"Invalid {vid=} for size={vs}.") + cz, mz, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz + + oid = oid if oid is not None else self.oid - vid = b"" if vid is None else vid[:vs].encode() + if oz and (not oid or len(oid) != oz): + raise hioing.MemoerError(f"Missing or invalid {oid=} for {oz=}") + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned - mid = encodeB64(bytes([0] * ps) + uuid.uuid4().bytes)[ps:] # prepad, convert, and strip + mid = encodeB64(bytes([0] * pz) + uuid.uuid1().bytes)[pz:] #pzrepad, convert, and prestrip + if cz != pz or cz != len(self.code): + raise hioing.MemoerError(f"Invalid code size {cz=} for {pz=} or " + f"code={self.code}") mid = self.code.encode() + mid # fully qualified mid with prefix code ml = len(memo) if self.curt: # rend header parts in base2 instead of base64 - hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 + hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 + sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 mid = decodeB64(mid) - vid = decodeB64(vid) - bs = (self.size - hs) # max standard gram body size without neck + bz = (self.size - hz) # max standard gram body size without neck # compute gram count based on overhead note added neck overhead in first gram # first gram is special its header is longer by ns than the other grams # which means its payload body is shorter by ns than the other gram bodies @@ -964,37 +1541,42 @@ def rend(self, memo, vid=None): # (ceil) to get cnt of other grams and add 1 for the first gram to get # total gram cnt. # gc = ceil((ml-(bs-ns))/bs + 1) = ceil((ml-bs+ns)/bs + 1) - gc = math.ceil((ml-bs+ns)/bs+1) # includes added neck ns overhead - mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload + gc = math.ceil((ml-bz+nz)/bz+1) # includes added neck ns overhead + mms = min(self.MaxMemoSize, (bz * self.MaxGramCount) - nz) # max memo payload if ml > mms: raise hioing.MemoerError(f"Memo length={ml} exceeds max={mms}.") if self.curt: - neck = gc.to_bytes(ns) + neck = gc.to_bytes(nz) else: - neck = helping.intToB64b(gc, l=ns) + neck = helping.intToB64b(gc, l=nz) gn = 0 while memo: if self.curt: - num = gn.to_bytes(ns) # num size must always be neck size + num = gn.to_bytes(nz) # num size must always be neck size else: - num = helping.intToB64b(gn, l=ns) # num size must always be neck size + num = helping.intToB64b(gn, l=nz) # num size must always be neck size - head = mid + vid + num + if oz: # need oid part, but can't mod here may need below to sign + if self.curt: + oidp = decodeB64(oid.encode()) # oid part b2 + else: + oidp = oid.encode() # oid part b64 bytes + head = mid + oidp + num + else: + head = mid + num if gn == 0: - gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end - del memo[:bs-ns] # del slice past end just deletes to end + gram = head + neck + memo[:bz-nz] # copy slice past end just copies to end + del memo[:bz-nz] # del slice past end just deletes to end else: - gram = head + memo[:bs] # copy slice past end just copies to end - del memo[:bs] # del slice past end just deletes to end + gram = head + memo[:bz] # copy slice past end just copies to end + del memo[:bz] # del slice past end just deletes to end - if ss: # sign - sig = self.sign(gram, vid) - if self.curt: - sig = decodeB64(sig) + if sz: # signed gram, .sign returns proper sig format when .curt + sig = self.sign(gram, oid) # raises MemoerError if invalid gram = gram + sig grams.append(gram) @@ -1004,12 +1586,49 @@ def rend(self, memo, vid=None): return grams + def send(self, gram, dst, *, echoic=False) -> int: + """Attemps to send bytes in txbs to remote destination dst. + + May use echoic=True and .echos to mock a transport layer for testing + Puts sent gram into .echos so that .recieve can extract it when using + same Memoer to send and recieve to itself via its own .echos + When using different Memoers each for send and recieve then must + manually copy from sender Memoer .echos to Receiver Memoer .echos + + Must be overridden in subclass. + This is a stub to define mixin interface. + + Returns: + count (int): bytes actually sent + + Parameters: + gram (bytearray): of bytes to send + dst (str): remote destination address + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + + """ + echoic = echoic or self.echoic # use parm when True else default .echoic + if echoic: + self.echos.append((bytes(gram), dst)) # make copy + + # for example: cnt = self.ls.sendto(gram, dst) + # cnt == int number of bytes sent + cnt = len(gram) # all sent + return cnt # bytes sent + + def _serviceOneTxMemo(self): - """Service one (memo, dst, vid) tuple from .txms deque where tuple is - of form: (memo: str, dst: str, vid: str) where: + """Service one (memo, dst, oid) tuple from .txms deque where tuple is + of form: (memo: str, dst: str, oid: str) where: memo is outgoing memo dst is destination address - vid is verification ID + oid is origin ID used to lookup sigkey to sign Calls .rend method to process the partitioning and packing as appropriate to convert memo into grams with headers and sign when @@ -1017,9 +1636,9 @@ def _serviceOneTxMemo(self): Appends (gram, dst) duple to .txgs deque. """ - memo, dst, vid = self.txms.popleft() # raises IndexError if empty deque + memo, dst, oid = self.txms.popleft() # raises IndexError if empty deque - for gram in self.rend(memo, vid): # partition memo into gram parts with head + for gram in self.rend(memo, oid): # partition memo into gram parts with head self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) @@ -1041,7 +1660,7 @@ def serviceTxMemos(self): def gramit(self, gram, dst): - """Append (gram, dst) duple to .txgs deque + """Append (gram, dst) duple to .txgs deque. Utility method for testing. Parameters: gram (bytes): gram to be sent @@ -1058,12 +1677,13 @@ def _serviceOnceTxGrams(self, *, echoic=False): echoic (bool): True means echo sends into receives via. echos False measn do not echo - Each entry in.txgs is duple of form: (gram: bytes, dst: str) + Each entry in .txgs is duple of form: (gram: bytes, dst: str) where: gram (bytes): is outgoing gram segment from associated memo dst (str): is far peer destination address - .txbs is duple of form: (gram: bytearray, dst: str) + The latest gram from .txgs is put in .txbs whose value is duple of form: + (gram: bytearray, dst: str) bytearray so can partial send where: gram (bytearray): holds incompletly sent gram portion if any dst (str | None): destination or None if last completely sent @@ -1085,17 +1705,17 @@ def _serviceOnceTxGrams(self, *, echoic=False): send remainder of a datagram when using nonblocking sends. When the far side peer is unavailable the gram is dropped. This means - that unreliable transports need to have a timeour retry mechanism. + that unreliable transports need to have a timeout retry mechanism. Internally, a dst of None in the .txbs duple indicates its ok to take another gram from the .txgs deque if any and start sending it. """ - gram, dst = self.txbs - if dst == None: + gram, dst = self.txbs # split out saved partial send if any + if dst == None: # no partial send remaining so get new gram try: - gram, dst = self.txgs.popleft() - gram = bytearray(gram) + gram, dst = self.txgs.popleft() # next gram + gram = bytearray(gram) # ensure bytearray except IndexError: return False # nothing more to send, return False to try later @@ -1127,8 +1747,8 @@ def _serviceOnceTxGrams(self, *, echoic=False): if cnt: del gram[:cnt] # remove from buffer those bytes sent if not gram: # all sent - dst = None # indicate by setting dst to None - self.txbs = (gram, dst) # update txbs to indicate if completely sent + dst = None # done indicated by setting dst to None + self.txbs = (gram, dst) # update .txbs to indicate if completely sent return (False if dst else True) # incomplete return False, else True @@ -1160,6 +1780,7 @@ def serviceTxGrams(self, *, echoic=False): break # try again later + def serviceAllTxOnce(self): """Service the transmit side once (non-greedy) one transmission. """ @@ -1276,29 +1897,124 @@ def exit(self): -class TymeeMemoer(tyming.Tymee, Memoer): - """TymeeMemoer mixin base class to add tymer support for unreliable transports - that need retry tymers. Subclass of tyming.Tymee +class SureMemoer(Tymee, Memoer): + """SureMemoer mixin base class that supports reliable (sure) memo delivery + over unreliable datagram transports. These need retry tymers and acknowledged + delivery services. + Subclass of Tymee and Memoer - - Inherited Class Attributes: - see superclass + Inherited Class Attributes (Memoer): + Version (Versionage): default version consisting of namedtuple of form + (major: int, minor: int) + Codex (GramDex): dataclass ref to gram codex + Codes (dict): maps codex names to codex values + Names (dict): maps codex values to codex names + Sodex (SGDex): dataclass ref to signed gram codex + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes + MaxMemoSize (int): absolute max memo size + MaxGramCount (int): absolute max gram count + BufSize (int): used to set default buffer size for transport datagram buffers Class Attributes: Tymeout (float): default timeout for retry tymer(s) if any - Inherited Attributes: - see superclass + Inherited Stubbed Attributes (Memoer): + name (str): unique name for Memoer transport. + Used to manage multiple instances. + opened (bool): True means transport open for use + False otherwise + bc (int|None): count of transport buffers of MaxGramSize + bs (int|None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize - Attributes: - tymeout (float): default timeout for retry tymer(s) if any + Inherited Stubbed Methods (Memoer): + send(gram, dst, *, echoic=False) -> int # send gram over transport to dst + receive(self, *, echoic=False) -> (bytes, str|tuple|None) # receive gram + Inherited Attributes (Memoer): + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. + sources (dict): keyed by mid (memoID) that holds the src for the memo. + This enables reattaching src to fused memo in rxms deque tuple. + counts (dict): keyed by mid (memoID) that holds the gram count from + the first gram for the memo. This enables lookup of the gram count when + fusing its grams. + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for + the memo indexed by its mid (memoID). This enables reattaching + the oid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise oid is None + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is tuple of form: + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is tuple of form + (memo: str, dst: str, oid: str | None) + memo is memo to be partitioned into gram + dst is dst addr for grams + oid is origin id when gram is to be signed or None otherwise + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when Encodesdatagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) + echos (deque): holding echo receive duples for testing. Each duple of + form: (gram: bytes, dst: str). + Attributes: + tymeout (float): default timeout for retry tymer(s) if any + tymers (dict): keys are tid and values are Tymers for retry tymers for + each inflight tx + + Inherited Properties (Tymee): + tyme (float | None): relative cycle time of associated Tymist which is + provided by calling .tymth function wrapper closure which is obtained + from Tymist.tymen(). + None means not assigned yet. + tymth (Callable | None): function wrapper closure returned by + Tymist.tymen() method. When .tymth is called it returns associated + Tymist.tyme. Provides injected dependency on Tymist cycle tyme base. + None means not assigned yet. + + Inherited Properties (Memoer): + code (bytes | None): gram code for gram header when rending for tx + curt (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = over head size + neck size + body size. + other gram size = over head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. + verific (bool): True means any rx grams must be signed. + False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter + keep (dict): labels are oids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") + oid (str|None): own oid defaults used to lookup keys to sign on tx """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout - def __init__(self, *, tymeout=None, **kwa): + def __init__(self, *, tymeout=None, code=GramDex.Signed, verific=True, **kwa): """ Initialization method for instance. Inherited Parameters: @@ -1308,9 +2024,10 @@ def __init__(self, *, tymeout=None, **kwa): tymeout (float): default for retry tymer if any """ - super(TymeeMemoer, self).__init__(**kwa) + super(SureMemoer, self).__init__(code=code, verific=verific, **kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout - #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer + self.tymers = {} + #Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer def wind(self, tymth): @@ -1318,8 +2035,9 @@ def wind(self, tymth): Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoer, self).wind(tymth) - #self.tymer.wind(tymth) + super(SureMemoer, self).wind(tymth) # wind Tymee superclass + for tid, tymer in self.tymers.items(): + tymer.wind(tymth) def serviceTymers(self): """Service all retry tymers @@ -1354,9 +2072,8 @@ def serviceAll(self): @contextmanager -def openTM(cls=None, name="test", **kwa): - """ - Wrapper to create and open TymeeMemoer instances +def openSM(cls=None, name="test", **kwa): + """Wrapper to create and open SureMemoer instances When used in with statement block, calls .close() on exit of with block Parameters: @@ -1364,17 +2081,17 @@ def openTM(cls=None, name="test", **kwa): name (str): unique identifer of Memoer peer. Enables management of transport by name. Usage: - with openTM() as peer: + with openSM() as peer: peer.receive() - with openTM(cls=MemoerSub) as peer: + with openSM(cls=SureMemoerSub) as peer: peer.receive() """ peer = None if cls is None: - cls = TymeeMemoer + cls = SureMemoer try: peer = cls(name=name, **kwa) peer.reopen() @@ -1386,13 +2103,14 @@ def openTM(cls=None, name="test", **kwa): peer.close() -class TymeeMemoerDoer(doing.Doer): - """TymeeMemoer Doer for unreliable transports that require retry tymers. +class SureMemoerDoer(doing.Doer): + """SureMemoerDoer Doer to provide reliable delivery over unreliable + datagram transports. This requires retry tymers and acknowldged services. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (TymeeMemoer) is underlying transport instance subclass of TymeeMemoer + .peer (SureMemoer) is underlying transport instance subclass of SureMemoer """ @@ -1400,9 +2118,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (TymeeMemoer): subclass instance + peer (SureMemoer): subclass instance """ - super(TymeeMemoerDoer, self).__init__(**kwa) + super(SureMemoerDoer, self).__init__(**kwa) self.peer = peer @@ -1411,8 +2129,9 @@ def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoerDoer, self).wind(tymth) - self.peer.wind(tymth) + super(SureMemoerDoer, self).wind(tymth) # wind this doer + self.peer.wind(tymth) # wind its peer + def enter(self, *, temp=None): @@ -1427,10 +2146,10 @@ def enter(self, *, temp=None): Doist or DoDoer winds its doers on enter """ - # inject temp into file resources here if any if self.tymth: - self.peer.wind(self.tymth) - self.peer.reopen(temp=temp) + self.peer.wind(self.tymth) # Doist or DoDoer winds its doers on enter + + self.peer.reopen(temp=temp) # inject temp into file resources here if any def recur(self, tyme): diff --git a/src/hio/core/udp/peermemoing.py b/src/hio/core/udp/peermemoing.py new file mode 100644 index 00000000..63dfabca --- /dev/null +++ b/src/hio/core/udp/peermemoing.py @@ -0,0 +1,286 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.udp.peermemoing Module +""" +from contextlib import contextmanager + +from ... import help + +from ...base import doing +from ...base.tyming import Tymer +from ..udp import Peer +from ..memo import Memoer + +logger = help.ogler.getLogger() + + +class PeerMemoer(Peer, Memoer): + """Class for sending memograms over UXD transport + Mixin base classes Peer and Memoer to attain memogram over uxd transport. + + + Inherited Class Attributes: + (Peer) + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max gram bytes for this transport + + (Memoer) + Version (Versionage): default version consisting of namedtuple of form + (major: int, minor: int) + Codex (GramDex): dataclass ref to gram codex + Codes (dict): maps codex names to codex values + Names (dict): maps codex values to codex names + Sodex (SGDex): dataclass ref to signed gram codex + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes + MaxMemoSize (int): absolute max memo size + MaxGramCount (int): absolute max gram count + + Inherited Attributes: + (Peer) + name (str): unique identifier of peer for managment purposes + ha (tuple): host address of form (host,port) of type (str, int) of this + peer's socket address. + + bc (int | None): count of transport buffers of MaxGramSize + bs (int): buffer size + wl (WireLog): instance ref for debug logging of over the wire tx and rx + + ls (socket.socket): local socket of this Peer + opened (bool): True local socket is created and opened. False otherwise + + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + + (Memoer) + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. + sources (dict): keyed by mid (memoID) that holds the src for the memo. + This enables reattaching src to fused memo in rxms deque tuple. + counts (dict): keyed by mid (memoID) that holds the gram count from + the first gram for the memo. This enables lookup of the gram count when + fusing its grams. + oids (dict[mid: (oid | None)]): keyed by mid that holds the origin ID str for + the memo indexed by its mid (memoID). This enables reattaching + the oid to memo when placing fused memo in rxms deque. + Vid is only present when signed header otherwise oid is None + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is tuple of form: + (memo: str, src: str, oid: str) where: + memo is fused memo, src is source addr, oid is origin ID + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is tuple of form + (memo: str, dst: str, oid: str | None) + memo is memo to be partitioned into gram + dst is dst addr for grams + oid is verification id when gram is to be signed or None otherwise + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when Encodesdatagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) + echos (deque): holding echo receive duples for testing. Each duple of + form: (gram: bytes, dst: str). + + + Inherited Properties: + (Peer) + host (str): element of .ha duple + port (int): element of .ha duple + path (tuple): .ha (host, port) alias to match .uxd + + (Memoer) + code (bytes | None): gram code for gram header when rending for tx + curt (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = over head size + neck size + body size. + other gram size = over head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. + verific (bool): True means any rx grams must be signed. + False otherwise + echoic (bool): True means use .echos in .send and .receive to mock the + transport layer for testing and debugging. + False means do not use .echos + Each entry in .echos is a duple of form: + (gram: bytes, src: str) + Default echo is duple that + indicates nothing to receive of form (b'', None) + When False may be overridden by a method parameter + keep (dict): labels are oids, values are Keyage instances + named tuple of signature key pair: + sigkey = private signing key + verkey = public verifying key + Keyage = namedtuple("Keyage", "sigkey verkey") + oid (str|None): own oid defaults used to lookup keys to sign on tx + + """ + + def __init__(self, *, bc=1024, **kwa): + """Initialization method for instance. + + Inherited Parameters: + bc (int | None): count of transport buffers of MaxGramSize + + See memoing.Memoer for other inherited paramters + See Peer for other inherited paramters + + + Parameters: + + """ + super(PeerMemoer, self).__init__(bc=bc, **kwa) + + + +@contextmanager +def openPM(cls=None, name="test", temp=True, reopen=True, **kwa): + """ + Wrapper to create and open UXD PeerMemoer instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of PeerMemoer peer. + Enables management of transport by name. + Provides unique path part so can have many peers each at + different paths but in same directory. + temp (bool): True means open in temporary directory, clear on close + Otherwise open in persistent directory, do not clear on close + reopen (bool): True (re)open with this init (default) + False not (re)open with this init but later + + See udping.Peer for other keyword parameter passthroughs + + Usage: + with openPM() as peer: + peer.receive() + + with openPM(cls=PeerMemoerSubclass) as peer: + peer.receive() + + """ + peer = None + if cls is None: + cls = PeerMemoer + try: + peer = cls(name=name, temp=temp, reopen=reopen, **kwa) + + yield peer + + finally: + if peer: + peer.close() + + +class PeerMemoerDoer(doing.Doer): + """PeerMemoerDoer Doer for unreliable UDP transport. + Does not require retry tymers. + + See Doer for inherited attributes, properties, and methods. + To test in WingIde must configure Debug I/O to use external console + + Attributes: + .peer (PeerMemoerDoer): underlying transport instance subclass of Memoer + + """ + + def __init__(self, peer, **kwa): + """Initialize instance. + + Parameters: + peer (Peer): is Memoer Subclass instance + """ + super(PeerMemoerDoer, self).__init__(**kwa) + self.peer = peer + + + def enter(self, *, temp=None): + """Do 'enter' context actions. Override in subclass. Not a generator method. + Set up resources. Comparable to context manager enter. + + Parameters: + temp (bool | None): True means use temporary file resources if any + None means ignore parameter value. Use self.temp + + Inject temp or self.temp into file resources here if any + + Doist or DoDoer winds its doers on enter + """ + # inject temp into file resources here if any + self.peer.reopen(temp=temp) + + + def recur(self, tyme): + """""" + self.peer.service() + + + def exit(self): + """""" + self.peer.close() + + +class SafePeerMemoerDoer(doing.Doer): + """PeerMemoerDoer Doer for unreliable UDP transport. + Does not require retry tymers. + + See Doer for inherited attributes, properties, and methods. + To test in WingIde must configure Debug I/O to use external console + + Attributes: + .peer (PeerMemoerDoer): underlying transport instance subclass of Memoer + + """ + + def __init__(self, peer, **kwa): + """Initialize instance. + + Parameters: + peer (Peer): is Memoer Subclass instance + """ + super(SafePeerMemoerDoer, self).__init__(**kwa) + self.peer = peer + + + def wind(self, tymth): + """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + Updates winds .tymer .tymth + """ + super(SafePeerMemoerDoer, self).wind(tymth) + self.peer.wind(tymth) + + + def enter(self, *, temp=None): + """Do 'enter' context actions. Override in subclass. Not a generator method. + Set up resources. Comparable to context manager enter. + + Parameters: + temp (bool | None): True means use temporary file resources if any + None means ignore parameter value. Use self.temp + + Inject temp or self.temp into file resources here if any + + Doist or DoDoer winds its doers on enter + """ + if self.tymth: # Doist or DoDoer already winds its doers on enter + self.peer.wind(self.tymth) + # inject temp into file resources here if any + self.peer.reopen(temp=temp) + + + def recur(self, tyme): + """""" + self.peer.service() + + + def exit(self): + """""" + self.peer.close() diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 8eb6f535..b4f482b2 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -17,42 +17,60 @@ logger = help.ogler.getLogger() + +UDP_IPV4_MAX_SAFE_PAYLOAD = 548 # IPV4 MTU 576 - 28 headers +UDP_IPv6_MAX_SAFE_PAYLOAD = 1240 # IPV6 MTU 1280 - 40 headers UDP_MAX_DATAGRAM_SIZE = (2 ** 16) - 1 # 65535 -UDP_MAX_SAFE_PAYLOAD = 548 # IPV4 MTU 576 - udp headers 28 -# IPV6 MTU is 1280 but headers are bigger UDP_MAX_PACKET_SIZE = min(1024, UDP_MAX_DATAGRAM_SIZE) # assumes IPV6 capable equipment - +# the only way to fragment ipv6 packet is for source to do it. Never done by +# routers en-route. IPV6 has many extension headers that are only used if set +# up at the source. The fragment header is an 8 byte extension header. +# https://www.geeksforgeeks.org/computer-networks/ipv6-fragmentation-header/ class Peer(hioing.Mixin): """Class to manage non blocking I/O on UDP socket. + Class Attributes: + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max bytes in in datagram for this transport + + Attributes: name (str): unique identifier of peer for managment purposes ha (tuple): host address of form (host,port) of type (str, int) of this peer's socket address. + + bc (int | None): count of transport buffers of MaxGramSize bs (int): buffer size wl (WireLog): instance ref for debug logging of over the wire tx and rx - bcast (bool): True enables sending to broadcast addresses from local socket - False otherwise + ls (socket.socket): local socket of this Peer opened (bool): True local socket is created and opened. False otherwise + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + Properties: host (str): element of .ha duple port (int): element of .ha duple + path (tuple): .ha (host, port) alias to match .uxd """ + BufSize = 65535 # 2 ** 16 - 1 default buffersize + MaxGramSize = UDP_IPv6_MAX_SAFE_PAYLOAD # 1240 def __init__(self, *, name='main', ha=None, - host='', + host='127.0.0.1', port=55000, - bufsize=1024, + bc=None, + bs=None, wl=None, bcast=False, + reopen=False, **kwa): """ Initialization method for instance. @@ -61,25 +79,47 @@ def __init__(self, *, ha (tuple): local socket (host, port) address duple of type (str, int) host (str): address where '' means any interface on host port (int): socket port - bs (int): buffer size - wl (WireLog): instance to log over the wire tx and rx + bc (int | None): count of transport buffers of MaxGramSize + bs (int | None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize + wl (WireLog): instance ref for debug logging of over the wire tx and rx bcast (bool): True enables sending to broadcast addresses from local socket False otherwise + reopen (bool): True (re)open with this init + False not (re)open with this init but later (default) """ - super(Peer, self).__init__(**kwa) + self.name = name self.ha = ha or (host, port) # ha = host address duple (host, port) host, port = self.ha host = coring.normalizeHost(host) # ip host address self.ha = (host, port) - self.bs = bufsize + self.bc = int(bc) if bc is not None and bc > 0 else None + if self.bc: + self.bs = self.MaxGramSize * self.bc + else: + self.bs = bs if bs is not None else self.BufSize + self.wl = wl self.bcast = bcast - self.ls = None # local socket for this Peer + self.ls = None # local socket for this Peer needs to be opened/bound self.opened = False + super(Peer, self).__init__(**kwa) + if reopen: + self.reopen() + + @property + def path(self): + """ + Property that returns .ha duple + stub to match uxd interface + """ + return self.ha @property def host(self): @@ -88,7 +128,6 @@ def host(self): """ return self.ha[0] - @host.setter def host(self, value): """ @@ -208,6 +247,7 @@ def receive(self, **kwa): return (data, sa) + def send(self, data, dst, **kwa): """Perform non blocking send on socket. @@ -215,7 +255,7 @@ def send(self, data, dst, **kwa): cnt (int): count of bytes actually sent, may be less than len(data). Parameters: - data (bytes): payload to send + data (bytes | bytearray): payload to send (txbs) dst (str): udp destination addr duple of form (host: str, port: int) """ try: diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index 606d4bd5..426f7dc7 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -5,10 +5,10 @@ from contextlib import contextmanager from ... import help -from ... import hioing + from ...base import doing from ..uxd import Peer -from ..memo import Memoer, GramDex +from ..memo import Memoer logger = help.ogler.getLogger() @@ -19,23 +19,19 @@ class PeerMemoer(Peer, Memoer): Inherited Class Attributes: - MaxGramSize (int): absolute max gram size on tx with overhead - See memoing.Memoer Class See Peer Class - - Inherited Attributes: See memoing.Memoer Class - See Peer Class - - Class Attributes: - Attributes: - + Inherited Attributes: + See Peer Class + See Memoer Class + Inherited Properties: + See Peer Class + See Memoer Class """ - def __init__(self, *, bc=4, **kwa): """Initialization method for instance. @@ -68,8 +64,8 @@ def openPM(cls=None, name="test", temp=True, reopen=True, clear=True, different paths but in same directory. temp (bool): True means open in temporary directory, clear on close Otherwise open in persistent directory, do not clear on close - reopen (bool): True (re)open with this init - False not (re)open with this init but later (default) + reopen (bool): True (re)open with this init (default) + False not (re)open with this init but later clear (bool): True means remove directory upon close when reopening False means do not remove directory upon close when reopening filed (bool): True means .path is file path not directory path diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 6e268e10..72f832fb 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -44,6 +44,12 @@ class Peer(filing.Filer): Mode (str): open mode such as "r+" Fext (str): default file extension such as "text" for "fname.text" + Class Attributes: + Umask (int): octal default umask permissions such as 0o022 + MaxUxdPathSize (int:) max characters in uxd file path + BufSize (int): used to set default buffer size for transport datagram buffers + MaxGramSize (int): max bytes in in datagram for this transport + Inherited Attributes: name (str): unique path component used in directory or file path name base (str): another unique path component inserted before name @@ -59,13 +65,6 @@ class Peer(filing.Filer): opened (bool): True means directory path, uxd file, and socket are created and opened. False otherwise - Class Attributes: - Umask (int): octal default umask permissions such as 0o022 - MaxUxdPathSize (int:) max characters in uxd file path - MaxGramSize (int): max bytes in in datagram for this transport - BufSize (int): used to set buffer size for transport datagram buffers - - Attributes: umask (int): unpermission mask for uxd file, usually octal 0o022 .umask is applied after .perm is set if any @@ -91,13 +90,20 @@ class Peer(filing.Filer): Fext = "uxd" Umask = 0o022 # default MaxUxdPathSize = 108 - MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass BufSize = 65535 # 2 ** 16 - 1 default buffersize + MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass - def __init__(self, *, umask=None, bc=None, bs=None, wl=None, - reopen=False, clear=True, - filed=False, extensioned=True, **kwa): + def __init__(self, *, + umask=None, + bc=None, + bs=None, + wl=None, + reopen=False, + clear=True, + filed=False, + extensioned=True, + **kwa): """Initialization method for instance. Inherited Parameters: @@ -123,13 +129,13 @@ def __init__(self, *, umask=None, bc=None, bs=None, wl=None, wl (WireLog): instance ref for debug logging of over the wire tx and rx """ self.umask = umask # only change umask if umask is not None below - self.bc = bc + + self.bc = int(bc) if bc is not None and bc > 0 else None if self.bc: self.bs = self.MaxGramSize * self.bc else: self.bs = bs if bs is not None else self.BufSize - self.wl = wl self.ls = None # local socket of this Peer, needs to be opened/bound diff --git a/tests/base/test_multidoing.py b/tests/base/test_multidoing.py index 3c617efd..d319d579 100644 --- a/tests/base/test_multidoing.py +++ b/tests/base/test_multidoing.py @@ -350,11 +350,11 @@ def test_boss_crew_basic_multi(): assert not doer.opened doers = [doer] - doist = doing.Doist(tock=0.01, real=True, limit=0.50, doers=doers, temp=True) + doist = doing.Doist(tock=0.01, real=True, limit=1.0, doers=doers, temp=True) assert doist.tyme == 0.0 # on next cycle assert doist.tock == 0.01 assert doist.real == True - assert doist.limit == 0.50 + assert doist.limit == 1.0 assert doist.doers == [doer] doist.do() diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index a74b1d7e..effd817a 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -3,15 +3,98 @@ tests.core.test_memoing module """ +from collections import deque from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 import pytest +from hio.hioing import MemoerError from hio.help import helping from hio.base import doing, tyming from hio.core.memo import memoing -from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer +from hio.core.memo import Versionage, Sizage, GramDex, SGDex, Memoer, Keyage + + +def _setupKeep(salt=None): + """Setup Keep for signed memos + + Parameters: + salt(str|bytes): salt used to generate key pairs and oids for keep + + Returns: + keep (dict): labels are oids, values are keyage instances + + + Ed25519_Seed: str = 'A' # Ed25519 256 bit random seed for private key + """ + try: + import pysodium + import blake3 + except ImportError as ex: + raise MemoerError("Missing cryptographic module support") from ex + + + salt = salt if salt is not None else b"abcdefghijklmnop" + if hasattr(salt, 'encode'): + salt = salt.encode() + + if len(salt) != 16: + raise MemoerError("Invalid provided salt") + + keep = {} + + # non transferable verkey as oid + sigseed = pysodium.crypto_pwhash(outlen=32, + passwd="0", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + + oid = Memoer._encodeOID(raw=verkey, code='B') # make fully qualified + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw + keep[oid] = keyage + + # transferable verkey as oid + sigseed = pysodium.crypto_pwhash(outlen=32, + passwd="1", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + + oid = Memoer._encodeOID(raw=verkey, code='D') # make fully qualified + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw + keep[oid] = keyage + + # digest of verkey as oid + sigseed = pysodium.crypto_pwhash(outlen=32, + passwd="2", + salt=salt, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + # creates signing/verification key pair from seed + verkey, sigkey = pysodium.crypto_sign_seed_keypair(sigseed) # raw + dig = digest = blake3.blake3(verkey).digest() + + oid = Memoer._encodeOID(raw=dig, code='E') + qvk = Memoer._encodeQVK(raw=verkey) # make fully qualified + qss = Memoer._encodeQSS(raw=sigseed) # make fully qualified + keyage = Keyage(qvk=qvk, qss=qss) # raw + keep[oid] = keyage + + return keep + def test_memoer_class(): """Test class attributes of Memoer class""" @@ -20,61 +103,132 @@ def test_memoer_class(): assert Memoer.Codex == memoing.GramDex assert Memoer.Codes == {'Basic': '__', 'Signed': '_-'} - assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} - - # Codes table with sizes of code (hard) and full primitive material - assert Memoer.Sizes == {'__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), - '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160)} - - - # verify all Codes + assert Memoer.Sizes == \ + { + '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), + '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), + } + # verify Sizes and Codes for code, val in Memoer.Sizes.items(): - cs = val.cs - ms = val.ms - vs = val.vs - ss = val.ss - ns = val.ns - hs = val.hs - - assert len(code) == cs == 2 + cz = val.cz + mz = val.mz + oz = val.oz + nz = val.nz + sz = val.sz + hz = val.hz + + assert len(code) == cz == 2 assert code[0] == '_' code[1] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_' - assert hs > 0 - assert hs == cs + ms + vs + ss + ns - assert ms # ms must not be empty - ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - assert ps == (cs % 4) # combined code + mid size must lie on 24 bit boundary - assert not vs % 4 # vid size must be on 24 bit boundary - assert not ss % 4 # sig size must be on 24 bit boundary - assert ns and not ns % 4 # neck (num or cnt) size must be on 24 bit boundary - assert hs and not hs % 4 # head size must be on 24 bit boundary - if vs: - assert ss # ss must not be empty if vs not empty + assert hz > 0 + assert hz == cz + mz + oz + sz + nz + assert mz # ms must not be empty + pz = (3 - ((mz) % 3)) % 3 # net pad size for mid + assert pz == (cz % 4) # combined code + mid size must lie on 24 bit boundary + assert not oz % 4 # oid size must be on 24 bit boundary + assert not sz % 4 # sig size must be on 24 bit boundary + assert nz and not nz % 4 # neck (num or cnt) size must be on 24 bit boundary + assert hz and not hz % 4 # head size must be on 24 bit boundary + if oz: + assert sz # sz must not be empty if oz not empty + assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} assert Memoer.Sodex == SGDex + + # Base2 Binary index representation of Text Base64 Char Codes #assert Memoer.Bodes == {b'\xff\xf0': '__', b'\xff\xe0': '_-'} - assert Memoer.MaxMemoSize == (2**32-1) # absolute max memo payload size + assert Memoer.MaxMemoSize == ((2**30-1)*4+8) # absolute max memo payload size assert Memoer.MaxGramSize == (2**16-1) # absolute max gram size assert Memoer.MaxGramCount == (2**24-1) # absolute max gram count + assert Memoer.BufSize == (2**16-1) # default buffersize + + verkey = (b"o\x91\xf4\xbe$Mu\x0b{}\xd3\xaa'g\xd1\xcf\x96\xfb\x1e\xb1S\x89H\\'ae\x06+\xb2(v") + oid = Memoer._encodeOID(raw=verkey) + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + raw, code = Memoer._decodeOID(oid) + assert raw == verkey + assert code == 'B' + _, _, oz, _, _, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz + assert len(oid) == 44 == oz + + sigseed = (b"\x9bF\n\xf1\xc2L\xeaBC:\xf7\xe9\xd71\xbc\xd2{\x7f\x81\xae5\x9c\xca\xf9\xdb\xac@`'\x0e\xa4\x10") + qss = Memoer._encodeQSS(raw=sigseed) + assert qss == 'AJtGCvHCTOpCQzr36dcxvNJ7f4GuNZzK-dusQGAnDqQQ' + raw, code = Memoer._decodeQSS(qss) + assert raw == sigseed + assert code == 'A' + + signature = (b'\xb0\xc0\xd5\t\xa0\xd3Q0\xfa8\x93B\x0c\x83\xb5.\xfbH\xa5\xde\xbf}6{' + b'\xcf|\xa3\x0el"\x84f\xd3sbHC\xb9\xb9\x85\xb0\xd2v\xed\x07\xcf|c\xa4\xd6\xdcE' + b'\xbe\x8a{w5=\xbf\x84_\x9e\xb3\x04') + sig = Memoer._encodeSig(raw=signature) + assert sig == '0BCwwNUJoNNRMPo4k0IMg7Uu-0il3r99NnvPfKMObCKEZtNzYkhDubmFsNJ27QfPfGOk1txFvop7dzU9v4RfnrME' + raw, code = Memoer._decodeSig(sig) + assert raw == signature + assert code == '0B' + _, _, _, _, sz, _ = Memoer.Sizes[SGDex.Signed] # cz mz oz nz sz hz + assert len(sig) == 88 == sz """Done Test""" +def test_setup_keep(): + """Test _setupkeep utility function + """ + try: + keep = _setupKeep() # default salt + except MemoerError as ex: + return + + assert keep == \ + { + 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2': Keyage(qvk='BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2', + qss='AJtGCvHCTOpCQzr36dcxvNJ7f4GuNZzK-dusQGAnDqQQ'), + 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH': Keyage(qvk='BJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH', + qss='AHyX4f9OBzqNYP5m1UU4wrqtLJaeur7c5klCAWmSjNNX'), + 'EGLDQ97VnSnJS19Dz0j3NcgASGjMgMm4R-DmyrPDRZtN': Keyage(qvk='BAwWHnYdWFkbm_owY7yrfDcOSz3Cio1PuMk9DA3lrcVa', + qss='AEBMs4f_nuBKFMW_OI6DANjED__MJgmefIFZBoLVB_OM') + } + + + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + + assert keep == \ + { + 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp': Keyage(qvk='BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp', + qss='AMP3eeRcXFZHsJvLr4M9zxNURBKF5TDjVJR4ue00WhQC'), + 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni': Keyage(qvk='BGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni', + qss='AOz1Sc-jfEjmNSEWt0-g4-cgshcdAfkvERlGSkueR8ek'), + 'EJ9RvVS6j6-stXpJEyaTi-MOJ1lZUQYkv9-Dc5GGHCVK': Keyage(qvk='BEw-enjGc-o7_Us34IZO-lDbqFG1OCGAg8e1_h1rX6cm', + qss='ANmPt8nNI9YgWAcrKl1_o5lXv1N0Le7qjqRxnduI3-8g') + } + """Done""" + + + + def test_memoer_basic(): """Test Memoer class basic """ peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert not peer.verific + assert not peer.echoic + assert peer.keep == dict() + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -99,6 +253,7 @@ def test_memoer_basic(): assert not peer.txgs assert peer.txbs == (b'', None) + # inject sent gram into .echos so it can recieve from its own as mock transport assert not peer.rxgs assert not peer.counts assert not peer.sources @@ -119,7 +274,9 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms - # send and receive via echo + assert peer.inbox[0] == ('Hello There', 'beta', None) + + # send and receive via .echos to itself as both sender and receiver memo = "See ya later!" dst = "beta" peer.memoit(memo, dst) @@ -130,7 +287,7 @@ def test_memoer_basic(): assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' - peer.serviceTxGrams(echoic=True) + peer.serviceTxGrams(echoic=True) # send to .echos assert not peer.txgs assert peer.txbs == (b'', None) assert peer.echos @@ -139,7 +296,7 @@ def test_memoer_basic(): assert not peer.rxms assert not peer.counts assert not peer.sources - peer.serviceReceives(echoic=True) + peer.serviceReceives(echoic=True) # receive own echo mid = list(peer.rxgs.keys())[0] assert peer.rxgs[mid][0] == b'See ya later!' assert peer.counts[mid] == 1 @@ -154,6 +311,9 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 assert peer.curt @@ -193,6 +353,11 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + assert peer.inbox[2] == ('Hello There', 'beta', None) + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -204,13 +369,16 @@ def test_memoer_small_gram_size(): peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == 33 # can't be smaller than head + neck + 1 assert not peer.verific + assert not peer.echoic + assert peer.keep == dict() + assert peer.oid is None peer = memoing.Memoer(size=38) assert peer.size == 38 @@ -265,6 +433,8 @@ def test_memoer_small_gram_size(): assert not peer.rxms assert not peer.echos + assert peer.inbox[0] == ('Hello There', 'beta', None) + # send and receive via echo memo = "See ya later!" dst = "beta" @@ -303,6 +473,9 @@ def test_memoer_small_gram_size(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 assert peer.curt @@ -356,6 +529,11 @@ def test_memoer_small_gram_size(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', None) + assert peer.inbox[1] == ('See ya later!', 'beta', None) + assert peer.inbox[2] == ('See ya later alligator!', 'beta', None) + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -368,10 +546,14 @@ def test_memoer_multiple(): assert peer.size == 38 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt assert not peer.verific + assert not peer.echoic + assert peer.keep == dict() + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -425,24 +607,144 @@ def test_memoer_multiple(): peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_memoer_multiple_echoic_service_tx_rx(): + """Test Memoer class with small gram size and multiple queued memos + Use .echoic property true so can service all + """ + peer = memoing.Memoer(size=38, echoic=True) + assert peer.size == 38 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.curt + assert not peer.verific + assert peer.echoic + assert peer.keep == dict() + assert peer.oid is None + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + + peer.serviceAllTx() + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAllRx() + + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it peer.close() assert peer.opened == False """ End Test """ + +def test_memoer_multiple_echoic_service_all(): + """Test Memoer class with small gram size and multiple queued memos + Use .echoic property true so can service all + """ + peer = memoing.Memoer(size=38, echoic=True) + assert peer.size == 38 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.curt + assert not peer.verific + assert peer.echoic + assert peer.keep == dict() + assert peer.oid is None + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + + peer.serviceAll() # services Rx first then Tx so have to serviceAll twice + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # Rx not serviced yet after Tx serviced + + peer.serviceAll() # services Rx first then Tx so have to serviceAll twice + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox[0] == ('Hello there.', 'alpha', None) + assert peer.inbox[1] == ('How ya doing?', 'beta', None) + + peer.inbox = deque() # clear it + + peer.close() + assert peer.opened == False + """ End Test """ + + def test_memoer_basic_signed(): """Test Memoer class basic signed code """ - peer = memoing.Memoer(code=GramDex.Signed) + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + peer = memoing.Memoer(code=GramDex.Signed, keep=keep, oid=oid) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 44, 88, 4, 160) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert not peer.verific + assert not peer.echoic + assert peer.keep == keep + assert peer.oid == oid peer.reopen() assert peer.opened == True @@ -455,9 +757,10 @@ def test_memoer_basic_signed(): memo = "Hello There" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('Hello There', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -477,7 +780,7 @@ def test_memoer_basic_signed(): mid = '_-ALBI68S1ZIxqwFOSWFF1L2' sig = ('A' * 88) - gram = (mid + vid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() + gram = (mid + oid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -488,16 +791,16 @@ def test_memoer_basic_signed(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms # send and receive via echo memo = "See ya later!" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('See ya later!', 'beta', vid) + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('See ya later!', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -526,7 +829,7 @@ def test_memoer_basic_signed(): assert not peer.counts assert not peer.sources peer.rxms[0] - assert peer.rxms[0] == ('See ya later!', 'beta', vid) + assert peer.rxms[0] == ('See ya later!', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms @@ -534,9 +837,9 @@ def test_memoer_basic_signed(): peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit(memo, dst, vid) - assert peer.txms[0] == ('Hello There', 'beta', vid) + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid + peer.memoit(memo, dst, oid) + assert peer.txms[0] == ('Hello There', 'beta', oid) peer.serviceTxMemos() assert not peer.txms g, d = peer.txgs[0] @@ -554,9 +857,9 @@ def test_memoer_basic_signed(): assert not peer.sources assert not peer.rxms mid = '_-ALBI68S1ZIxqwFOSWFF1L2' - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + oid = 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' # not default .oid sig = ('A' * 88) - head = decodeB64((mid + vid + 'AAAA' + 'AAAB').encode()) + head = decodeB64((mid + oid + 'AAAA' + 'AAAB').encode()) tail = decodeB64(sig.encode()) gram = head + memo.encode() + tail assert peer.wiff(gram) # base2 @@ -571,10 +874,16 @@ def test_memoer_basic_signed(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox[0] == ('Hello There', 'beta', oid) + assert peer.inbox[1] == ('See ya later!', 'beta', oid) + assert peer.inbox[2] == ('Hello There', 'beta', oid) + + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -582,22 +891,37 @@ def test_memoer_basic_signed(): def test_memoer_multiple_signed(): """Test Memoer class with small gram size and multiple queued memos signed """ - peer = memoing.Memoer(code=GramDex.Signed, size=170) + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' + + peer = memoing.Memoer(code=GramDex.Signed, size=170, keep=keep, oid=oid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt assert not peer.verific + assert not peer.echoic + assert peer.keep == keep + assert peer.oid == oid peer.reopen() assert peer.opened == True # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + peer.memoit("Hello there.", "alpha") # use default for vidAlpha + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -639,8 +963,8 @@ def test_memoer_multiple_signed(): assert not peer.counts assert not peer.sources assert len(peer.rxms) == 2 - assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vid) + assert peer.rxms[0] == ('Hello there.', 'alpha', oid) + assert peer.rxms[1] == ('How ya doing?', 'beta', oidBeta) peer.serviceRxMemos() assert not peer.rxms @@ -650,10 +974,9 @@ def test_memoer_multiple_signed(): peer.size = 129 assert peer.size == 129 - # send and receive multiple via echo - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - peer.memoit("Hello there.", "alpha", vid) - peer.memoit("How ya doing?", "beta", vid) + # send and receive multiple via echo in base2 .curt = True mode + peer.memoit("Hello there.", "alpha") # use default for vidAlpha + peer.memoit("How ya doing?", "beta", oidBeta) assert len(peer.txms) == 2 peer.serviceTxMemos() assert not peer.txms @@ -695,11 +1018,19 @@ def test_memoer_multiple_signed(): assert not peer.counts assert not peer.sources assert len(peer.rxms) == 2 - assert peer.rxms[0] == ('Hello there.', 'alpha', vid) - assert peer.rxms[1] == ('How ya doing?', 'beta', vid) + assert peer.rxms[0] == ('Hello there.', 'alpha', oid) + assert peer.rxms[1] == ('How ya doing?', 'beta', oidBeta) peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta) + ]) + peer.close() assert peer.opened == False @@ -712,13 +1043,16 @@ def test_memoer_verific(): peer = memoing.Memoer(verific=True) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 4 + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt - # (code, mid, vid, sig, neck, head) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize assert peer.verific + assert not peer.echoic + assert peer.keep == dict() + assert peer.oid is None peer.reopen() assert peer.opened == True @@ -741,9 +1075,9 @@ def test_memoer_verific(): assert not peer.sources assert not peer.rxms mid = '_-ALBI68S1ZIxqwFOSWFF1L2' - vid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' + oid = 'BKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sig = ('A' * 88) - gram = (mid + vid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() + gram = (mid + oid + 'AAAA' + 'AAAB' + "Hello There" + sig).encode() echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -755,10 +1089,123 @@ def test_memoer_verific(): assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', vid) + assert peer.rxms[0] == ('Hello There', 'beta', oid) peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', oid) + ]) + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_memoer_multiple_signed_verific_echoic_service_all(): + """Test Memoer class with small gram size and multiple queued memos signed + using echos for transport + """ + salt = b"ABCDEFGHIJKLMNOP" + try: + keep = _setupKeep(salt=salt) + except MemoerError as ex: + return + + oid = list(keep.keys())[0] + assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' + + vidBeta = list(keep.keys())[1] + assert vidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' + + # verific forces rx memos to be signed or dropped + # to force signed tx then use Signed code + peer = memoing.Memoer(code=GramDex.Signed, size=170, verific=True, + echoic=True, keep=keep, oid=oid) + assert peer.size == 170 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.curt + assert peer.verific + assert peer.echoic + assert peer.keep == keep + assert peer.oid == oid + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', vidBeta) + ]) + + # test in base2 mode + peer.curt = True + assert peer.curt + peer.size = 129 + assert peer.size == 129 + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", vidBeta) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', vidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', vidBeta), + ]) + + peer.inbox = deque() # clear it + peer.close() assert peer.opened == False """ End Test """ @@ -805,19 +1252,40 @@ def test_memoer_doer(): """End Test """ -def test_tymee_memoer_basic(): - """Test TymeeMemoer class basic + + +def test_sure_memoer_basic(): + """Test SureMemoer class basic """ - peer = memoing.TymeeMemoer() - assert peer.tymeout == 0.0 + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + + oid = list(keep.keys())[0] + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' + + peer = memoing.SureMemoer(echoic=True, keep=keep, oid=oid) + assert peer.size == 65535 assert peer.name == "main" assert peer.opened == False - assert peer.code == memoing.GramDex.Basic == '__' + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt - # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert peer.verific + assert peer.echoic + assert peer.keep == keep + assert peer.oid == oid + + assert peer.Sizes[peer.code] == Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160) + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize - assert not peer.verific + assert peer.tymeout == 0.0 + assert peer.tymers == {} peer.reopen() assert peer.opened == True @@ -825,116 +1293,104 @@ def test_tymee_memoer_basic(): assert not peer.txms assert not peer.txgs assert peer.txbs == (b'', None) - peer.service() + peer.service() # alias for .serviceAll assert peer.txbs == (b'', None) memo = "Hello There" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('Hello There', 'beta', None) - peer.serviceTxMemos() + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('Hello There', 'beta', oidBeta) + + peer.service() # services Rx then Tx so rx of tx not serviced until 2nd pass + assert not peer.txms - m, d = peer.txgs[0] - assert not peer.wiff(m) # base64 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.counts assert not peer.sources assert not peer.rxms - mid = '__ALBI68S1ZIxqwFOSWFF1L2' - gram = (mid + 'AAAA' + 'AAAB' + "Hello There").encode() - echo = (gram, "beta") - peer.echos.append(echo) - peer.serviceReceives(echoic=True) - assert peer.rxgs[mid][0] == bytearray(b'Hello There') - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' - peer.serviceRxGrams() + + peer.service() + + assert not peer.echos assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms - # send and receive via echo + assert peer.inbox == deque( + [ + ('Hello There', 'beta', oidBeta), + ]) + + # send and receive some more memo = "See ya later!" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('See ya later!', 'beta', None) - peer.serviceTxMemos() + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('See ya later!', 'beta', oidBeta) + + peer.service() + assert not peer.txms - m, d = peer.txgs[0] - assert not peer.wiff(m) # base64 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams(echoic=True) assert not peer.txgs assert peer.txbs == (b'', None) - assert peer.echos + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.rxms assert not peer.counts assert not peer.sources - peer.serviceReceives(echoic=True) - mid = list(peer.rxgs.keys())[0] - assert peer.rxgs[mid][0] == b'See ya later!' - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' + + peer.service() + assert not peer.echos - peer.serviceRxGrams() assert not peer.rxgs assert not peer.counts assert not peer.sources - peer.rxms[0] - assert peer.rxms[0] == ('See ya later!', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', oidBeta), + ('See ya later!', 'beta', oidBeta), + ]) + # test binary q2 encoding of transmission gram header peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" - peer.memoit(memo, dst) - assert peer.txms[0] == ('Hello There', 'beta', None) - peer.serviceTxMemos() + peer.memoit(memo, dst, oidBeta) + assert peer.txms[0] == ('Hello There', 'beta', oidBeta) + + peer.service() + assert not peer.txms - m, d = peer.txgs[0] - assert peer.wiff(m) # base2 - assert m.endswith(memo.encode()) - assert d == dst == 'beta' - peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) + assert len(peer.echos) == 1 assert not peer.rxgs assert not peer.counts assert not peer.sources assert not peer.rxms - mid = '__ALBI68S1ZIxqwFOSWFF1L2' - headneck = decodeB64((mid + 'AAAA' + 'AAAB').encode()) - gram = headneck + b"Hello There" - assert peer.wiff(gram) # base2 - echo = (gram, "beta") - peer.echos.append(echo) - peer.serviceReceives(echoic=True) - assert peer.rxgs[mid][0] == bytearray(b'Hello There') - assert peer.counts[mid] == 1 - assert peer.sources[mid] == 'beta' - peer.serviceRxGrams() + + peer.service() + assert not peer.echos assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta', None) - peer.serviceRxMemos() assert not peer.rxms + assert peer.inbox == deque( + [ + ('Hello There', 'beta', oidBeta), + ('See ya later!', 'beta', oidBeta), + ('Hello There', 'beta', oidBeta), + ]) + # Test wind tymist = tyming.Tymist(tock=1.0) peer.wind(tymth=tymist.tymen()) @@ -946,15 +1402,20 @@ def test_tymee_memoer_basic(): assert peer.opened == False """ End Test """ - -def test_open_tm(): +def test_open_sm(): """Test contextmanager decorator openTM for openTymeeMemoer """ - with (memoing.openTM(name='zeta') as zeta): + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + + with (memoing.openSM(name='zeta') as zeta): assert zeta.opened assert zeta.name == 'zeta' assert zeta.tymeout == 0.0 + assert zeta.tymers == {} assert not zeta.opened @@ -962,9 +1423,119 @@ def test_open_tm(): """ End Test """ -def test_tymee_memoer_doer(): - """Test TymeeMemoerDoer class +def test_sure_memoer_multiple_echoic_service_all(): + """Test SureMemoer class with small gram size and multiple queued memos signed + using echos for transport """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + + oid = list(keep.keys())[0] + assert oid == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' + + oidBeta = list(keep.keys())[1] + assert oidBeta == 'DJb1Z0pHx36MCOuIHWR4yPxfIiBxVzg6UCamv8fAN8gH' + + # verific forces rx memos to be signed or dropped + # to force signed tx then use Signed code + + with memoing.openSM(size=170, echoic=True, keep=keep, oid=oid) as peer: + assert peer.size == 170 + assert peer.name == "test" + assert peer.opened == True + assert peer.bc is None + assert peer.bs == memoing.Memoer.BufSize == 65535 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.curt + assert peer.verific + assert peer.echoic + assert peer.keep == keep + assert peer.oid is oid + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", oidBeta) + assert len(peer.txms) == 2 + + peer.service() # services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta) + ]) + + + # test in base2 mode + peer.curt = True + assert peer.curt + peer.size = 129 + assert peer.size == 129 + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta", oidBeta) + assert len(peer.txms) == 2 + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 # rx not serviced yet + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceAll() # servicAll services Rx first then Tx so have to repeat + assert not peer.echos + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + assert peer.inbox == deque( + [ + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), + ('Hello there.', 'alpha', oid), + ('How ya doing?', 'beta', oidBeta), + ]) + + peer.inbox = deque() # clear it + + assert peer.opened == False + """ End Test """ + + +def test_sure_memoer_doer(): + """Test SureMemoerDoer class + """ + try: + keep = _setupKeep() # uses default salt + except MemoerError as ex: + return + tock = 0.03125 ticks = 4 limit = ticks * tock @@ -975,9 +1546,9 @@ def test_tymee_memoer_doer(): assert doist.limit == limit == 0.125 assert doist.doers == [] - peer = memoing.TymeeMemoer() + peer = memoing.SureMemoer() - tmgdoer = memoing.TymeeMemoerDoer(peer=peer) + tmgdoer = memoing.SureMemoerDoer(peer=peer) assert tmgdoer.peer == peer assert not tmgdoer.peer.opened assert tmgdoer.tock == 0.0 # ASAP @@ -989,11 +1560,11 @@ def test_tymee_memoer_doer(): tymist = tyming.Tymist(tock=1.0) tmgdoer.wind(tymth=tymist.tymen()) - assert tmgdoer.tyme == tymist.tyme == 0.0 - assert peer.tyme == tymist.tyme == 0.0 + assert tmgdoer.tymth == tmgdoer.peer.tymth + assert tmgdoer.tyme == peer.tyme == tymist.tyme == 0.0 + tymist.tick() - assert tmgdoer.tyme == tymist.tyme == 1.0 - assert peer.tyme == tymist.tyme == 1.0 + assert tmgdoer.tyme == peer.tyme == tymist.tyme == 1.0 """End Test """ @@ -1001,15 +1572,20 @@ def test_tymee_memoer_doer(): if __name__ == "__main__": test_memoer_class() + test_setup_keep() test_memoer_basic() test_memoer_small_gram_size() test_memoer_multiple() + test_memoer_multiple_echoic_service_tx_rx() + test_memoer_multiple_echoic_service_all() test_memoer_basic_signed() test_memoer_multiple_signed() test_memoer_verific() + test_memoer_multiple_signed_verific_echoic_service_all() test_open_memoer() test_memoer_doer() - test_tymee_memoer_basic() - test_open_tm() - test_tymee_memoer_doer() + test_sure_memoer_basic() + test_open_sm() + test_sure_memoer_multiple_echoic_service_all() + test_sure_memoer_doer() diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py new file mode 100644 index 00000000..2744786a --- /dev/null +++ b/tests/core/udp/test_peer_memoing.py @@ -0,0 +1,321 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_peer_memoer module + +""" +import os +import platform +import time + +import pytest + +from hio.help import helping +from hio.base import doing, tyming +from hio.core.memo import GramDex +from hio.core.udp import udping, peermemoing + + +def test_memoer_peer_basic(): + """Test MemoerPeer class""" + + alphaPort = 6103 + betaPort = 6104 + + alpha = peermemoing.PeerMemoer(name="alpha", temp=True) + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.size == 1240 # default MaxGramSize for udp + assert alpha.bc == 1024 + assert not alpha.opened + assert alpha.ha == ('127.0.0.1', 55000) + assert alpha.path == alpha.ha + + + size = 38 # force gram size to be smaller than default so forces segmentation + alpha = peermemoing.PeerMemoer(name="alpha", temp=True, size=size, port=alphaPort) + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.size == size + assert alpha.bc == 1024 + assert not alpha.opened + assert alpha.reopen() + assert alpha.opened + assert alpha.ha == ('127.0.0.1', alphaPort) + assert alpha.path == alpha.ha + #assert alpha.actualBufSizes() == (1269760, 1269760) == (alpha.bc * alpha.MaxGramSize, alpha.bc * alpha.MaxGramSize) + + beta = peermemoing.PeerMemoer(name="beta", temp=True, size=size, port=betaPort) + assert beta.reopen() + assert beta.opened + assert beta.ha == ('127.0.0.1', betaPort) + assert beta.path == beta.ha + #assert beta.actualBufSizes() == (1269760, 1269760) == (beta.bc * beta.MaxGramSize, beta.bc * beta.MaxGramSize) + + # alpha sends + alpha.memoit("Hello there.", beta.ha) + alpha.memoit("How ya doing?", beta.path) + assert len(alpha.txms) == 2 + alpha.serviceTxMemos() + assert not alpha.txms + assert len(alpha.txgs) == 4 + for m, d in alpha.txgs: + assert not alpha.wiff(m) # base64 + assert d == beta.path + alpha.serviceTxGrams() + assert not alpha.txgs + assert alpha.txbs == (b'', None) + assert not alpha.rxgs + assert not alpha.rxms + assert not alpha.counts + assert not alpha.sources + + # beta receives + beta.serviceReceives() + time.sleep(0.05) + assert not beta.echos + assert len(beta.rxgs) == 2 + assert len(beta.counts) == 2 + assert len(beta.sources) == 2 + + mid = list(beta.rxgs.keys())[0] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'Hello ') + assert beta.rxgs[mid][1] == bytearray(b'there.') + + mid = list(beta.rxgs.keys())[1] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'How ya') + assert beta.rxgs[mid][1] == bytearray(b' doing?') + + beta.serviceRxGrams() + assert not beta.rxgs + assert not beta.counts + assert not beta.sources + assert len(beta.rxms) == 2 + assert beta.rxms[0] == ('Hello there.', alpha.path, None) + assert beta.rxms[1] == ('How ya doing?', alpha.path, None) + beta.serviceRxMemos() + assert not beta.rxms + + # beta sends + beta.memoit("Well is not this a fine day?", alpha.path) + beta.memoit("Do you want to do lunch?", alpha.path) + assert len(beta.txms) == 2 + beta.serviceTxMemos() + assert not beta.txms + beta.serviceTxGrams() + assert not beta.txgs + assert beta.txbs == (b'', None) + assert not beta.rxgs + assert not beta.rxms + assert not beta.counts + assert not beta.sources + + # alpha receives + alpha.serviceReceives() + time.sleep(0.05) + assert not alpha.echos + assert len(alpha.rxgs) == 2 + assert len(alpha.counts) == 2 + assert len(alpha.sources) == 2 + + mid = list(alpha.rxgs.keys())[0] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 4 + assert len(alpha.rxgs[mid]) == 4 + + mid = list(alpha.rxgs.keys())[1] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 3 + assert len(alpha.rxgs[mid]) == 3 + + alpha.serviceRxGrams() + assert not alpha.rxgs + assert not alpha.counts + assert not alpha.sources + assert len(alpha.rxms) == 2 + assert alpha.rxms[0] == ('Well is not this a fine day?', beta.path, None) + assert alpha.rxms[1] == ('Do you want to do lunch?', beta.path, None) + alpha.serviceRxMemos() + assert not alpha.rxms + + assert beta.close() + assert not beta.opened + + assert alpha.close() + assert not alpha.opened + + """Done Test""" + + +def test_memoer_peer_open(): + """Test MemoerPeer class with context manager openPM""" + + host = '127.0.0.1' # default + alphaPort = 6103 + betaPort = 6104 + size = 38 + + with (peermemoing.openPM(name='alpha', size=size, port=alphaPort) as alpha, + peermemoing.openPM(name='beta', size=size, port=betaPort) as beta): + + + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.size == size + assert alpha.bc == 1024 + + assert alpha.opened + assert alpha.ha == alpha.path == (host, alphaPort) + + assert beta.bc == 1024 + assert beta.opened + assert beta.ha == beta.path == (host, betaPort) + + # alpha sends + alpha.memoit("Hello there.", beta.path) + alpha.memoit("How ya doing?", beta.path) + assert len(alpha.txms) == 2 + alpha.serviceTxMemos() + assert not alpha.txms + assert len(alpha.txgs) == 4 + for m, d in alpha.txgs: + assert not alpha.wiff(m) # base64 + assert d == beta.path + alpha.serviceTxGrams() + assert not alpha.txgs + assert alpha.txbs == (b'', None) + assert not alpha.rxgs + assert not alpha.rxms + assert not alpha.counts + assert not alpha.sources + + # beta receives + while not beta.rxgs: + beta.serviceReceives() + time.sleep(0.05) + assert not beta.echos + assert len(beta.rxgs) == 2 + assert len(beta.counts) == 2 + assert len(beta.sources) == 2 + + mid = list(beta.rxgs.keys())[0] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'Hello ') + assert beta.rxgs[mid][1] == bytearray(b'there.') + + mid = list(beta.rxgs.keys())[1] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'How ya') + assert beta.rxgs[mid][1] == bytearray(b' doing?') + + beta.serviceRxGrams() + assert not beta.rxgs + assert not beta.counts + assert not beta.sources + assert len(beta.rxms) == 2 + assert beta.rxms[0] == ('Hello there.', alpha.path, None) + assert beta.rxms[1] == ('How ya doing?', alpha.path, None) + beta.serviceRxMemos() + assert not beta.rxms + + # beta sends + beta.memoit("Well is not this a fine day?", alpha.path) + beta.memoit("Do you want to do lunch?", alpha.path) + assert len(beta.txms) == 2 + beta.serviceTxMemos() + assert not beta.txms + beta.serviceTxGrams() + assert not beta.txgs + assert beta.txbs == (b'', None) + assert not beta.rxgs + assert not beta.rxms + assert not beta.counts + assert not beta.sources + + # alpha receives + while not alpha.rxgs: + alpha.serviceReceives() + time.sleep(0.05) + assert not alpha.echos + assert len(alpha.rxgs) == 2 + assert len(alpha.counts) == 2 + assert len(alpha.sources) == 2 + + mid = list(alpha.rxgs.keys())[0] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 4 + assert len(alpha.rxgs[mid]) == 4 + + mid = list(alpha.rxgs.keys())[1] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 3 + assert len(alpha.rxgs[mid]) == 3 + + alpha.serviceRxGrams() + assert not alpha.rxgs + assert not alpha.counts + assert not alpha.sources + assert len(alpha.rxms) == 2 + assert alpha.rxms[0] == ('Well is not this a fine day?', beta.path, None) + assert alpha.rxms[1] == ('Do you want to do lunch?', beta.path, None) + alpha.serviceRxMemos() + assert not alpha.rxms + + assert not beta.opened + assert not alpha.opened + + """Done Test""" + + + +def test_peermemoer_doer(): + """Test PeerMemoerDoer class + """ + + tock = 0.03125 + ticks = 4 + limit = ticks * tock + doist = doing.Doist(tock=tock, real=True, limit=limit) + assert doist.tyme == 0.0 # on next cycle + assert doist.tock == tock == 0.03125 + assert doist.real == True + assert doist.limit == limit == 0.125 + assert doist.doers == [] + + peer = peermemoing.PeerMemoer(name="test", temp=True, reopen=False) + assert peer.opened == False + assert peer.ha == peer.path == ('127.0.0.1', 55000) + + doer = peermemoing.PeerMemoerDoer(peer=peer) + assert doer.peer == peer + assert not doer.peer.opened + assert doer.tock == 0.0 # ASAP + + doers = [doer] + doist.do(doers=doers) + assert doist.tyme == limit + assert peer.opened == False + """Done Test""" + + +if __name__ == "__main__": + test_memoer_peer_basic() + test_memoer_peer_open() + test_peermemoer_doer() + + diff --git a/tests/core/udp/test_udp.py b/tests/core/udp/test_udping.py similarity index 74% rename from tests/core/udp/test_udp.py rename to tests/core/udp/test_udping.py index cf246626..dc365a49 100644 --- a/tests/core/udp/test_udp.py +++ b/tests/core/udp/test_udping.py @@ -38,6 +38,7 @@ def test_udp_basic(): assert alpha.reopen() assert alpha.opened assert alpha.ha == alphaHa + assert alpha.path == alpha.ha beta = udping.Peer(name='beta',host=host, port = betaPort, wl=wl) # host on port 6102 assert not beta.opened @@ -45,6 +46,7 @@ def test_udp_basic(): assert beta.reopen() assert beta.opened assert beta.ha == betaHa + assert beta.path == beta.ha msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) @@ -105,6 +107,121 @@ def addrBytes(ha): assert not wl.opened """Done Test""" +def test_udp(): + """ Test the udp connection between two peers with bc and bs + + """ + + tymist = tyming.Tymist() + with (wiring.openWL(samed=True, filed=True) as wl): + host = '127.0.0.1' + alphaPort = 6101 + betaPort = 6102 + alphaHa = (host, alphaPort) + betaHa = (host, betaPort) + bc = 1024 + + alpha = udping.Peer(host=host, port = alphaPort, wl=wl, bc=bc) + assert not alpha.opened + assert alpha.name == 'main' # default + assert alpha.bc == bc + assert alpha.MaxGramSize == 1240 # max safe payload + assert alpha.bs == bc * alpha.MaxGramSize == 1269760 + assert alpha.actualBufSizes() == (0, 0) # not opened yet + assert alpha.reopen() + assert alpha.opened + assert alpha.ha == alphaHa + #assert alpha.actualBufSizes() == (alpha.bs, alpha.bs) == (1269760, 1269760) + sizes = alpha.actualBufSizes() + assert sizes[0] > 16383 + assert sizes[1] > 16383 + + bs = 2 ** 15 - 1 + assert bs == 32767 + beta = udping.Peer(name='beta',host=host, port = betaPort, wl=wl, bs=bs) # host on port 6102 + assert not beta.opened + assert beta.name == 'beta' + assert beta.bc is None + assert beta.bs == bs + assert beta.actualBufSizes() == (0, 0) # not opened yet + assert beta.reopen() + assert beta.opened + assert beta.ha == betaHa + bses = beta.actualBufSizes() + assert bses[0] >= beta.bs + assert bses[1] >= beta.bs + + msgOut = b"alpha sends to beta" + alpha.send(msgOut, beta.ha) + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) + assert msgOut == msgIn + assert src[1] == alpha.port # ports equal + + msgOut = b"alpha sends to alpha" + alpha.send(msgOut, alpha.ha) + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) + assert msgOut == msgIn + assert src[1] == alpha.port # ports equal + + msgOut = b"beta sends to alpha" + beta.send(msgOut, alpha.ha) + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) + assert msgOut == msgIn + assert src[1] == beta.port # ports equal + + msgOut = b"beta sends to beta" + beta.send(msgOut, beta.ha) + time.sleep(0.05) + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) + assert msgOut == msgIn + assert src[1] == beta.port # ports equal + + alpha.close() + beta.close() + + assert not alpha.opened + assert not beta.opened + + wl.flush() # just to test + assert wl.samed # rx and tx same buffer + + def addrBytes(ha): + return f"('{ha[0]}', {ha[1]})".encode("ascii") + + assert wl.readRx() == (b"\nTx ('127.0.0.1', 6102):\nalpha sends to beta\n\nRx ('127.0.0.1', 6101)" + b":\nalpha sends to beta\n\nTx ('127.0.0.1', 6101):\nalpha sends to alpha\n" + b"\nRx ('127.0.0.1', 6101):\nalpha sends to alpha\n\nTx ('127.0.0.1', 6101" + b"):\nbeta sends to alpha\n\nRx ('127.0.0.1', 6102):\nbeta sends to alpha\n" + b"\nTx ('127.0.0.1', 6102):\nbeta sends to beta\n\nRx ('127.0.0.1', 6102):" + b'\nbeta sends to beta\n') + assert wl.readTx() == (b"\nTx ('127.0.0.1', 6102):\nalpha sends to beta\n\nRx ('127.0.0.1', 6101)" + b":\nalpha sends to beta\n\nTx ('127.0.0.1', 6101):\nalpha sends to alpha\n" + b"\nRx ('127.0.0.1', 6101):\nalpha sends to alpha\n\nTx ('127.0.0.1', 6101" + b"):\nbeta sends to alpha\n\nRx ('127.0.0.1', 6102):\nbeta sends to alpha\n" + b"\nTx ('127.0.0.1', 6102):\nbeta sends to beta\n\nRx ('127.0.0.1', 6102):" + b'\nbeta sends to beta\n') + + + assert wl.readTx() == wl.readRx() + + assert not wl.opened + """Done Test""" + def test_open_peer(): """Test the udp openPeer context manager connection between two peers @@ -135,28 +252,40 @@ def test_open_peer(): msgOut = b"alpha sends to beta" alpha.send(msgOut, beta.ha) time.sleep(0.05) - msgIn, src = beta.receive() + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"alpha sends to alpha" alpha.send(msgOut, alpha.ha) time.sleep(0.05) - msgIn, src = alpha.receive() + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == alpha.port # ports equal msgOut = b"beta sends to alpha" beta.send(msgOut, alpha.ha) time.sleep(0.05) - msgIn, src = alpha.receive() + msgIn = b"" + while not msgIn: + msgIn, src = alpha.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal msgOut = b"beta sends to beta" beta.send(msgOut, beta.ha) time.sleep(0.05) - msgIn, src = beta.receive() + msgIn = b"" + while not msgIn: + msgIn, src = beta.receive() + time.sleep(0.05) assert msgOut == msgIn assert src[1] == beta.port # ports equal @@ -402,6 +531,7 @@ def test_peer_doer(): if __name__ == "__main__": test_udp_basic() + test_udp() test_open_peer() test_peer_doer() #test_udp_broadcast() diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoing.py similarity index 96% rename from tests/core/uxd/test_peer_memoer.py rename to tests/core/uxd/test_peer_memoing.py index 3511ed83..1cf4dfd8 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoing.py @@ -14,8 +14,6 @@ from hio.core.uxd import uxding, peermemoing - - def test_memoer_peer_basic(): """Test MemoerPeer class""" if platform.system() == "Windows": @@ -24,8 +22,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 assert alpha.bc == 4 @@ -147,15 +144,14 @@ def test_memoer_peer_open(): """Test MemoerPeer class with context manager openPM""" if platform.system() == "Windows": return - with (peermemoing.openPM(name='alpha', size=38) as alpha, - peermemoing.openPM(name='beta', size=38) as beta): + with (peermemoing.openPM(name='alpha', size=38) as alpha, + peermemoing.openPM(name='beta', size=38) as beta): assert alpha.name == "alpha" assert alpha.code == GramDex.Basic assert not alpha.curt - # (code, mid, vid, sig, neck, head) part sizes - assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 assert alpha.bc == 4 @@ -257,11 +253,9 @@ def test_memoer_peer_open(): assert not alpha.rxms - assert beta.close() assert not beta.opened assert not os.path.exists(beta.path) - assert alpha.close() assert not alpha.opened assert not os.path.exists(alpha.path) diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxding.py similarity index 100% rename from tests/core/uxd/test_uxd.py rename to tests/core/uxd/test_uxding.py