From 52b3c4d4529035f636a7cc2809d0b4023ca81b6d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 27 Aug 2025 14:28:14 -0600 Subject: [PATCH 01/36] started work on Hog metrical logger for Hold --- ref/hier.md | 3 +- src/hio/base/filing.py | 35 +++++++-- src/hio/base/hier/__init__.py | 1 + src/hio/base/hier/acting.py | 19 +++-- src/hio/base/hier/hogging.py | 123 ++++++++++++++++++++++++++++++++ tests/base/hier/test_hogging.py | 18 +++++ 6 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 src/hio/base/hier/hogging.py create mode 100644 tests/base/hier/test_hogging.py diff --git a/ref/hier.md b/ref/hier.md index 52328e2..0cb38c4 100644 --- a/ref/hier.md +++ b/ref/hier.md @@ -55,7 +55,7 @@ These properties are mutually supporting. Together they provide a powerful found for solving the hard problem of automation systems, that is, the coordination problem. The coordination problem poses the following question; how best to coordinate the various components of an automation system especially -a coordiantion system that is dynamically adaptive (in-stride) to both a +a coordination system that is dynamically adaptive (in-stride) to both a changing environment and changing goals and objectives for mission success? @@ -107,6 +107,7 @@ A boxwork consists of a graph of connected boxes. Each box belongs to a pile of stacked boxes. A given pile constitues a hierarchical state. State changes by transitioning from one pile to a different pile as determined by a transition from one box to another box in the boxwork. + ### Box Piles Each box may have an over box and zero or more under boxes. A box at the top diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index 0cbcd8c..9ef5384 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -40,9 +40,7 @@ class Filer(hioing.Mixin): Mode (str): open mode such as "r+" Fext (str): default file extension such as "text" for "fname.text" - Attributes: - name (str): unique path component used in directory or file path name base (str): another unique path component inserted before name temp (bool): True means use TempHeadDir in /tmp directory headDirPath (str): head directory path @@ -59,6 +57,12 @@ class Filer(hioing.Mixin): opened (bool): True means directory created and if filed then file is opened. False otherwise + Properties: + name (str): unique path component used in directory or file path name + + Hidden: + _name (str): unique name for .name property + File/Directory Creation Mode Notes: .Perm provides default restricted access permissions to directory and/or files @@ -126,7 +130,8 @@ def __init__(self, *, name='main', base="", temp=False, headDirPath=None, fext (str): File extension when filed or extensioned """ - super(Filer, self).__init__(**kwa) # Mixin for Mult-inheritance MRO + self.name = name # set name property and ensure superclass uses same name + super(Filer, self).__init__(name=self.name, **kwa) # Mixin for Mult-inheritance MRO # ensure relative path parts are relative because of odd path.join behavior if os.path.isabs(name): @@ -134,7 +139,6 @@ def __init__(self, *, name='main', base="", temp=False, headDirPath=None, if os.path.isabs(base): raise hioing.FilerError(f"Not relative {base=} path.") - self.name = name self.base = base self.temp = True if temp else False self.headDirPath = headDirPath if headDirPath is not None else self.HeadDirPath @@ -150,6 +154,29 @@ def __init__(self, *, name='main', base="", temp=False, headDirPath=None, if reopen: self.reopen(clear=clear, reuse=reuse, clean=clean, **kwa) + @property + def name(self): + """Property getter for ._name + + Returns: + name (str): unique identifier of instance used as unique path + component in directory or file path name + """ + return self._name + + + @name.setter + def name(self, name): + """Property setter for ._name + + Parameters: + name (str): unique identifier of instance used as unique path + component in directory or file path name + """ + if (not hasattr(self, "_name") or + (hasattr(self, "_name") and name != self._name)): + self._name = name + def reopen(self, temp=None, headDirPath=None, perm=None, clear=False, reuse=False, clean=False, mode=None, fext=None, **kwa): diff --git a/src/hio/base/hier/__init__.py b/src/hio/base/hier/__init__.py index 23dd261..4a839e4 100644 --- a/src/hio/base/hier/__init__.py +++ b/src/hio/base/hier/__init__.py @@ -15,4 +15,5 @@ from .holding import Hold from .durqing import Durq from .dusqing import Dusq +from .hogging import Hog diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 51e419d..81eb142 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -156,8 +156,6 @@ class ActBase(Mixin): Index (int): default naming index for subclass instances. Each subclass overrides with a subclass specific Index value to track subclass specific instance default names. - Names (tuple[str]): tuple of aliases (names) under which this subclas - appears in .Registry. Created by @register Attributes: hold (Hold): data shared by boxwork @@ -169,16 +167,14 @@ class ActBase(Mixin): nabe (str): action nabe (context) for .act Hidden - ._name (str|None): unique name of instance - ._iopts (dict): input-output-paramters for .act - ._nabe (str): action nabe (context) for .act + _name (str): unique name of instance for .name property + _iopts (dict): input-output-paramters for .act for .iops property + _nabe (str): action nabe (context) for .act for .nabe property """ Registry = {} # subclass registry Instances = {} # instance name registry Index = 0 # naming index for default names of this subclasses instances - #Names = () # tuple of aliases for this subclass created by @register - @classmethod def registerbyname(cls, name=None): @@ -215,24 +211,25 @@ def _clearall(cls): klas._clear() - def __init__(self, *, name=None, iops=None, nabe=Nabes.endo, hold=None, **kwa): """Initialization method for instance. Parameters: name (str|None): unique name of this instance. When None then - generate name from .Index + generate name from .Index. Used for .name property which is + used for registering Act instance in Act registry iops (dict|None): input-output-parameters for .act. When None then set to empty dict. nabe (str): action nabe (context) for act. default is "endo" hold (None|Hold): data shared across boxwork """ - super(ActBase, self).__init__(**kwa) # in case of MRO - self.name = name # set name property + self.name = name # set name property and register in .Instances self._iops = dict(iops) if iops is not None else {} # make copy self._nabe = nabe if nabe is not Nabes.native else Nabes.endo self.hold = hold if hold is not None else Hold() + # ensure super class uses same name + super(ActBase, self).__init__(name=self.name, **kwa) # in case of MRO for .__init__ def __call__(self): diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py new file mode 100644 index 0000000..fb16a48 --- /dev/null +++ b/src/hio/base/hier/hogging.py @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +""" +hio.base.hier.hogging Module + +Provides support for hold logging (hogging) + + +""" +from __future__ import annotations # so type hints of classes get resolved later + +from ..filing import Filer +from .acting import Act + +class Hog(Act, Filer): + """Hog is Act that supports metrical logging of hold items based on logging + rules such as time period, update, or change. + + Act comes before Filer in .__mro__ so Act.name property is used not Filer.name + + Inherited Class Attributes: + Registry (dict): subclass registry whose items are (name, cls) where: + name is unique name for subclass + cls is reference to class object + Instances (dict): instance registry whose items are (name, instance) where: + name is unique instance name and instance is instance reference + Index (int): default naming index for subclass instances. Each subclass + overrides with a subclass specific Index value to track + subclass specific instance default names. + + HeadDirPath (str): default abs dir path head such as "/usr/local/var" + TailDirPath (str): default rel dir path tail when using head + CleanTailDirPath (str): default rel dir path tail when creating clean + AltHeadDirPath (str): default alt dir path head such as "~" + as fallback when desired head not permitted. + AltTailDirPath (str): default alt rel dir path tail as fallback + when using alt head. + AltCleanTailDirPath (str): default alt rel path tail when creating clean + TempHeadDir (str): default temp abs dir path head such as "/tmp" + TempPrefix (str): default rel dir path prefix when using temp head + TempSuffix (str): default rel dir path suffix when using temp head and tail + Perm (int): explicit default octal perms such as 0o1700 + Mode (str): open mode such as "r+" + Fext (str): default file extension such as "text" for "fname.text" + + Inherited Attributes see Act, File + hold (Hold): data shared by boxwork + + name (str): overriden by .name property from Act (see name property) + base (str): another unique path component inserted before name + temp (bool): True means use TempHeadDir in /tmp directory + headDirPath (str): head directory path + path (str | None): full directory or file path once created else None + perm (int): octal OS permissions for path directory and/or file + filed (bool): True means .path ends in file. + False means .path ends in directory + extensioned (bool): When not filed: + True means ensure .path ends with fext + False means do not ensure .path ends with fext + mode (str): file open mode if filed + fext (str): file extension if filed + file (File | None): File instance when filed and created. + opened (bool): True means directory created and if filed then file + is opened. False otherwise + + + Inherited Properties see Act, File + name (str): unique name string of instance used for registering Act + instance in Act registry as well providing a unique path + component used in file path name + iops (dict): input-output-parameters for .act + nabe (str): action nabe (context) for .act + + + Hidden: + _name (str): unique name of instance for .name property + _iopts (dict): input-output-paramters for .act for .iops property + _nabe (str): action nabe (context) for .act for .nabe property + + + """ + + def __init__(self, **kwa): + """Initialize instance. + + Inherited Parameters: + name (str|None): unique name of this instance. When None then + generate name from .Index. Used for .name property which is + used for registering Act instance in Act registry as well + providing a unique path component used in file path name. + When system employs more than one installation, name allows + differentiating each installation by name + iops (dict|None): input-output-parameters for .act. When None then + set to empty dict. + nabe (str): action nabe (context) for act. default is "endo" + hold (None|Hold): data shared across boxwork + + base (str): optional directory path segment inserted before name + that allows further differentation with a hierarchy. "" means + optional. + temp (bool): assign to .temp + True then open in temporary directory, clear on close + Otherwise then open persistent directory, do not clear on close + headDirPath (str): optional head directory pathname for main database + Default .HeadDirPath + perm (int): optional numeric os dir permissions for database + directory and database files. Default .DirMode + reopen (bool): True (re)open with this init + False not (re)open with this init but later (default) + clear (bool): True means remove directory upon close when reopening + False means do not remove directory upon close when reopening + reuse (bool): True means reuse self.path if already exists + False means do not reuse but remake self.path + clean (bool): True means path uses clean tail variant + False means path uses normal tail variant + filed (bool): True means .path is file path not directory path + False means .path is directiory path not file path + extensioned (bool): When not filed: + True means ensure .path ends with fext + False means do not ensure .path ends with fext + mode (str): File open mode when filed + fext (str): File extension when filed or extensioned + """ + super(Hog, self).__init__(**kwa) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py new file mode 100644 index 0000000..fb6adb7 --- /dev/null +++ b/tests/base/hier/test_hogging.py @@ -0,0 +1,18 @@ +# -*- encoding: utf-8 -*- +"""tests.base.hier.test_acting module + +""" +from __future__ import annotations # so type hints of classes get resolved later + +import pytest + +from hio.base.hier import Hog + +def test_hog_basic(): + """Test Hog class""" + + """Done Test""" + + +if __name__ == "__main__": + test_hog_basic() From 2abd7c0d708ffecffb6621e132fbadb652ece48b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 14:15:14 -0600 Subject: [PATCH 02/36] Some basic tests of Hog as subclass of Act and Filer --- src/hio/base/filing.py | 18 +++-- src/hio/base/hier/__init__.py | 2 +- src/hio/base/hier/hogging.py | 107 +++++++++++++++++++++++- src/hio/help/helping.py | 5 +- tests/base/hier/test_hogging.py | 139 +++++++++++++++++++++++++++++++- tests/base/test_filing.py | 6 +- 6 files changed, 258 insertions(+), 19 deletions(-) diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index 9ef5384..a2c4c6e 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -130,12 +130,14 @@ def __init__(self, *, name='main', base="", temp=False, headDirPath=None, fext (str): File extension when filed or extensioned """ - self.name = name # set name property and ensure superclass uses same name + if not hasattr(self, "_name") or name != self.name: # avoid collision subclass + self.name = name + super(Filer, self).__init__(name=self.name, **kwa) # Mixin for Mult-inheritance MRO # ensure relative path parts are relative because of odd path.join behavior - if os.path.isabs(name): - raise hioing.FilerError(f"Not relative {name=} path.") + if os.path.isabs(self.name): + raise hioing.FilerError(f"Not relative name={self.name} path.") if os.path.isabs(base): raise hioing.FilerError(f"Not relative {base=} path.") @@ -170,12 +172,12 @@ def name(self, name): """Property setter for ._name Parameters: - name (str): unique identifier of instance used as unique path - component in directory or file path name + name (str): unique identifier of instance """ - if (not hasattr(self, "_name") or - (hasattr(self, "_name") and name != self._name)): - self._name = name + #if not Renam.match(name): + #raise HierError(f"Invalid {name=}.") + self._name = name + def reopen(self, temp=None, headDirPath=None, perm=None, clear=False, diff --git a/src/hio/base/hier/__init__.py b/src/hio/base/hier/__init__.py index 4a839e4..60e0599 100644 --- a/src/hio/base/hier/__init__.py +++ b/src/hio/base/hier/__init__.py @@ -15,5 +15,5 @@ from .holding import Hold from .durqing import Durq from .dusqing import Dusq -from .hogging import Hog +from .hogging import Hog, openHog, HogDoer diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index fb16a48..01adb2f 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -8,9 +8,14 @@ """ from __future__ import annotations # so type hints of classes get resolved later +from contextlib import contextmanager + +from ..doing import Doer from ..filing import Filer -from .acting import Act +from .acting import Act, register + +@register() class Hog(Act, Filer): """Hog is Act that supports metrical logging of hold items based on logging rules such as time period, update, or change. @@ -79,7 +84,7 @@ class Hog(Act, Filer): """ - def __init__(self, **kwa): + def __init__(self, filed=True, extensioned=True, fext="hog", **kwa): """Initialize instance. Inherited Parameters: @@ -120,4 +125,100 @@ def __init__(self, **kwa): mode (str): File open mode when filed fext (str): File extension when filed or extensioned """ - super(Hog, self).__init__(**kwa) + super(Hog, self).__init__(filed=filed, + extensioned=extensioned, + fext=fext, + **kwa) + + + +@contextmanager +def openHog(cls=None, name=None, temp=True, reopen=True, clear=False, **kwa): + """Context manager wrapper Hog instances for managing a filesystem directory + and or files in a directory. + + Defaults to using temporary directory path. + Context 'with' statements call .close on exit of 'with' block + + Parameters: + cls is Class instance of subclass instance + name is str name of Filer instance path part so can have multiple Filers + at different paths that each use different dirs or files + temp is Boolean, True means open in temporary directory, clear on close + Otherwise open in persistent directory, do not clear on close + reopen (bool): True (re)open with this init + False not (re)open with this init but later (default) + clear (bool): True means remove directory upon close when reopening + False means do not remove directory upon close when reopening + See hogging.Hog for other keyword parameter passthroughs + + Usage: + + with openHog(name="bob") as hog: + + with openHog(name="eve", cls=HogSubClass) as hog: + + """ + hog = None + if cls is None: + cls = Hog + try: + hog = cls(name=name, temp=temp, reopen=reopen, clear=clear, **kwa) + yield hog + + finally: + if hog: + hog.close(clear=hog.temp or clear) # clears if hog.temp + + + +class HogDoer(Doer): + """ + Basic Hog Doer + + Attributes: + done (bool): completion state: + True means completed + Otherwise incomplete. Incompletion maybe due to close or abort. + hog (Hog): instance + + Properties: + tyme (float): relative cycle time of associated Tymist .tyme obtained + via injected .tymth function wrapper closure. + tymth (func): closure returned by Tymist .tymeth() method. + When .tymth is called it returns associated Tymist .tyme. + .tymth provides injected dependency on Tymist tyme base. + tock (float)): desired time in seconds between runs or until next run, + non negative, zero means run asap + + """ + + def __init__(self, hog, **kwa): + """ + Parameters: + tymist (Tymist): instance + tock (float): initial value of .tock in seconds + hog (Hog): instance + """ + super(HogDoer, self).__init__(**kwa) + self.hog = hog + + + def enter(self, *, temp=None): + """Do 'enter' context actions. Override in subclass. Not a generator method. + Set up resources. Comparable to context manager enter. + + Parameters: + temp (bool | None): True means use temporary file resources if any + None means ignore parameter value. Use self.temp + + Inject temp or self.temp into file resources here if any + """ + # inject temp into file resources here if any + if not self.hog.opened: + self.hog.reopen(temp=temp) + + + def exit(self): + """""" + self.hog.close(clear=self.hog.temp) diff --git a/src/hio/help/helping.py b/src/hio/help/helping.py index 02f3e79..cb7f8c2 100644 --- a/src/hio/help/helping.py +++ b/src/hio/help/helping.py @@ -384,15 +384,14 @@ def ocfn(path, mode='r+', perm=(stat.S_IRUSR | stat.S_IWUSR)): 384 == 0o600 436 == octal 0664 """ - try: - + try: # create new file newfd = os.open(path, os.O_EXCL | os.O_CREAT | os.O_RDWR, perm) if "b" in mode: file = os.fdopen(newfd,"w+b") # w+ truncate read and/or write else: file = os.fdopen(newfd,"w+") # w+ truncate read and/or write - except OSError as ex: + except OSError as ex: # open existing file with mode if ex.errno == errno.EEXIST: file = open(path, mode) # r+ do not truncate read and/or write else: diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index fb6adb7..c323bce 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -4,15 +4,152 @@ """ from __future__ import annotations # so type hints of classes get resolved later +import os +import platform +import tempfile + + import pytest -from hio.base.hier import Hog +from hio.base import Doist +from hio.base.hier import Hog, openHog, HogDoer + + def test_hog_basic(): """Test Hog class""" + Hog._clearall() # clear Hog.Instances for debugging + + hog = Hog(temp=True) # test defaults + assert hog.temp + assert hog.name == "Hog0" + assert hog.opened + assert hog.filed + assert hog.extensioned + assert hog.fext == 'hog' + assert hog.file + assert not hog.file.closed + assert os.path.exists(hog.path) + + tempDirPath = (os.path.join(os.path.sep, "tmp") + if platform.system() == "Darwin" + else tempfile.gettempdir()) + tempDirPath = os.path.normpath(tempDirPath) + path = os.path.normpath(hog.path) # '/tmp/hio_u36wdtp5_test/hio/Hog1.hog' + assert path.startswith(os.path.join(tempDirPath, "hio_")) + assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + + """Done Test""" + + +def test_open_hog(): + """Test openHog context manager""" + Hog._clearall() # clear Hog.Instances for debugging + + with openHog() as hog: # test defaults + assert hog.temp + assert hog.name == "Hog0" + assert hog.opened + assert hog.filed + assert hog.extensioned + assert hog.fext == 'hog' + assert hog.file + assert not hog.file.closed + assert os.path.exists(hog.path) + + tempDirPath = (os.path.join(os.path.sep, "tmp") + if platform.system() == "Darwin" + else tempfile.gettempdir()) + tempDirPath = os.path.normpath(tempDirPath) + path = os.path.normpath(hog.path) # '/tmp/hio_u36wdtp5_test/hio/Hog1.hog' + assert path.startswith(os.path.join(tempDirPath, "hio_")) + assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) """Done Test""" +def test_hog_doer(): + """Test HogDoer""" + Hog._clearall() # clear Hog.Instances for debugging + + # create two HogDoer instances and run them + + hog0 = Hog(name='test0', temp=True, reopen=False) + assert hog0.temp + assert hog0.name == "test0" + assert not hog0.opened + assert hog0.filed + assert hog0.extensioned + assert hog0.fext == 'hog' + assert not hog0.file + assert not hog0.path + + hogDoer0 = HogDoer(hog=hog0) + assert hogDoer0.hog == hog0 + assert not hogDoer0.hog.opened + + hog1 = Hog(name='test1', temp=True, reopen=False) + assert not hog1.opened + assert not hog1.file + assert not hog1.path + + hogDoer1 = HogDoer(hog=hog1) + assert hogDoer1.hog == hog1 + assert not hogDoer1.hog.opened + + limit = 0.25 + tock = 0.03125 + doist = Doist(limit=limit, tock=tock) + + doers = [hogDoer0, hogDoer1] + + doist.doers = doers + doist.enter() + assert len(doist.deeds) == 2 + assert [val[1] for val in doist.deeds] == [0.0, 0.0] # retymes + for doer in doers: + assert doer.hog.opened + assert doer.hog.file + assert os.path.join('_test', 'hio', 'test') in doer.hog.path + + doist.recur() + assert doist.tyme == 0.03125 # on next cycle + assert len(doist.deeds) == 2 + for doer in doers: + assert doer.hog.opened + assert doer.hog.file + + for dog, retyme, index in doist.deeds: + dog.close() + + for doer in doers: + assert not doer.hog.opened + assert not doer.hog.file + assert not os.path.exists(doer.hog.path) + + + # start over but not opened + doist.tyme = 0.0 + doist.do(doers=doers) + assert doist.tyme == limit + for doer in doers: + assert not doer.hog.opened + assert not doer.hog.file + assert not os.path.exists(doer.hog.path) + + """End Test""" + if __name__ == "__main__": test_hog_basic() + test_open_hog() + test_hog_doer() diff --git a/tests/base/test_filing.py b/tests/base/test_filing.py index f698f13..114554c 100644 --- a/tests/base/test_filing.py +++ b/tests/base/test_filing.py @@ -307,9 +307,9 @@ def test_filing(): #test openfiler with defaults temp == True with filing.openFiler() as filer: - tempDirPath = os.path.join(os.path.sep, "tmp") if platform.system() == "Darwin" else tempfile.gettempdir() - #dirPath = os.path.join(tempDirPath, 'hio_hcbvwdnt_test', 'hio', 'test') - #assert dirPath.startswith(os.path.join(tempDirPath, 'hio_')) + tempDirPath = (os.path.join(os.path.sep, "tmp") + if platform.system() == "Darwin" + else tempfile.gettempdir()) tempDirPath = os.path.normpath(tempDirPath) path = os.path.normpath(filer.path) assert path.startswith(os.path.join(tempDirPath, "hio_")) From 79834098e25da5e6b6d4569e7762c3464f754e93 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 16:53:10 -0600 Subject: [PATCH 03/36] Added alias for Hog of log and Log in Act Registry --- src/hio/base/hier/acting.py | 20 +++++++++++++------- src/hio/base/hier/hogging.py | 2 +- tests/base/hier/test_hogging.py | 8 ++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 81eb142..c3fe12c 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -262,16 +262,22 @@ def name(self, name=None): name (str|None): unique identifier of instance. When None generate unique name using .Index """ - while name is None or name in self.Instances: - name = self.__class__.__name__ + str(self.Index) - self.__class__.Index += 1 # do not shadow class attribute + if not hasattr(self, "_name"): # ._name not yet assigned + while name is None or name in self.Instances: + name = self.__class__.__name__ + str(self.Index) + self.__class__.Index += 1 # do not shadow class attribute - if not Renam.match(name): - raise HierError(f"Invalid {name=}.") + if not Renam.match(name): + raise HierError(f"Invalid {name=}.") - self.Instances[name] = self - self._name = name + self._name = name + if self.name not in self.Instances: + self.Instances[self.name] = self + + if self.Instances[self.name] is not self: + raise HierError(f"Another {self.__class__.__name__} instance" + f" named {self.name} already assigned to .Instances") @property def iops(self): diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 01adb2f..e17253e 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -15,7 +15,7 @@ from .acting import Act, register -@register() +@register(names=('log', 'Log')) class Hog(Act, Filer): """Hog is Act that supports metrical logging of hold items based on logging rules such as time period, update, or change. diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index c323bce..ef3a627 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -39,6 +39,10 @@ def test_hog_basic(): assert path.startswith(os.path.join(tempDirPath, "hio_")) assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + assert Hog.Registry[Hog.__name__] is Hog + assert Hog.Registry['log'] is Hog + assert Hog.Registry['Log'] is Hog + hog.close(clear=True) assert not hog.opened assert not hog.file @@ -71,6 +75,10 @@ def test_open_hog(): assert path.startswith(os.path.join(tempDirPath, "hio_")) assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + assert Hog.Registry[Hog.__name__] is Hog + assert Hog.Registry['log'] is Hog + assert Hog.Registry['Log'] is Hog + assert not hog.opened assert not hog.file assert not os.path.exists(hog.path) From 1913dd46acc5f976a3a4801dd6e01daf0a022cb3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 18:09:19 -0600 Subject: [PATCH 04/36] some clean up --- src/hio/base/hier/acting.py | 6 +++--- src/hio/base/hier/boxing.py | 2 +- src/hio/base/hier/hogging.py | 10 +++++++--- tests/base/hier/test_hogging.py | 8 +++++++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index c3fe12c..982929a 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -397,7 +397,7 @@ def act(self, **iops): # passed in by call if not self.compiled: # not yet compiled so lazy self.compile() # first time only recompile to ._code H = self.hold # ensure H is in locals() for exec - # note iops already in locals() for exec + # note iops as this method's parameter already in locals() for exec return exec(self._code) @@ -513,11 +513,11 @@ def __init__(self, dest=None, need=None, **kwa): When Need instance then use directly """ - kwa.update(nabe=Nabes.godo) # override must be godo nabe + kwa.update(nabe=Nabes.godo) # override parameter, must be godo nabe super(Goact, self).__init__(**kwa) self.dest = dest if dest is not None else 'next' # default is next self.need = need if need is not None else Need() # default need evals to True - if self.nabe != Nabes.godo: + if self.nabe != Nabes.godo: # in case super class overrides nabe raise HierError(f"Invalid nabe='{self.nabe}' for Goact " f"'{self.name}'") diff --git a/src/hio/base/hier/boxing.py b/src/hio/base/hier/boxing.py index e1467a3..dbe5714 100644 --- a/src/hio/base/hier/boxing.py +++ b/src/hio/base/hier/boxing.py @@ -876,7 +876,7 @@ def do(self, deed: None|str|Type[ActBase]|Callable=None, nabe=None, *, else: # deed is registered act class name or alias act = klas(**parms) # create act from klas with **parms - nabe = act.nabe # act init may override passed in nabe + nabe = act.nabe # act init may ignore nabe parameter passed to init try: getattr(m.box, nabeDispatch[nabe]).append(act) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index e17253e..1723dcb 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -12,11 +12,12 @@ from ..doing import Doer from ..filing import Filer -from .acting import Act, register +from .hiering import Nabes +from .acting import ActBase, register @register(names=('log', 'Log')) -class Hog(Act, Filer): +class Hog(ActBase, Filer): """Hog is Act that supports metrical logging of hold items based on logging rules such as time period, update, or change. @@ -84,7 +85,8 @@ class Hog(Act, Filer): """ - def __init__(self, filed=True, extensioned=True, fext="hog", **kwa): + def __init__(self, filed=True, extensioned=True, fext="hog", + nabe=Nabes.afdo, **kwa): """Initialize instance. Inherited Parameters: @@ -128,7 +130,9 @@ def __init__(self, filed=True, extensioned=True, fext="hog", **kwa): super(Hog, self).__init__(filed=filed, extensioned=extensioned, fext=fext, + nabe=nabe, **kwa) + #self.iops.update(H=self.hold) # inject .hold diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index ef3a627..6909581 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -12,7 +12,7 @@ import pytest from hio.base import Doist -from hio.base.hier import Hog, openHog, HogDoer +from hio.base.hier import Nabes, Hog, openHog, HogDoer @@ -43,6 +43,9 @@ def test_hog_basic(): assert Hog.Registry['log'] is Hog assert Hog.Registry['Log'] is Hog + assert hog() == {} # default returns iops + assert hog.nabe == Nabes.afdo + hog.close(clear=True) assert not hog.opened assert not hog.file @@ -79,6 +82,9 @@ def test_open_hog(): assert Hog.Registry['log'] is Hog assert Hog.Registry['Log'] is Hog + assert hog() == {} # default returns iops + assert hog.nabe == Nabes.afdo + assert not hog.opened assert not hog.file assert not os.path.exists(hog.path) From 662f6e70ca5ac05e3e6adb2f6f301b2e089641ab Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 18:19:32 -0600 Subject: [PATCH 05/36] clean up doc strings --- src/hio/base/hier/acting.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 982929a..03177e6 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -372,12 +372,14 @@ def __init__(self, deed=None, **kwa): dock (None|Dock): durable bags in dock (on disc) shared by boxwork Parameters: - deed (None|Callable): callable to be actioned with iops + deed (None|str|Callable): compilable exec str or callable to be + actioned with iops + None means use default lambda """ super(Act, self).__init__(**kwa) self._code = None - self.iops.update(H=self.hold) # inject .hold + self.iops.update(H=self.hold) # inject .hold so callable deed sees it self.deed = deed if deed is not None else (lambda **iops: iops) if not callable(self.deed): # need to compile self.compile() # compile at init time so know if compilable @@ -387,16 +389,17 @@ def act(self, **iops): # passed in by call """Act called by ActBase. Parameters: - iops (dict): input output parms for deed when deed is callable. + iops (dict): input output parms for deed when deed is callable not + compilable exec string """ - if callable(self.deed): + if callable(self.deed): # not compilable str return self.deed(**iops) if not self.compiled: # not yet compiled so lazy self.compile() # first time only recompile to ._code - H = self.hold # ensure H is in locals() for exec + H = self.hold # ensure H is in locals() for exec when deed is str # note iops as this method's parameter already in locals() for exec return exec(self._code) @@ -406,7 +409,7 @@ def deed(self): """Property getter for ._deed Returns: - deed (str): evalable boolean expression or callable. + deed (str|Callable): compilable exec statement str or callable. """ return self._deed @@ -416,7 +419,7 @@ def deed(self, deed): """Property setter for ._expr Parameters: - expr (str): evalable boolean expression. + deed (str|Callable): compilable exec statement str or Callable """ self._deed = deed self._code = None # force lazy recompilation @@ -442,8 +445,6 @@ def compile(self): self._code = compile(self.deed, '', 'exec') - - @register() class Goact(ActBase): """Goact (go act) is subclass of ActBase whose .act evaluates conditional From 642d267a381fa3304db0b6f1dc499f620c4f82da Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 18:21:34 -0600 Subject: [PATCH 06/36] clean up doc strings mroe --- src/hio/base/hier/acting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 03177e6..acb12a6 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -422,7 +422,7 @@ def deed(self, deed): deed (str|Callable): compilable exec statement str or Callable """ self._deed = deed - self._code = None # force lazy recompilation + self._code = None # force lazy recompilation when compilable (not Callable) @property @@ -431,7 +431,7 @@ def compiled(self): Returns: compiled (bool): True means ._code holds compiled ._expr - False means not yet compiled + False means not yet compiled or Callable """ return True if self._code is not None else False From 9d2405898cb313ee2dc6da9c9908e448e3802595 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Aug 2025 18:32:45 -0600 Subject: [PATCH 07/36] more refinement of doc strings --- src/hio/base/hier/acting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index acb12a6..a2b4b8b 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -395,12 +395,12 @@ def act(self, **iops): # passed in by call """ if callable(self.deed): # not compilable str - return self.deed(**iops) + return self.deed(**iops) # injected H=self.hold into iops - if not self.compiled: # not yet compiled so lazy + if not self.compiled: # deed str not yet compiled so lazy compile self.compile() # first time only recompile to ._code - H = self.hold # ensure H is in locals() for exec when deed is str - # note iops as this method's parameter already in locals() for exec + H = self.hold # ensure H is in locals() for exec compile deed str + # note iops dict value of method parameter also in locals() for exec return exec(self._code) From d398cdd10b8ca226487ddc1b898ca15937c3cb79 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Aug 2025 06:57:11 -0600 Subject: [PATCH 08/36] added .act method to Hog stub --- src/hio/base/hier/hogging.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 1723dcb..7c1f1ba 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -132,7 +132,16 @@ def __init__(self, filed=True, extensioned=True, fext="hog", fext=fext, nabe=nabe, **kwa) - #self.iops.update(H=self.hold) # inject .hold + + def act(self, **iops): + """Act called by ActBase. + + Parameters: + iops (dict): input output parms + + """ + + return iops From f35e83ec6f70418ee4a5b9d427ddfa3e8146e34e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Aug 2025 16:58:53 -0600 Subject: [PATCH 09/36] setup of attributes and parameters for Hog to log --- src/hio/base/hier/acting.py | 70 +++++++++++++---------- src/hio/base/hier/hogging.py | 99 ++++++++++++++++++++++++++++++++- tests/base/hier/test_hogging.py | 6 ++ 3 files changed, 143 insertions(+), 32 deletions(-) diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index a2b4b8b..1dd6ebf 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -234,13 +234,21 @@ def __init__(self, *, name=None, iops=None, nabe=Nabes.endo, hold=None, **kwa): def __call__(self): """Make ActBase instance a callable object. - Call its .act method with self.iops as parameters + Call its .act method with expanded self.iops as parameters + + This enables compiled exec/eval str to have **iops in local scope """ - return self.act(**self.iops) + return self.act(**self.iops) # put self.iops into local scope of .act def act(self, **iops): - """Act called by Actor. Should override in subclass.""" + """Act called by Actor. Should override in subclass. + + Parameters: + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str + + """ return iops # for debugging @@ -299,9 +307,6 @@ def nabe(self): return self._nabe - - - @register() class Act(ActBase): """Act for do verb deeds as or executable statements orcallables. @@ -389,13 +394,11 @@ def act(self, **iops): # passed in by call """Act called by ActBase. Parameters: - iops (dict): input output parms for deed when deed is callable not - compilable exec string - - + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ if callable(self.deed): # not compilable str - return self.deed(**iops) # injected H=self.hold into iops + return self.deed(**self.iops) # injected H=self.hold into iops if not self.compiled: # deed str not yet compiled so lazy compile self.compile() # first time only recompile to ._code @@ -528,7 +531,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ if self.need(): @@ -624,7 +628,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -729,7 +734,8 @@ def act(self, **iops): # passed in by call """Act called by ActBase. Parameters: - iops (dict): input output parms for deed when deed is callable. + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ key, field = self.lhs @@ -737,7 +743,7 @@ def act(self, **iops): # passed in by call self.hold[key][field] = self.rhs elif callable(self.rhs): - self.hold[key][field] = self.rhs(**iops) + self.hold[key][field] = self.rhs(**self.iops) else: if not self.compiled: # not yet compiled so lazy @@ -820,10 +826,6 @@ def compile(self): self._code = compile(self.rhs, '', 'eval') - - -# Dark DockMark - @register() class Mark(ActBase): """Mark (Mine Mark) is base classubclass of ActBase whose .act marks a @@ -913,7 +915,8 @@ def act(self, **iops): Override in subclass Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -963,7 +966,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1016,7 +1020,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1068,7 +1073,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1122,7 +1128,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1223,7 +1230,8 @@ def act(self, **iops): Override in subclass Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1275,7 +1283,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1331,7 +1340,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1387,7 +1397,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name @@ -1444,7 +1455,8 @@ def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str """ boxer = self.iops['_boxer'] # get boxer name diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 7c1f1ba..1f51f13 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -48,6 +48,13 @@ class Hog(ActBase, Filer): Mode (str): open mode such as "r+" Fext (str): default file extension such as "text" for "fname.text" + Class Attributes: + ReservedTags (dict[str]): of reserved tags to protect from collision with + defined parameter names that may not be used for + log tags when using **kwa to provide hold + log leys. Uses dict since inclusion test is + faster than with list + Inherited Attributes see Act, File hold (Hold): data shared by boxwork @@ -77,6 +84,35 @@ class Hog(ActBase, Filer): nabe (str): action nabe (context) for .act + Attributes: + begun (bool): True means logging has begun with header + False means logging has not yet begun needs header + began (str|None): realtime datetime stamp of when logging began, may be non 0.0 + None means not yet running + first (float|None): tyme when began logging, None means not yet running + last (float|None): tyme when last logged, None means not yet running + realtime equiv of last = began + (last - first) + rule (str): condition for log to fire one of + ('once', 'every', 'spanned', 'updated', 'changed') + span (float): tyme span for periodic logging + header (str): header for log file(s) + flushSpan (float): tyme span between flushes (flush) + flushLast (float|None): tyme last flushed, None means not yet running + cycleCount (int): number of cycled logs, 0 means do not cycle (count) + cyclePaths (list[str]): paths for cycled logs + cycleSpan (float): min tyme span for cycling logs (cycle). 0.0 means + cycle based on cycleHigh not tyme span. One of + cycleSpan or cycleHigh must be non zero + cycleLast (float|None): tyme last cycled. None means not yet running + cycleLow (int): minimum size in bytes required for cycling log based on + cycleSpan. 0 means no minimum + cycleHigh (int): maximum size in bytes allowed for each cycled log + 0 means no maximum. One of cycleSpan or cycleHigh must + be non zero + logs (dict): labeled keys of holds to log. Item labels are log tags + logs item value is key in hold that provides value to log + logs item key is tag in log header + Hidden: _name (str): unique name of instance for .name property _iopts (dict): input-output-paramters for .act for .iops property @@ -84,9 +120,15 @@ class Hog(ActBase, Filer): """ + ReservedTags = dict(name=True, iops=True, nabe=True, hold=True, base=True, + temp=True, headDirPath=True, perm=True, reopen=True, + clear=True, reuse=True, clean=True, filed=True, + extensioned=True, ) - def __init__(self, filed=True, extensioned=True, fext="hog", - nabe=Nabes.afdo, **kwa): + + def __init__(self, nabe=Nabes.afdo, filed=True, extensioned=True, fext="hog", + rule=None, span=0.0, flush=0.0, + count=0, cycle=0.0, low=0, high=0, logs=None, **kwa): """Initialize instance. Inherited Parameters: @@ -126,6 +168,28 @@ def __init__(self, filed=True, extensioned=True, fext="hog", False means do not ensure .path ends with fext mode (str): File open mode when filed fext (str): File extension when filed or extensioned + + Parameters: + rule (str|None): condition for log to fire, default every + (once, every, spanned, updated, changed) + span (float): periodic time span when rule is spanned. 0.0 means + every tyme + flush (float): flush tyme span, tyme between flushes, 0.0 means + every tyme + count (int): number of cycled logs, 0 means do not cycle + cycle (float): cycle tyme span, tyme between log cycles + low (int): minimum size in bytes required for cycling log + 0 means no minimum + high (int): maximum size in bytes allowed for each cycled log + 0 means no maximum + logs (None|dict): labeled keys of holds to log. Item labels are tags + logs item value is key in hold that provides value to log + logs item key is tag in log header + None means use unreserved items in **kwa wrt .ReservedTags + + + When made (created and inited) by boxer.do then have "_boxer" and + "_box" keys in self.iops = dict(_boxer=self.name, _box=m.box.name, **iops) """ super(Hog, self).__init__(filed=filed, extensioned=extensioned, @@ -133,12 +197,41 @@ def __init__(self, filed=True, extensioned=True, fext="hog", nabe=nabe, **kwa) + self.begun = None + self.began = None + self.first = None + self.last = None + self.rule = rule if rule is not None else "every" + self.span = span + self.header = '' # need to init + self.flushSpan = flush + self.flushLast = None + self.cycleCount = count + self.cyclePaths = [] # need to init + self.cycleSpan = cycle + self.cycleLast = None + self.cycleLow = low + self.cycleHigh = high + + if logs is None: + logs = {} + for tag, val in kwa.items(): + if tag not in self.ReservedTags: + logs[tag] = val # value is key of hold to log + + self.logs = logs + + + def act(self, **iops): """Act called by ActBase. Parameters: - iops (dict): input output parms + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str + When made by boxer.do then have "_boxer" and "_box" keys in self.iops + iops = dict(_boxer=self.name, _box=m.box.name, **iops) """ return iops diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 6909581..0c5b135 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -45,6 +45,9 @@ def test_hog_basic(): assert hog() == {} # default returns iops assert hog.nabe == Nabes.afdo + assert hog.hold + assert not hog.hold.subery + assert hog.logs == {} hog.close(clear=True) assert not hog.opened @@ -84,6 +87,9 @@ def test_open_hog(): assert hog() == {} # default returns iops assert hog.nabe == Nabes.afdo + assert hog.hold + assert not hog.hold.subery + assert hog.logs == {} assert not hog.opened assert not hog.file From 57b7429065b17c25633b41aff8bd3dc98eb6ab3e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Aug 2025 10:23:15 -0600 Subject: [PATCH 10/36] some clean up --- src/hio/base/filing.py | 2 +- src/hio/base/hier/acting.py | 112 +++++++++++++++++------------------ src/hio/base/hier/hogging.py | 25 +++++++- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index a2c4c6e..e1c9754 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -226,7 +226,7 @@ def reopen(self, temp=None, headDirPath=None, perm=None, clear=False, mode=self.mode, fext=self.fext, **kwa) - elif self.filed: # assumes dir in self.path exists + elif self.filed: # would not be here unless self.path already exists self.file = ocfn(self.path, mode=self.mode) self.opened = True if not self.filed else self.file and not self.file.closed diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 1dd6ebf..0aeed6d 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -613,13 +613,13 @@ def __init__(self, nabe=Nabes.endo, **kwa): super(EndAct, self).__init__(nabe=nabe, **kwa) try: - boxer = self.iops['_boxer'] # get boxer name + boxerName = self.iops['_boxer'] # get boxer name except KeyError as ex: raise HierError(f"Missing iops '_boxer' for '{self.name}' instance " f"of Act self.__class__.__name__") from ex - keys = ("", "boxer", boxer, "end") # _boxer_boxername_end + keys = ("", "boxer", boxerName, "end") # _boxer_boxername_end if keys not in self.hold: self.hold[keys] = Bag() # create bag at end default value = None @@ -632,8 +632,8 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - keys = ("", "boxer", boxer, "end") + boxerName = self.iops['_boxer'] # get boxer name + keys = ("", "boxer", boxerName, "end") self.hold[keys].value = True @@ -919,8 +919,8 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name @@ -955,9 +955,9 @@ def __init__(self, nabe=Nabes.enmark, **kwa): """ super(LapseMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "lapse") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "lapse") if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -970,9 +970,9 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "lapse") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "lapse") # mark box tyme via bag._now tyme self.hold[keys].value = self.hold[keys]._now # _now tyme of mark bag return self.hold[keys].value @@ -1009,9 +1009,9 @@ def __init__(self, nabe=Nabes.remark, **kwa): """ super(RelapseMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "relapse") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "relapse") if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1024,9 +1024,9 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "relapse") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "relapse") # mark box tyme via bag._now tyme self.hold[keys].value = self.hold[keys]._now # _now tyme of mark bag return self.hold[keys].value @@ -1061,9 +1061,9 @@ def __init__(self, nabe=Nabes.redo, **kwa): """ super(Count, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "count") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "count") if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1077,9 +1077,9 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "count") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "count") bag = self.hold[keys] # count bag if bag.value is None: bag.value = 0 # start counter @@ -1117,9 +1117,9 @@ def __init__(self, nabe=Nabes.exdo, **kwa): """ super(Discount, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "count") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "count") if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1132,9 +1132,9 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name - keys = ("", "boxer", boxer, "box", box, "count") + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name + keys = ("", "boxer", boxerName, "box", boxName, "count") self.hold[keys].value = None # reset count to None return self.hold[keys].value @@ -1234,8 +1234,8 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name key = self.iops['_key'] # get bag key @@ -1270,10 +1270,10 @@ def __init__(self, nabe=Nabes.enmark, **kwa): """ super(UpdateMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name key = self.iops['_key'] # get bag key - keys = ("", "boxer", boxer, "box", box, "update", key) + keys = ("", "boxer", boxerName, "box", boxName, "update", key) if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1287,10 +1287,10 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] key = self.iops['_key'] - keys = ("", "boxer", boxer, "box", box, "update", key) + keys = ("", "boxer", boxerName, "box", boxName, "update", key) # mark bag tyme self.hold[keys].value = self.hold[key]._tyme return self.hold[keys].value @@ -1326,11 +1326,11 @@ def __init__(self, nabe=Nabes.remark, **kwa): """ super(ReupdateMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name key = self.iops['_key'] # get bag key - keys = ("", "boxer", boxer, "box", box, "reupdate", key) + keys = ("", "boxer", boxerName, "box", boxName, "reupdate", key) if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1344,10 +1344,10 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] key = self.iops['_key'] - keys = ("", "boxer", boxer, "box", box, "reupdate", key) + keys = ("", "boxer", boxerName, "box", boxName, "reupdate", key) # mark bag tyme self.hold[keys].value = self.hold[key]._tyme return self.hold[keys].value @@ -1384,11 +1384,11 @@ def __init__(self, nabe=Nabes.enmark, **kwa): """ super(ChangeMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name key = self.iops['_key'] # get bag key - keys = ("", "boxer", boxer, "box", box, "change", key) + keys = ("", "boxer", boxerName, "box", boxName, "change", key) if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1401,11 +1401,11 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] key = self.iops['_key'] bag = self.hold[key] - keys = ("", "boxer", boxer, "box", box, "change", key) + keys = ("", "boxer", boxerName, "box", boxName, "change", key) self.hold[keys].value = bag._astuple() # bag field value tuple as mark return self.hold[keys].value @@ -1442,11 +1442,11 @@ def __init__(self, nabe=Nabes.remark, **kwa): """ super(RechangeMark, self).__init__(nabe=nabe, **kwa) - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] # get box name + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] # get box name key = self.iops['_key'] # get bag key - keys = ("", "boxer", boxer, "box", box, "rechange", key) + keys = ("", "boxer", boxerName, "box", boxName, "rechange", key) if keys not in self.hold: self.hold[keys] = Bag() # create bag default value = None @@ -1459,11 +1459,11 @@ def act(self, **iops): local scope in case act compliles exec/eval str """ - boxer = self.iops['_boxer'] # get boxer name - box = self.iops['_box'] + boxerName = self.iops['_boxer'] # get boxer name + boxName = self.iops['_box'] key = self.iops['_key'] bag = self.hold[key] - keys = ("", "boxer", boxer, "box", box, "rechange", key) + keys = ("", "boxer", boxerName, "box", boxName, "rechange", key) self.hold[keys].value = bag._astuple() # bag field value tuple as mark return self.hold[keys].value diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 1f51f13..951cbcd 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -126,8 +126,8 @@ class Hog(ActBase, Filer): extensioned=True, ) - def __init__(self, nabe=Nabes.afdo, filed=True, extensioned=True, fext="hog", - rule=None, span=0.0, flush=0.0, + def __init__(self, iops=None, nabe=Nabes.afdo, filed=True, extensioned=True, fext="hog", + reuse=True, rule=None, span=0.0, flush=0.0, count=0, cycle=0.0, low=0, high=0, logs=None, **kwa): """Initialize instance. @@ -190,13 +190,34 @@ def __init__(self, nabe=Nabes.afdo, filed=True, extensioned=True, fext="hog", When made (created and inited) by boxer.do then have "_boxer" and "_box" keys in self.iops = dict(_boxer=self.name, _box=m.box.name, **iops) + + + since filed it should reopen without truncating so does not overwrite + existing log file of same name. Always init by writing header so + even if change logs the header demarcation allows recovery of logged + data that includes different sets of logs. Need to test this + header should include UUID and date time stamp so even when process + quits and restarts have known uniqueness of data. This includes matching + up cycle sets of logs where the cycle count changes so there may be + stale cycle logs from old bigger set but same name. + Because each log record of data always starts with tyme as float, + which starts with numeric characters any consumer of log file can find restart demarcations of restart + header interior (not at start) because header starts with non numeric + characters This way rotating logs + """ + if iops and "_boxer" in iops: # '_boxer' is boxer.name when made by boxer + pass # set base to boxer name + super(Hog, self).__init__(filed=filed, extensioned=extensioned, fext=fext, + reuse=reuse, + iops=iops, nabe=nabe, **kwa) + self.begun = None self.began = None self.first = None From 6c0a5d585e80a9c4c27b35efb66d47e45215e86a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Aug 2025 12:59:03 -0600 Subject: [PATCH 11/36] more work on Hog hold log --- src/hio/base/hier/hogging.py | 23 +++++++++++++++-------- tests/base/hier/test_hogging.py | 8 +++++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 951cbcd..5f5582c 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -9,6 +9,7 @@ from __future__ import annotations # so type hints of classes get resolved later from contextlib import contextmanager +import inspect from ..doing import Doer from ..filing import Filer @@ -112,6 +113,8 @@ class Hog(ActBase, Filer): logs (dict): labeled keys of holds to log. Item labels are log tags logs item value is key in hold that provides value to log logs item key is tag in log header + marks (dict): tyme or value tuples marks of holds logged with updated or + changed. Label is hold key. Hidden: _name (str): unique name of instance for .name property @@ -126,9 +129,10 @@ class Hog(ActBase, Filer): extensioned=True, ) - def __init__(self, iops=None, nabe=Nabes.afdo, filed=True, extensioned=True, fext="hog", - reuse=True, rule=None, span=0.0, flush=0.0, - count=0, cycle=0.0, low=0, high=0, logs=None, **kwa): + def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, + extensioned=True, fext="hog", reuse=True, rule=None, + span=0.0, flush=0.0, count=0, cycle=0.0, low=0, high=0, + logs=None, **kwa): """Initialize instance. Inherited Parameters: @@ -206,15 +210,17 @@ def __init__(self, iops=None, nabe=Nabes.afdo, filed=True, extensioned=True, fex characters This way rotating logs """ - if iops and "_boxer" in iops: # '_boxer' is boxer.name when made by boxer - pass # set base to boxer name + if not base: + if iops and "_boxer" in iops: # '_boxer' in iops when made by boxer + base = iops['_boxer'] # set base to boxer.name - super(Hog, self).__init__(filed=filed, + super(Hog, self).__init__(iops=iops, + nabe=nabe, + base=base, + filed=filed, extensioned=extensioned, fext=fext, reuse=reuse, - iops=iops, - nabe=nabe, **kwa) @@ -241,6 +247,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, filed=True, extensioned=True, fex logs[tag] = val # value is key of hold to log self.logs = logs + self.marks = {} diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 0c5b135..e763138 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -7,7 +7,7 @@ import os import platform import tempfile - +import inspect import pytest @@ -20,6 +20,11 @@ def test_hog_basic(): """Test Hog class""" Hog._clearall() # clear Hog.Instances for debugging + # at some point could create utility function here that walks the .mro + # hierachy using inspect to collect all the keyword args to reserve them + # and double check Hog.Reserved is correct + # python how to get the keywords for given method signature including superclasses + hog = Hog(temp=True) # test defaults assert hog.temp assert hog.name == "Hog0" @@ -97,6 +102,7 @@ def test_open_hog(): """Done Test""" + def test_hog_doer(): """Test HogDoer""" Hog._clearall() # clear Hog.Instances for debugging From bf50ebf1ed8197bcfdf49b202b42310b21cca89b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Aug 2025 15:27:11 -0600 Subject: [PATCH 12/36] more comments --- src/hio/base/hier/hogging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 5f5582c..239b4e9 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -199,7 +199,8 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, since filed it should reopen without truncating so does not overwrite existing log file of same name. Always init by writing header so even if change logs the header demarcation allows recovery of logged - data that includes different sets of logs. Need to test this + data that includes different sets of logs. When reopening need to + seek to end of file or else it will overwrite. Need to test reopen logic header should include UUID and date time stamp so even when process quits and restarts have known uniqueness of data. This includes matching up cycle sets of logs where the cycle count changes so there may be From 57581253f3c46ba9990473fdc6be70cf49e34009 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Aug 2025 11:58:53 -0600 Subject: [PATCH 13/36] added boxer tyme bag in hold so record current boxer time and also provide known bag as TymeDom for tyme reference which the Hog Act needs to log --- src/hio/base/hier/boxing.py | 51 +++++++++++++----- src/hio/base/hier/hogging.py | 39 +++++++------- src/hio/help/timing.py | 57 ++++++++++++++++++++ tests/base/hier/test_boxing.py | 93 ++++++++++++++++++++++++++++++++- tests/base/hier/test_hogging.py | 4 +- tests/conftest.py | 17 ++++++ tests/help/test_timing.py | 63 ++++++++++++++++++++++ 7 files changed, 288 insertions(+), 36 deletions(-) diff --git a/src/hio/base/hier/boxing.py b/src/hio/base/hier/boxing.py index dbe5714..5c4a718 100644 --- a/src/hio/base/hier/boxing.py +++ b/src/hio/base/hier/boxing.py @@ -83,8 +83,15 @@ class Box(Tymee): Box instance holds references (links) to its over box and its under boxes. Box instance holds the acts to be executed in their nabe. - Inherited Attributes, Properties - see Tymee + Inherited Properties + .tyme (float | None): relative cycle time of associated Tymist which is + provided by calling .tymth function wrapper closure which is obtained + from Tymist.tymen(). + None means not assigned yet. + .tymth (Callable | None): function wrapper closure returned by + Tymist.tymen() method. When .tymth is called it returns associated + Tymist.tyme. Provides injected dependency on Tymist cycle tyme base. + None means not assigned yet. Attributes: hold (Hold): data shared by boxwork @@ -131,6 +138,11 @@ class Box(Tymee): def __init__(self, *, name='box', hold=None, over=None, **kwa): """Initialize instance. + Inherited Parameters: + tymth (closure): injected function wrapper closure returned by + .tymen() of Tymist instance. + Calling tymth() returns associated Tymist .tyme. + Parameters: name (str): unique identifier of box hold (None|Hold): data shared by boxwork @@ -349,6 +361,11 @@ class Boxer(Tymee): def __init__(self, *, name='boxer', hold=None, fun=None, durable=False, **kwa): """Initialize instance. + Inherited Parameters: + tymth (closure): injected function wrapper closure returned by + .tymen() of Tymist instance. + Calling tymth() returns associated Tymist .tyme. + Parameters: name (str): unique identifier of box hold (None|Hold): data shared by boxwork @@ -455,34 +472,42 @@ def run(self, tock=0.0): self.box = None # no active box anymore return False # signal failure due to end in enter before first pass - akeys = ("", "boxer", self.name, "active") - if akeys not in self.hold: - self.hold[akeys] = Bag() - self.hold[akeys].value = self.box.name # assign active box name + activeKey = self.hold.tokey(("", "boxer", self.name, "active")) + if activeKey not in self.hold: # setup active box bag + self.hold[activeKey] = Bag() + self.hold[activeKey].value = self.box.name # assign active box name + + tockKey = self.hold.tokey(("", "boxer", self.name, "tock")) + if tockKey not in self.hold: # setup tock bag + self.hold[tockKey] = Bag() + self.hold[tockKey].value = tock # assign tock + + tymeKey = self.hold.tokey(("", "boxer", self.name, "tyme")) + if tymeKey not in self.hold: # setup tyme bag + self.hold[tymeKey] = Bag() - tkey = self.hold.tokey(("", "boxer", self.name, "tock")) - if tkey not in self.hold: - self.hold[tkey] = Bag() - self.hold[tkey].value = tock # assign tock # finished of enter next() delegation 'yield from' delegation tyme = yield(tock) # pause end of next, resume start of send + self.hold[tymeKey].value = tyme # assign tyme + # begin first pass after send() self.rendo(rendos) # rendo nabe, action remarks and renacts self.endo(endos) # endo nabe, action enmarks and enacts self.redo() # redo nabe all boxes in pile top down while True: # run forever - tock = self.hold[tkey].value # get tock in case it changed + tock = self.hold[tockKey].value # get tock in case it changed tyme = yield(tock) # resume on send after tyme tick + self.hold[tymeKey].value = tyme # assign tyme rendos = [] # make empty for new pass, reset on transit endos = [] # make empty for new pass, reset on transit if self.endial(): # previous pass actioned desire to end self.end() # exdos all active boxes in self.box.pile self.box = None # no active box - self.hold[akeys].value = None # assign active box name to None + self.hold[activeKey].value = None # assign active box name to None return True # signal successful end after last pass transit = False @@ -497,7 +522,7 @@ def run(self, tock=0.0): self.exdo(exdos) # exdo bottom up self.rexdo(rexdos) # rexdo bottom up (boxes retained) self.box = dest # set new active box - self.hold[akeys].value = self.box.name # active box name + self.hold[activeKey].value = self.box.name # active box name transit = True break diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 239b4e9..64abd1f 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -86,10 +86,8 @@ class Hog(ActBase, Filer): Attributes: - begun (bool): True means logging has begun with header - False means logging has not yet begun needs header - began (str|None): realtime datetime stamp of when logging began, may be non 0.0 - None means not yet running + started (bool): True means logging has begun with header + False means logging has not yet begun needs header first (float|None): tyme when began logging, None means not yet running last (float|None): tyme when last logged, None means not yet running realtime equiv of last = began + (last - first) @@ -110,11 +108,11 @@ class Hog(ActBase, Filer): cycleHigh (int): maximum size in bytes allowed for each cycled log 0 means no maximum. One of cycleSpan or cycleHigh must be non zero - logs (dict): labeled keys of holds to log. Item labels are log tags - logs item value is key in hold that provides value to log - logs item key is tag in log header - marks (dict): tyme or value tuples marks of holds logged with updated or - changed. Label is hold key. + hits (dict): hold items to log. Item label is log header tag + Item value is hold key that provides value to log + marks (dict): tyme or value tuples marks of hold items logged with + updated or changed rule. Label is hold key. + Hidden: _name (str): unique name of instance for .name property @@ -132,7 +130,7 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, fext="hog", reuse=True, rule=None, span=0.0, flush=0.0, count=0, cycle=0.0, low=0, high=0, - logs=None, **kwa): + hits=None, **kwa): """Initialize instance. Inherited Parameters: @@ -186,9 +184,8 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, 0 means no minimum high (int): maximum size in bytes allowed for each cycled log 0 means no maximum - logs (None|dict): labeled keys of holds to log. Item labels are tags - logs item value is key in hold that provides value to log - logs item key is tag in log header + hits (None|dict): hold items to log. Item label is log header tag + Item value is hold key that provides value to log None means use unreserved items in **kwa wrt .ReservedTags @@ -225,8 +222,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, **kwa) - self.begun = None - self.began = None + self.started = False self.first = None self.last = None self.rule = rule if rule is not None else "every" @@ -241,17 +237,16 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.cycleLow = low self.cycleHigh = high - if logs is None: - logs = {} + if hits is None: + hits = {} for tag, val in kwa.items(): if tag not in self.ReservedTags: - logs[tag] = val # value is key of hold to log + hits[tag] = val # value is key of hold to log - self.logs = logs + self.hits = hits self.marks = {} - def act(self, **iops): """Act called by ActBase. @@ -262,6 +257,10 @@ def act(self, **iops): When made by boxer.do then have "_boxer" and "_box" keys in self.iops iops = dict(_boxer=self.name, _box=m.box.name, **iops) """ + if not self.started: + # seed to end + # began datetime + pass # create header return iops diff --git a/src/hio/help/timing.py b/src/hio/help/timing.py index cbe945f..32d8137 100644 --- a/src/hio/help/timing.py +++ b/src/hio/help/timing.py @@ -3,6 +3,7 @@ """ import time +import datetime from .. import hioing @@ -204,3 +205,59 @@ def latest(self): return self._last +# datetime utilities +def nowUTC(): + """ + Returns timezone aware datetime of current UTC time + Convenience function that allows monkeypatching in tests to mock time + """ + return (datetime.datetime.now(datetime.timezone.utc)) + + +def nowIso8601(): + """ + Returns time now in RFC-3339 profile of ISO 8601 format. + use now(timezone.utc) + + YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] + .strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '2020-08-22T17:50:09.988921+00:00' + Assumes TZ aware + For nanosecond use instead attotime or datatime64 in pandas or numpy + """ + return (nowUTC().isoformat(timespec='microseconds')) + + +def toIso8601(dt=None): + """ + Returns str datetime dt in RFC-3339 profile of ISO 8601 format. + Converts datetime object dt to ISO 8601 formt + If dt is missing use now(timezone.utc) + + YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] + .strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '2020-08-22T17:50:09.988921+00:00' + Assumes TZ aware + For nanosecond use instead attotime or datatime64 in pandas or numpy + """ + if dt is None: + dt = nowUTC() # make it aware + + return (dt.isoformat(timespec='microseconds')) # force include microseconds + + +def fromIso8601(dts): + """ + Returns datetime object from RFC-3339 profile of ISO 8601 format str or bytes. + Converts dts from ISO 8601 format to datetime object + + YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] + .strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '2020-08-22T17:50:09.988921+00:00' + Assumes TZ aware + For nanosecond use instead attotime or datatime64 in pandas or numpy + """ + if hasattr(dts, "decode"): + dts = dts.decode("utf-8") + return (datetime.datetime.fromisoformat(dts)) + diff --git a/tests/base/hier/test_boxing.py b/tests/base/hier/test_boxing.py index 5969903..f7d8916 100644 --- a/tests/base/hier/test_boxing.py +++ b/tests/base/hier/test_boxing.py @@ -565,6 +565,16 @@ def fun(H, bx, go, do, on, at, be, *pa): assert hold.count.value == 2 assert boxer.endial() + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'count', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] + """Done Test""" @@ -650,6 +660,16 @@ def fun(H, bx, go, do, on, at, be, *pa): assert boxer.box is None assert boxer.endial() + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'count', + '_boxer_boxer_box_mid_update_count', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] """Done Test""" @@ -737,6 +757,16 @@ def fun(H, bx, go, do, on, at, be, *pa): assert boxer.box is None assert boxer.endial() + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'count', + '_boxer_boxer_box_mid_change_count', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] """Done Test""" def test_boxer_run_on_count(): @@ -808,6 +838,16 @@ def fun(H, bx, go, do, on, at, be, *pa): assert boxer.box is None assert boxer.endial() + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] + """Done Test""" @@ -917,6 +957,18 @@ def fun(H, bx, go, do, on, at, be, *pa): assert hold._boxer_boxer_tock.value == 1.0 assert hold._boxer_boxer_end.value == True + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'stuff', + 'crud', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] + """Done Test""" def test_boxer_run_lapse(): @@ -1048,6 +1100,19 @@ def fun(H, bx, go, do, on, at, be, *pa): assert hold._boxer_boxer_tock.value == 1.0 assert hold._boxer_boxer_end.value == True + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'stuff', + 'crud', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_box_bot0_lapse', + '_boxer_boxer_box_bot1_lapse', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] """Done Test""" @@ -1153,6 +1218,19 @@ def fun(H, bx, go, do, on, at, be, *pa): assert hold._boxer_boxer_tock.value == 1.0 assert hold._boxer_boxer_end.value == True + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'stuff', + 'crud', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_box_mid_relapse', + '_boxer_boxer_box_bot1_lapse', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] """Done Test""" @@ -1228,7 +1306,20 @@ def fun(H, bx, go, do, on, at, be, *pa): assert hold._boxer_boxer_box_bot1_lapse._tyme == 2.0 assert hold._boxer_boxer_box_bot1_lapse._now == 8.0 - + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'test', + 'buf', + 'puf', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_box_bot0_lapse', + '_boxer_boxer_box_bot1_lapse', + '_boxer_boxer_end', + '_boxer_boxer_active', + '_boxer_boxer_tock', + '_boxer_boxer_tyme' + ] """Done Test""" diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index e763138..e784ad5 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -52,7 +52,7 @@ def test_hog_basic(): assert hog.nabe == Nabes.afdo assert hog.hold assert not hog.hold.subery - assert hog.logs == {} + assert hog.hits == {} hog.close(clear=True) assert not hog.opened @@ -94,7 +94,7 @@ def test_open_hog(): assert hog.nabe == Nabes.afdo assert hog.hold assert not hog.hold.subery - assert hog.logs == {} + assert hog.hits == {} assert not hog.opened assert not hog.file diff --git a/tests/conftest.py b/tests/conftest.py index cae182f..1bba1be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,5 +6,22 @@ """ import os + import pytest +from hio.help import timing + + +@pytest.fixture() +def mockHelpingNowIso8601(monkeypatch): + """ + Replace nowIso8601 universally with fixed value for testing + """ + + def mockNowIso8601(): + """ + Use predetermined value for now (current time) + '2021-01-01T00:00:00.000000+00:00' + """ + return "2021-06-27T21:26:21.233257+00:00" + monkeypatch.setattr(timing, "nowIso8601", mockNowIso8601) diff --git a/tests/help/test_timing.py b/tests/help/test_timing.py index ae703a8..16d39a5 100644 --- a/tests/help/test_timing.py +++ b/tests/help/test_timing.py @@ -4,10 +4,14 @@ """ import time +import datetime + import pytest from hio.help.timing import TimerError, RetroTimerError from hio.help.timing import Timer, MonoTimer +from hio.help.timing import nowIso8601, fromIso8601, toIso8601 + def test_timer(): @@ -138,5 +142,64 @@ def test_monotimer(): """End Test """ + +def test_iso8601(): # mocked in pytest + """ + Test datetime ISO 8601 helpers + """ + # dts = datetime.datetime.now(datetime.timezone.utc).isoformat() + dts = '2020-08-22T20:34:41.687702+00:00' + dt = fromIso8601(dts) + assert dt.year == 2020 + assert dt.month == 8 + assert dt.day == 22 + + dtb = b'2020-08-22T20:34:41.687702+00:00' + dt = fromIso8601(dts) + assert dt.year == 2020 + assert dt.month == 8 + assert dt.day == 22 + + + dts1 = nowIso8601() + dt1 = fromIso8601(dts1) + + # Add a small delay to ensure timestamps are different + time.sleep(0.001) + + dts2 = nowIso8601() + dt2 = fromIso8601(dts2) + + assert dt2 > dt1 + + assert dts1 == toIso8601(dt1) + assert dts2 == toIso8601(dt2) + + time.sleep(0.001) + + dts3 = toIso8601() + dt3 = fromIso8601(dts3) + + assert dt3 > dt2 + + td = dt3 - dt2 # timedelta + assert td.microseconds > 0.0 + + dt4 = dt + datetime.timedelta(seconds=25.0) + dts4 = toIso8601(dt4) + assert dts4 == '2020-08-22T20:35:06.687702+00:00' + dt4 = fromIso8601(dts4) + assert (dt4 - dt).seconds == 25.0 + + # test for microseconds zero + dts = "2021-01-01T00:00:00.000000+00:00" + dt = fromIso8601(dts) + dts1 = toIso8601(dt) + assert dts1 == dts + + """ End Test """ + if __name__ == "__main__": test_timer() + test_monotimer() + test_iso8601() From 4fae1ed196a42858efb2f35f29c213dc4a04deea Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Aug 2025 15:09:09 -0600 Subject: [PATCH 14/36] started work on hog header --- src/hio/base/hier/hogging.py | 26 +++++++++++++++++--------- src/hio/help/__init__.py | 4 +++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 64abd1f..a0d54c8 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -8,6 +8,8 @@ """ from __future__ import annotations # so type hints of classes get resolved later +import os +import uuid from contextlib import contextmanager import inspect @@ -15,6 +17,7 @@ from ..filing import Filer from .hiering import Nabes from .acting import ActBase, register +from ...help import nowIso8601 @register(names=('log', 'Log')) @@ -128,9 +131,9 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, - extensioned=True, fext="hog", reuse=True, rule=None, - span=0.0, flush=0.0, count=0, cycle=0.0, low=0, high=0, - hits=None, **kwa): + extensioned=True, mode='a+', fext="hog", reuse=True, + rule=None, span=0.0, flush=0.0, count=0, cycle=0.0, + low=0, high=0, hits=None, **kwa): """Initialize instance. Inherited Parameters: @@ -194,10 +197,12 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, since filed it should reopen without truncating so does not overwrite - existing log file of same name. Always init by writing header so + existing log file of same name. use mode 'a+'. Otherwise when reopening + would need to seek to end of file as default 'r+' goes to beginning. + Need to test reopen logic + Always init by writing header so even if change logs the header demarcation allows recovery of logged - data that includes different sets of logs. When reopening need to - seek to end of file or else it will overwrite. Need to test reopen logic + data that includes different sets of logs. header should include UUID and date time stamp so even when process quits and restarts have known uniqueness of data. This includes matching up cycle sets of logs where the cycle count changes so there may be @@ -258,9 +263,12 @@ def act(self, **iops): iops = dict(_boxer=self.name, _box=m.box.name, **iops) """ if not self.started: - # seed to end - # began datetime - pass # create header + # using mode "a+" don't need to seek to end + # self.file.seek(0, os.SEEK_END) # seek to end of file + hid = 'hog_' + uuid.uuid1().hex # hog id uuid for this run (not iteration) + stamp = nowIso8601() # current real datetime as ISO8601 string + + # create header return iops diff --git a/src/hio/help/__init__.py b/src/hio/help/__init__.py index 01dcf1b..fb9a074 100644 --- a/src/hio/help/__init__.py +++ b/src/hio/help/__init__.py @@ -18,7 +18,9 @@ from .helping import NonStringIterable, NonStringSequence from .decking import Deck from .hicting import Hict, Mict -from .timing import Timer, MonoTimer, TimerError, RetroTimerError +from .timing import (Timer, MonoTimer, TimerError, RetroTimerError, + nowIso8601, toIso8601, fromIso8601) + from .naming import Namer from .doming import (MapDom, IceMapDom, modify, modize, RawDom, IceRawDom, registerify, RegDom, IceRegDom, namify, TymeDom, IceTymeDom) From a127d1c33363f85176976f7c837cba610512c451 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Aug 2025 17:08:36 -0600 Subject: [PATCH 15/36] fixed the pytest mocking problem --- src/hio/base/hier/hogging.py | 32 ++++++++++------ src/hio/daemon.py | 2 +- tests/base/hier/test_hogging.py | 65 ++++++++++++++++++++++++++++++++- tests/conftest.py | 21 +++++++++-- tests/help/test_timing.py | 2 +- 5 files changed, 103 insertions(+), 19 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index a0d54c8..5c7f588 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -12,13 +12,16 @@ import uuid from contextlib import contextmanager import inspect +from collections import namedtuple from ..doing import Doer from ..filing import Filer from .hiering import Nabes from .acting import ActBase, register -from ...help import nowIso8601 +from ...help import timing +Ruleage = namedtuple("Rules", 'once every span update change') +Rules = Ruleage(once='once', every='every', span='span', update='update', change='change') @register(names=('log', 'Log')) class Hog(ActBase, Filer): @@ -94,10 +97,11 @@ class Hog(ActBase, Filer): first (float|None): tyme when began logging, None means not yet running last (float|None): tyme when last logged, None means not yet running realtime equiv of last = began + (last - first) - rule (str): condition for log to fire one of - ('once', 'every', 'spanned', 'updated', 'changed') + rule (str): condition for log to fire one of Rules + (once, every, span, update, change) span (float): tyme span for periodic logging header (str): header for log file(s) + hid (str): universally unique hog ID for given run of hog flushSpan (float): tyme span between flushes (flush) flushLast (float|None): tyme last flushed, None means not yet running cycleCount (int): number of cycled logs, 0 means do not cycle (count) @@ -132,8 +136,8 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, mode='a+', fext="hog", reuse=True, - rule=None, span=0.0, flush=0.0, count=0, cycle=0.0, - low=0, high=0, hits=None, **kwa): + hid=None, rule=Rules.every, span=0.0, flush=0.0, + count=0, cycle=0.0, low=0, high=0, hits=None, **kwa): """Initialize instance. Inherited Parameters: @@ -175,8 +179,10 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, fext (str): File extension when filed or extensioned Parameters: - rule (str|None): condition for log to fire, default every - (once, every, spanned, updated, changed) + hid (str|None): universally unique hog ID for given run of hog + None means create one using uuid lib + rule (str|None): condition for log to fire, one of Rules default every + (once, every, span, update, change) span (float): periodic time span when rule is spanned. 0.0 means every tyme flush (float): flush tyme span, tyme between flushes, 0.0 means @@ -230,8 +236,10 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.started = False self.first = None self.last = None - self.rule = rule if rule is not None else "every" + self.rule = rule self.span = span + self.hid = hid + self.stamp = '' # need to init self.header = '' # need to init self.flushSpan = flush self.flushLast = None @@ -265,10 +273,12 @@ def act(self, **iops): if not self.started: # using mode "a+" don't need to seek to end # self.file.seek(0, os.SEEK_END) # seek to end of file - hid = 'hog_' + uuid.uuid1().hex # hog id uuid for this run (not iteration) - stamp = nowIso8601() # current real datetime as ISO8601 string + if self.hid is None: + self.hid = 'hog_' + uuid.uuid1().hex # hog id uuid for this run (not iteration) + self.stamp = timing.nowIso8601() # current real datetime as ISO8601 string - # create header + self.header = (f"hid\t{self.hid}\tstamp\t{self.stamp}\trule\t{self.rule}" + f"\tcount\t{self.cycleCount}\n") return iops diff --git a/src/hio/daemon.py b/src/hio/daemon.py index 36878a0..763ff6c 100644 --- a/src/hio/daemon.py +++ b/src/hio/daemon.py @@ -1,7 +1,7 @@ """ hio daemon -Background Server Daemon for keri +Background Server Daemon for hio """ diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index e784ad5..ec8ebe7 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -2,7 +2,6 @@ """tests.base.hier.test_acting module """ -from __future__ import annotations # so type hints of classes get resolved later import os import platform @@ -11,9 +10,10 @@ import pytest +import hio from hio.base import Doist from hio.base.hier import Nabes, Hog, openHog, HogDoer - +from hio.help.timing import nowIso8601 def test_hog_basic(): @@ -53,6 +53,8 @@ def test_hog_basic(): assert hog.hold assert not hog.hold.subery assert hog.hits == {} + assert hog.header.startswith('hid') + # 'hid\thog_99ced11e86b111f0ac77f2acaf456f91\tdatetime\t2025-08-31T21:29:39.628696+00:00\trule\tevery\tcount\t0\n' hog.close(clear=True) assert not hog.opened @@ -95,6 +97,7 @@ def test_open_hog(): assert hog.hold assert not hog.hold.subery assert hog.hits == {} + assert hog.header.startswith('hid') assert not hog.opened assert not hog.file @@ -175,7 +178,65 @@ def test_hog_doer(): """End Test""" +def test_hog_log(mockHelpingNowIso8601): + """Test Hog clas with logging""" + Hog._clearall() # clear Hog.Instances for debugging + + # at some point could create utility function here that walks the .mro + # hierachy using inspect to collect all the keyword args to reserve them + # and double check Hog.Reserved is correct + # python how to get the keywords for given method signature including superclasses + + dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked + assert dts == '2021-06-27T21:26:21.233257+00:00' + hid = 'hog_3db602c486bd11f0bdf3f2acaf456f91' # for test + + hog = Hog(temp=True, hid=hid) # test defaults + assert hog.temp + assert hog.name == "Hog0" + assert hog.opened + assert hog.filed + assert hog.extensioned + assert hog.fext == 'hog' + assert hog.file + assert not hog.file.closed + assert os.path.exists(hog.path) + + tempDirPath = (os.path.join(os.path.sep, "tmp") + if platform.system() == "Darwin" + else tempfile.gettempdir()) + tempDirPath = os.path.normpath(tempDirPath) + path = os.path.normpath(hog.path) # '/tmp/hio_u36wdtp5_test/hio/Hog1.hog' + assert path.startswith(os.path.join(tempDirPath, "hio_")) + assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + + assert Hog.Registry[Hog.__name__] is Hog + assert Hog.Registry['log'] is Hog + assert Hog.Registry['Log'] is Hog + + assert hog() == {} # default returns iops + assert hog.nabe == Nabes.afdo + assert hog.hold + assert not hog.hold.subery + assert hog.hits == {} + assert hog.header.startswith('hid') + assert hog.header == ('hid\thog_3db602c486bd11f0bdf3f2acaf456f91\tstamp\t' + '2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n') + assert hog.hid == hid + assert hog.stamp == dts + + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + + """Done Test""" + + if __name__ == "__main__": test_hog_basic() test_open_hog() test_hog_doer() + + diff --git a/tests/conftest.py b/tests/conftest.py index 1bba1be..7c9e900 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,23 @@ https://docs.pytest.org/en/latest/pythonpath.html """ -import os - import pytest -from hio.help import timing +import hio + +@pytest.fixture() +def mockHelpingNowUTC(monkeypatch): + """ + Replace nowUTC universally with fixed value for testing + """ + + def mockNowUTC(): + """ + Use predetermined value for now (current time) + '2021-01-01T00:00:00.000000+00:00' + """ + return hio.help.timing.fromIso8601("2021-01-01T00:00:00.000000+00:00") + + monkeypatch.setattr(hio.help.timing, "nowUTC", mockNowUTC) @pytest.fixture() @@ -24,4 +37,4 @@ def mockNowIso8601(): """ return "2021-06-27T21:26:21.233257+00:00" - monkeypatch.setattr(timing, "nowIso8601", mockNowIso8601) + monkeypatch.setattr(hio.help.timing, "nowIso8601", mockNowIso8601) diff --git a/tests/help/test_timing.py b/tests/help/test_timing.py index 16d39a5..ff0b6f2 100644 --- a/tests/help/test_timing.py +++ b/tests/help/test_timing.py @@ -143,7 +143,7 @@ def test_monotimer(): """End Test """ -def test_iso8601(): # mocked in pytest +def test_iso8601(): """ Test datetime ISO 8601 helpers """ From f3602577ab6bb613e7d1ae1ae3930703597d66f0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Aug 2025 17:14:16 -0600 Subject: [PATCH 16/36] change hid to rid for run id not hog id since changes per run --- src/hio/base/hier/hogging.py | 8 ++++---- tests/base/hier/test_hogging.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 5c7f588..812269b 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -101,7 +101,7 @@ class Hog(ActBase, Filer): (once, every, span, update, change) span (float): tyme span for periodic logging header (str): header for log file(s) - hid (str): universally unique hog ID for given run of hog + rid (str): universally unique run ID for given run of hog flushSpan (float): tyme span between flushes (flush) flushLast (float|None): tyme last flushed, None means not yet running cycleCount (int): number of cycled logs, 0 means do not cycle (count) @@ -136,7 +136,7 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, mode='a+', fext="hog", reuse=True, - hid=None, rule=Rules.every, span=0.0, flush=0.0, + rid=None, rule=Rules.every, span=0.0, flush=0.0, count=0, cycle=0.0, low=0, high=0, hits=None, **kwa): """Initialize instance. @@ -179,7 +179,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, fext (str): File extension when filed or extensioned Parameters: - hid (str|None): universally unique hog ID for given run of hog + rid (str|None): universally unique run ID for given run of hog None means create one using uuid lib rule (str|None): condition for log to fire, one of Rules default every (once, every, span, update, change) @@ -238,7 +238,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.last = None self.rule = rule self.span = span - self.hid = hid + self.hid = rid self.stamp = '' # need to init self.header = '' # need to init self.flushSpan = flush diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index ec8ebe7..8f1d741 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -189,9 +189,9 @@ def test_hog_log(mockHelpingNowIso8601): dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked assert dts == '2021-06-27T21:26:21.233257+00:00' - hid = 'hog_3db602c486bd11f0bdf3f2acaf456f91' # for test + rid = 'hog_3db602c486bd11f0bdf3f2acaf456f91' # for test - hog = Hog(temp=True, hid=hid) # test defaults + hog = Hog(temp=True, rid=rid) # test defaults assert hog.temp assert hog.name == "Hog0" assert hog.opened @@ -222,7 +222,7 @@ def test_hog_log(mockHelpingNowIso8601): assert hog.header.startswith('hid') assert hog.header == ('hid\thog_3db602c486bd11f0bdf3f2acaf456f91\tstamp\t' '2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n') - assert hog.hid == hid + assert hog.hid == rid assert hog.stamp == dts hog.close(clear=True) From 5f1d3659140f1f17d39b525dc92c797c72de42a3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Aug 2025 17:27:14 -0600 Subject: [PATCH 17/36] fixed Hog.rid generation --- src/hio/base/hier/hogging.py | 8 ++++---- tests/base/hier/test_hogging.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 812269b..aeb89f5 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -238,7 +238,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.last = None self.rule = rule self.span = span - self.hid = rid + self.rid = rid self.stamp = '' # need to init self.header = '' # need to init self.flushSpan = flush @@ -273,11 +273,11 @@ def act(self, **iops): if not self.started: # using mode "a+" don't need to seek to end # self.file.seek(0, os.SEEK_END) # seek to end of file - if self.hid is None: - self.hid = 'hog_' + uuid.uuid1().hex # hog id uuid for this run (not iteration) + if self.rid is None: + self.rid = self.name + "_" + uuid.uuid1().hex # hog id uuid for this run (not iteration) self.stamp = timing.nowIso8601() # current real datetime as ISO8601 string - self.header = (f"hid\t{self.hid}\tstamp\t{self.stamp}\trule\t{self.rule}" + self.header = (f"rid\t{self.rid}\tstamp\t{self.stamp}\trule\t{self.rule}" f"\tcount\t{self.cycleCount}\n") return iops diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 8f1d741..c7b9a82 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -53,8 +53,7 @@ def test_hog_basic(): assert hog.hold assert not hog.hold.subery assert hog.hits == {} - assert hog.header.startswith('hid') - # 'hid\thog_99ced11e86b111f0ac77f2acaf456f91\tdatetime\t2025-08-31T21:29:39.628696+00:00\trule\tevery\tcount\t0\n' + assert hog.header.startswith('rid') hog.close(clear=True) assert not hog.opened @@ -97,7 +96,7 @@ def test_open_hog(): assert hog.hold assert not hog.hold.subery assert hog.hits == {} - assert hog.header.startswith('hid') + assert hog.header.startswith('rid') assert not hog.opened assert not hog.file @@ -189,7 +188,7 @@ def test_hog_log(mockHelpingNowIso8601): dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked assert dts == '2021-06-27T21:26:21.233257+00:00' - rid = 'hog_3db602c486bd11f0bdf3f2acaf456f91' # for test + rid = 'Hog0_3db602c486bd11f0bdf3f2acaf456f91' # for test hog = Hog(temp=True, rid=rid) # test defaults assert hog.temp @@ -219,10 +218,10 @@ def test_hog_log(mockHelpingNowIso8601): assert hog.hold assert not hog.hold.subery assert hog.hits == {} - assert hog.header.startswith('hid') - assert hog.header == ('hid\thog_3db602c486bd11f0bdf3f2acaf456f91\tstamp\t' + assert hog.header.startswith('rid') + assert hog.header == ('rid\tHog0_3db602c486bd11f0bdf3f2acaf456f91\tstamp\t' '2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n') - assert hog.hid == rid + assert hog.rid == rid assert hog.stamp == dts hog.close(clear=True) From 394cefea717c8b71ff9ed72692a842e2bf9ac512 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 11:05:03 -0600 Subject: [PATCH 18/36] reordered boxer state addition of hold fields more work on Hog header --- src/hio/base/hier/boxing.py | 23 +++++++------ src/hio/base/hier/hogging.py | 48 +++++++++++++++++++++++++-- tests/base/hier/test_boxing.py | 18 +++++------ tests/base/hier/test_hogging.py | 57 ++++++++++++++++++++++++++------- 4 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/hio/base/hier/boxing.py b/src/hio/base/hier/boxing.py index 5c4a718..f08b197 100644 --- a/src/hio/base/hier/boxing.py +++ b/src/hio/base/hier/boxing.py @@ -472,6 +472,12 @@ def run(self, tock=0.0): self.box = None # no active box anymore return False # signal failure due to end in enter before first pass + # setup boxer state in hold tyme, active box, and tock + tymeKey = self.hold.tokey(("", "boxer", self.name, "tyme")) + if tymeKey not in self.hold: # setup tyme bag + self.hold[tymeKey] = Bag() + self.hold[tymeKey].value = self.tyme + activeKey = self.hold.tokey(("", "boxer", self.name, "active")) if activeKey not in self.hold: # setup active box bag self.hold[activeKey] = Bag() @@ -482,15 +488,10 @@ def run(self, tock=0.0): self.hold[tockKey] = Bag() self.hold[tockKey].value = tock # assign tock - tymeKey = self.hold.tokey(("", "boxer", self.name, "tyme")) - if tymeKey not in self.hold: # setup tyme bag - self.hold[tymeKey] = Bag() - - # finished of enter next() delegation 'yield from' delegation + # tyme injected from yield should be self.tyme when recur by Doist or DoDoer tyme = yield(tock) # pause end of next, resume start of send - - self.hold[tymeKey].value = tyme # assign tyme + self.hold[tymeKey].value = tyme # assign tyme for Hog same as self.tyme # begin first pass after send() self.rendo(rendos) # rendo nabe, action remarks and renacts @@ -498,9 +499,10 @@ def run(self, tock=0.0): self.redo() # redo nabe all boxes in pile top down while True: # run forever - tock = self.hold[tockKey].value # get tock in case it changed + tock = self.hold[tockKey].value # get tock in case Act changed it + # tyme injected from yield should be self.tyme when recur by Doist or DoDoer tyme = yield(tock) # resume on send after tyme tick - self.hold[tymeKey].value = tyme # assign tyme + self.hold[tymeKey].value = tyme # assign tyme for Hog same as self.tyme rendos = [] # make empty for new pass, reset on transit endos = [] # make empty for new pass, reset on transit @@ -1431,7 +1433,8 @@ def recur(self, tock=None): False completed unsuccessfully Note that "tyme" is not a parameter when recur is a generator method - since doist tyme is injected by the explicit yield below. + since doist tyme is injected into the explicit yield below by the + Doist or DoDoer send(tyme) in their recur method for generator Doers. The recur method itself returns a generator so parameters to this method are to setup the generator not to be used at recur time. diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index aeb89f5..a405ff8 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -18,7 +18,7 @@ from ..filing import Filer from .hiering import Nabes from .acting import ActBase, register -from ...help import timing +from ...help import timing # import timing to pytest mock of nowIso8601 works Ruleage = namedtuple("Rules", 'once every span update change') Rules = Ruleage(once='once', every='every', span='span', update='update', change='change') @@ -119,6 +119,12 @@ class Hog(ActBase, Filer): Item value is hold key that provides value to log marks (dict): tyme or value tuples marks of hold items logged with updated or changed rule. Label is hold key. + activeKey (str|None): hold key to active box name of boxer given by iops + None otherwise + tockKey (str|None): hold key to tock value of boxer given by iops + None otherwise + tockKey (str|None): hold key to tyme value of boxer given by iops + None otherwise Hidden: @@ -250,6 +256,15 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.cycleLow = low self.cycleHigh = high + self.activeKey = None + self.tockKey = None + self.tymeKey = None + if "_boxer" in self.iops: # assign keys for boxer box state + boxerName = self.iops["_boxer"] + self.activeKey = self.hold.tokey(("", "boxer", boxerName, "active")) + self.tockKey = self.hold.tokey(("", "boxer", boxerName, "tock")) + self.tymeKey = self.hold.tokey(("", "boxer", boxerName, "tyme")) + if hits is None: hits = {} for tag, val in kwa.items(): @@ -277,11 +292,40 @@ def act(self, **iops): self.rid = self.name + "_" + uuid.uuid1().hex # hog id uuid for this run (not iteration) self.stamp = timing.nowIso8601() # current real datetime as ISO8601 string - self.header = (f"rid\t{self.rid}\tstamp\t{self.stamp}\trule\t{self.rule}" + metaLine = (f"rid\t{self.rid}\tbase\t{self.base}\tname\t{self.name}" + f"\tstamp\t{self.stamp}\trule\t{self.rule}" f"\tcount\t{self.cycleCount}\n") + if self.tymeKey and self.tymeKey in self.hold: # need tyme for logging + hits = dict(tyme=self.tymeKey) # logging tyme + for tag, key in self.hits.items(): # copy valid .hits + if key in self.hold: # invalid hold key + hits[tag] = key # make copy in order + + else: # need tyme in order to log anything with respect to tyme + hits = {} # nothing to log + + self.hits = hits + + if len(self.hits) == 1: # only tyme so default to boxer state + if self.activeKey and self.activeKey in self.hold: + self.hits["active"] = self.activeKey + if self.tockKey and self.tockKey in self.hold: + self.hits["tock"] = self.tockKey + + # need to expand tags for hits with vector bags in hold + tagLine = '\t'.join(f"{tag}.value" for tag in self.hits.keys()) + "\n" + + self.header = metaLine + tagLine + + return iops + def logOne(self): + """Log one record of .hits values from .hold""" + for key in self.hits.values(): # hit values are hold keys + pass + @contextmanager diff --git a/tests/base/hier/test_boxing.py b/tests/base/hier/test_boxing.py index f7d8916..646c6d2 100644 --- a/tests/base/hier/test_boxing.py +++ b/tests/base/hier/test_boxing.py @@ -570,9 +570,9 @@ def fun(H, bx, go, do, on, at, be, *pa): '_hold_subery', 'count', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', - '_boxer_boxer_tock', - '_boxer_boxer_tyme' + '_boxer_boxer_tock' ] """Done Test""" @@ -666,9 +666,9 @@ def fun(H, bx, go, do, on, at, be, *pa): 'count', '_boxer_boxer_box_mid_update_count', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -763,9 +763,9 @@ def fun(H, bx, go, do, on, at, be, *pa): 'count', '_boxer_boxer_box_mid_change_count', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -843,9 +843,9 @@ def fun(H, bx, go, do, on, at, be, *pa): '_hold_subery', '_boxer_boxer_box_mid_count', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -964,9 +964,9 @@ def fun(H, bx, go, do, on, at, be, *pa): 'crud', '_boxer_boxer_box_mid_count', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -1109,9 +1109,9 @@ def fun(H, bx, go, do, on, at, be, *pa): '_boxer_boxer_box_bot0_lapse', '_boxer_boxer_box_bot1_lapse', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -1227,9 +1227,9 @@ def fun(H, bx, go, do, on, at, be, *pa): '_boxer_boxer_box_mid_relapse', '_boxer_boxer_box_bot1_lapse', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" @@ -1316,9 +1316,9 @@ def fun(H, bx, go, do, on, at, be, *pa): '_boxer_boxer_box_bot0_lapse', '_boxer_boxer_box_bot1_lapse', '_boxer_boxer_end', + '_boxer_boxer_tyme', '_boxer_boxer_active', '_boxer_boxer_tock', - '_boxer_boxer_tyme' ] """Done Test""" diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index c7b9a82..561b2b7 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -11,9 +11,9 @@ import pytest import hio -from hio.base import Doist -from hio.base.hier import Nabes, Hog, openHog, HogDoer -from hio.help.timing import nowIso8601 +from hio.base import Doist, Tymist +from hio.base.hier import Nabes, Hog, openHog, HogDoer, Hold, Bag +from hio.help.timing import nowIso8601 # timing so pytest mock nowIso8601 works def test_hog_basic(): @@ -186,13 +186,41 @@ def test_hog_log(mockHelpingNowIso8601): # and double check Hog.Reserved is correct # python how to get the keywords for given method signature including superclasses + tymist = Tymist() + dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked assert dts == '2021-06-27T21:26:21.233257+00:00' rid = 'Hog0_3db602c486bd11f0bdf3f2acaf456f91' # for test - hog = Hog(temp=True, rid=rid) # test defaults + boxerName = "BoxerTest" + boxName = "BoxTop" + iops = dict(_boxer=boxerName, _box=boxName) + + hold = Hold() + + activeKey = hold.tokey(("", "boxer", boxerName, "active")) + hold[activeKey] = Bag() + hold[activeKey].value = boxName + + tockKey = hold.tokey(("", "boxer", boxerName, "tock")) + hold[tockKey] = Bag() + hold[tockKey].value = tymist.tock + + tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) + hold[tymeKey] = Bag() + hold[tymeKey].value = tymist.tyme + + name = "pig" + + # default when no hits logs box state + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid) # test defaults assert hog.temp - assert hog.name == "Hog0" + assert hog.name == name + assert hog.iops == iops + assert hog.nabe == Nabes.afdo + assert hog.hold == hold + + assert hog.base == boxerName assert hog.opened assert hog.filed assert hog.extensioned @@ -207,22 +235,29 @@ def test_hog_log(mockHelpingNowIso8601): tempDirPath = os.path.normpath(tempDirPath) path = os.path.normpath(hog.path) # '/tmp/hio_u36wdtp5_test/hio/Hog1.hog' assert path.startswith(os.path.join(tempDirPath, "hio_")) - assert hog.path.endswith(os.path.join('_test', 'hio', 'Hog0.hog')) + assert hog.path.endswith(os.path.join('_test', 'hio', boxerName, f'{name}.hog')) assert Hog.Registry[Hog.__name__] is Hog assert Hog.Registry['log'] is Hog assert Hog.Registry['Log'] is Hog - assert hog() == {} # default returns iops + assert hog() == iops # default returns iops assert hog.nabe == Nabes.afdo assert hog.hold assert not hog.hold.subery - assert hog.hits == {} - assert hog.header.startswith('rid') - assert hog.header == ('rid\tHog0_3db602c486bd11f0bdf3f2acaf456f91\tstamp\t' - '2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n') + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'active': '_boxer_BoxerTest_active', + 'tock': '_boxer_BoxerTest_tock' + } + assert hog.rid == rid assert hog.stamp == dts + assert hog.header.startswith('rid') + assert hog.header == ('rid\tHog0_3db602c486bd11f0bdf3f2acaf456f91\tbase\tBoxerTest\tname\tpig\t' + 'stamp\t2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n' + 'tyme.value\tactive.value\ttock.value\n') hog.close(clear=True) assert not hog.opened From d6b5eddcd6f492efc7893144c78034c944a1e880 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 11:14:55 -0600 Subject: [PATCH 19/36] more work on header --- src/hio/base/hier/hogging.py | 9 +++++++-- tests/base/hier/test_hogging.py | 7 +++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index a405ff8..dd52dc1 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -313,10 +313,15 @@ def act(self, **iops): if self.tockKey and self.tockKey in self.hold: self.hits["tock"] = self.tockKey + tagKeyLine = '\t'.join(f"{tag}.key" for tag in self.hits.keys()) + "\n" + keyLine = '\t'.join(key for key in self.hits.values()) + "\n" + # need to expand tags for hits with vector bags in hold - tagLine = '\t'.join(f"{tag}.value" for tag in self.hits.keys()) + "\n" + tagValLine = '\t'.join(f"{tag}.value" for tag in self.hits.keys()) + "\n" + + - self.header = metaLine + tagLine + self.header = metaLine + tagKeyLine + keyLine + tagValLine return iops diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 561b2b7..1297b9e 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -256,8 +256,11 @@ def test_hog_log(mockHelpingNowIso8601): assert hog.stamp == dts assert hog.header.startswith('rid') assert hog.header == ('rid\tHog0_3db602c486bd11f0bdf3f2acaf456f91\tbase\tBoxerTest\tname\tpig\t' - 'stamp\t2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n' - 'tyme.value\tactive.value\ttock.value\n') + 'stamp\t2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n' + 'tyme.key\tactive.key\ttock.key\n' + '_boxer_BoxerTest_tyme\t_boxer_BoxerTest_active\t_boxer_BoxerTest_tock\n' + 'tyme.value\tactive.value\ttock.value\n') + hog.close(clear=True) assert not hog.opened From 56b61821e0e6e49cfe3de4dad10336a082a49e56 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 11:20:46 -0600 Subject: [PATCH 20/36] more test cases --- tests/base/hier/test_hogging.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 1297b9e..b13ec1b 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -55,6 +55,8 @@ def test_hog_basic(): assert hog.hits == {} assert hog.header.startswith('rid') + assert list(hog.hold.keys()) == ['_hold_subery'] + hog.close(clear=True) assert not hog.opened assert not hog.file @@ -98,6 +100,8 @@ def test_open_hog(): assert hog.hits == {} assert hog.header.startswith('rid') + assert list(hog.hold.keys()) == ['_hold_subery'] + assert not hog.opened assert not hog.file assert not os.path.exists(hog.path) @@ -198,6 +202,10 @@ def test_hog_log(mockHelpingNowIso8601): hold = Hold() + tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) + hold[tymeKey] = Bag() + hold[tymeKey].value = tymist.tyme + activeKey = hold.tokey(("", "boxer", boxerName, "active")) hold[activeKey] = Bag() hold[activeKey].value = boxName @@ -206,10 +214,6 @@ def test_hog_log(mockHelpingNowIso8601): hold[tockKey] = Bag() hold[tockKey].value = tymist.tock - tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) - hold[tymeKey] = Bag() - hold[tymeKey].value = tymist.tyme - name = "pig" # default when no hits logs box state @@ -262,6 +266,14 @@ def test_hog_log(mockHelpingNowIso8601): 'tyme.value\tactive.value\ttock.value\n') + assert list(hog.hold.keys()) == \ + [ + '_hold_subery', + '_boxer_BoxerTest_tyme', + '_boxer_BoxerTest_active', + '_boxer_BoxerTest_tock', + ] + hog.close(clear=True) assert not hog.opened assert not hog.file From b6e1d638e50aeb057bad27ec2720780db3584255 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 15:24:28 -0600 Subject: [PATCH 21/36] changed rid to be base64 not hex --- src/hio/base/hier/hogging.py | 58 +++++++++++++++++++++++++-------- tests/base/hier/test_bagging.py | 14 ++++++++ tests/base/hier/test_hogging.py | 39 ++++++++++++++++++---- 3 files changed, 91 insertions(+), 20 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index dd52dc1..03f1bca 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -13,6 +13,8 @@ from contextlib import contextmanager import inspect from collections import namedtuple +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 from ..doing import Doer from ..filing import Filer @@ -100,6 +102,8 @@ class Hog(ActBase, Filer): rule (str): condition for log to fire one of Rules (once, every, span, update, change) span (float): tyme span for periodic logging + onced (bool): True means logged at least one (first) record + False means not yet logged one (first) record header (str): header for log file(s) rid (str): universally unique run ID for given run of hog flushSpan (float): tyme span between flushes (flush) @@ -211,6 +215,8 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, since filed it should reopen without truncating so does not overwrite existing log file of same name. use mode 'a+'. Otherwise when reopening would need to seek to end of file as default 'r+' goes to beginning. + using mode "a+" don't need to seek to end + self.file.seek(0, os.SEEK_END) # seek to end of file Need to test reopen logic Always init by writing header so even if change logs the header demarcation allows recovery of logged @@ -244,6 +250,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.last = None self.rule = rule self.span = span + self.onced = False self.rid = rid self.stamp = '' # need to init self.header = '' # need to init @@ -286,15 +293,18 @@ def act(self, **iops): iops = dict(_boxer=self.name, _box=m.box.name, **iops) """ if not self.started: - # using mode "a+" don't need to seek to end - # self.file.seek(0, os.SEEK_END) # seek to end of file + if self.rid is None: - self.rid = self.name + "_" + uuid.uuid1().hex # hog id uuid for this run (not iteration) + # hog id uuid for this run (not iteration) + # create B64 version of uuid with stripped trailing pad bytes + uid = encodeB64(bytes.fromhex(uuid.uuid1().hex))[:-2].decode() + self.rid = self.name + "_" + uid self.stamp = timing.nowIso8601() # current real datetime as ISO8601 string - metaLine = (f"rid\t{self.rid}\tbase\t{self.base}\tname\t{self.name}" - f"\tstamp\t{self.stamp}\trule\t{self.rule}" - f"\tcount\t{self.cycleCount}\n") + metaLine = (f"rid\tbase\tname\tstamp\trule\tcount\n") + + metaValLine = (f"{self.rid}\t{self.base}\t{self.name}" + f"\t{self.stamp}\t{self.rule}\t{self.cycleCount}\n") if self.tymeKey and self.tymeKey in self.hold: # need tyme for logging hits = dict(tyme=self.tymeKey) # logging tyme @@ -317,19 +327,39 @@ def act(self, **iops): keyLine = '\t'.join(key for key in self.hits.values()) + "\n" # need to expand tags for hits with vector bags in hold - tagValLine = '\t'.join(f"{tag}.value" for tag in self.hits.keys()) + "\n" - + tagValLine = ('\t'.join(f"{tag}.{fld}" for tag, key in self.hits.items() + for fld in self.hold[key]._names) + "\n") + + self.header = metaLine + metaValLine + tagKeyLine + keyLine + tagValLine + self.file.write(self.header) + self.first = self.hold[self.hits["tyme"]].value if self.hits else None + self.started = True + + if not self.onced: + self.file.write(self.record()) + self.onced = True + else: + pass + return iops - self.header = metaLine + tagKeyLine + keyLine + tagValLine + def record(self): + """Generate on record line .hits values from .hold + Returns: + record (str): one newline delimited line of tab delimited values + from .hits. Each hit time value is key into hold + vector holds each get entry in record per field - return iops - def logOne(self): - """Log one record of .hits values from .hold""" - for key in self.hits.values(): # hit values are hold keys - pass + """ + # hit values are hold keys + if self.hits: + return ('\t'.join(f"{self.hold[key][fld]}" + for key in self.hits.values() + for fld in self.hold[key]._names) + "\n") + else: + return "" diff --git a/tests/base/hier/test_bagging.py b/tests/base/hier/test_bagging.py index dac0373..bedb4e4 100644 --- a/tests/base/hier/test_bagging.py +++ b/tests/base/hier/test_bagging.py @@ -162,6 +162,20 @@ def test_bag(): t = b._astuple() assert t == (7, ) + # Adding fields? + with pytest.raises(TypeError): + bag = Bag(value=5, test=6) + + bag = Bag(value=5) + assert bag._names == ('value', ) + flds = fields(bag) + assert len(flds) == 1 + bag.test = 6 # added attribute on the fly + assert "test" not in bag._names + flds = fields(bag) + assert len(flds) == 1 + + """Done Test""" diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index b13ec1b..a0374c2 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -54,6 +54,7 @@ def test_hog_basic(): assert not hog.hold.subery assert hog.hits == {} assert hog.header.startswith('rid') + assert hog.rid.startswith(hog.name) # 'Hog0_KQzSlod5EfC1TvKsr0VvkQ' assert list(hog.hold.keys()) == ['_hold_subery'] @@ -194,12 +195,15 @@ def test_hog_log(mockHelpingNowIso8601): dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked assert dts == '2021-06-27T21:26:21.233257+00:00' - rid = 'Hog0_3db602c486bd11f0bdf3f2acaf456f91' # for test + boxerName = "BoxerTest" boxName = "BoxTop" iops = dict(_boxer=boxerName, _box=boxName) + uid = 'KQzSlod5EfC1TvKsr0VvkQ' # for test + rid = f"{boxerName}_{uid}" + hold = Hold() tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) @@ -259,12 +263,35 @@ def test_hog_log(mockHelpingNowIso8601): assert hog.rid == rid assert hog.stamp == dts assert hog.header.startswith('rid') - assert hog.header == ('rid\tHog0_3db602c486bd11f0bdf3f2acaf456f91\tbase\tBoxerTest\tname\tpig\t' - 'stamp\t2021-06-27T21:26:21.233257+00:00\trule\tevery\tcount\t0\n' - 'tyme.key\tactive.key\ttock.key\n' - '_boxer_BoxerTest_tyme\t_boxer_BoxerTest_active\t_boxer_BoxerTest_tock\n' - 'tyme.value\tactive.value\ttock.value\n') + assert hog.header == ('rid\tbase\tname\tstamp\trule\tcount\n' + 'BoxerTest_KQzSlod5EfC1TvKsr0VvkQ\tBoxerTest\tpig\t2021-06-27T21:26:21.233257+00:00\tevery\t0\n' + 'tyme.key\tactive.key\ttock.key\n' + '_boxer_BoxerTest_tyme\t_boxer_BoxerTest_active\t_boxer_BoxerTest_tock\n' + 'tyme.value\tactive.value\ttock.value\n') + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + assert lines == \ + [ + 'rid\tbase\tname\tstamp\trule\tcount\n', + 'BoxerTest_KQzSlod5EfC1TvKsr0VvkQ\tBoxerTest\tpig\t' + '2021-06-27T21:26:21.233257+00:00\tevery\t0\n', + 'tyme.key\tactive.key\ttock.key\n', + '_boxer_BoxerTest_tyme\t_boxer_BoxerTest_active\t_boxer_BoxerTest_tock\n', + 'tyme.value\tactive.value\ttock.value\n', + '0.0\tBoxTop\t0.03125\n' + ] + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','pig','2021-06-27T21:26:21.233257+00:00','every','0'), + ('tyme.key', 'active.key', 'tock.key'), + ('_boxer_BoxerTest_tyme', '_boxer_BoxerTest_active', '_boxer_BoxerTest_tock'), + ('tyme.value', 'active.value', 'tock.value'), + ('0.0', 'BoxTop', '0.03125') + ] assert list(hog.hold.keys()) == \ [ From a8737d710d2d3d26165d5638b883a2294cf8ede5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 16:51:36 -0600 Subject: [PATCH 22/36] added logic for other log rules need to test --- src/hio/base/hier/__init__.py | 2 +- src/hio/base/hier/boxing.py | 7 +-- src/hio/base/hier/hogging.py | 42 ++++++++++++- tests/base/hier/test_hogging.py | 107 +++++++++++++++++++++++++++++++- 4 files changed, 148 insertions(+), 10 deletions(-) diff --git a/src/hio/base/hier/__init__.py b/src/hio/base/hier/__init__.py index 60e0599..182e0aa 100644 --- a/src/hio/base/hier/__init__.py +++ b/src/hio/base/hier/__init__.py @@ -15,5 +15,5 @@ from .holding import Hold from .durqing import Durq from .dusqing import Dusq -from .hogging import Hog, openHog, HogDoer +from .hogging import Rules, Hog, openHog, HogDoer diff --git a/src/hio/base/hier/boxing.py b/src/hio/base/hier/boxing.py index f08b197..21a887d 100644 --- a/src/hio/base/hier/boxing.py +++ b/src/hio/base/hier/boxing.py @@ -408,14 +408,13 @@ def name(self, name): self._name = name - def wind(self, tymth=None): + def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Override in subclasses to update any dependencies on a change in tymist.tymth base Parameters: - tymth (Callable|None): closure of injected tyme from tymist.tymen() - None if not yet injected + tymth (Callable): closure of injected tyme from tymist.tymen() """ super().wind(tymth=tymth) for dom in self.hold.values(): @@ -1395,7 +1394,7 @@ def wind(self, tymth): Updates winds .tymer .tymth """ super(BoxerDoer, self).wind(tymth) - self.boxer.wind(tymth) + self.boxer.rewind(tymth) def enter(self, *, temp=None): diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 03f1bca..0dc4b17 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -332,14 +332,52 @@ def act(self, **iops): self.header = metaLine + metaValLine + tagKeyLine + keyLine + tagValLine self.file.write(self.header) - self.first = self.hold[self.hits["tyme"]].value if self.hits else None self.started = True + tyme = self.hold[self.hits["tyme"]].value if self.hits else None + if not self.onced: self.file.write(self.record()) + if self.hits: + for tag, key in self.hits.items(): + if self.rule == Rules.update: + self.marks[key].value = self.hold[key]._tyme + elif self.rule == Rules.change: + self.marks[key].value = self.hold[key]._astuple() + + self.first = tyme + self.last = tyme self.onced = True else: - pass + match self.rule: + case Rules.every: + self.file.write(self.record()) + self.last = tyme + case Rules.span: + if tyme is not None and tyme - self.last >= self.span: + self.file.write(self.record()) + self.last = tyme + + case Rules.update: + if tyme is not None: + for key, mark in self.marks: # marked tyme + holdTyme = self.hold[key]._tyme + if holdTyme > mark: # updated since marked + self.file.write(self.record()) + self.last = tyme + self.marks[key].value = holdTyme + + case Rules.change: + if tyme is not None: + for key, mark in self.marks: # marked value tuple + holdValue = self.hold[key]._astuple() + if holdValue != mark: # changed since marked + self.file.write(self.record()) + self.last = tyme + self.marks[key].value = holdValue + + case _: + pass return iops diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index a0374c2..0cc83f5 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -12,7 +12,8 @@ import hio from hio.base import Doist, Tymist -from hio.base.hier import Nabes, Hog, openHog, HogDoer, Hold, Bag +from hio.base.hier import Nabes, Rules, Hog, openHog, HogDoer, Hold, Bag +from hio.help import TymeDom from hio.help.timing import nowIso8601 # timing so pytest mock nowIso8601 works @@ -218,9 +219,14 @@ def test_hog_log(mockHelpingNowIso8601): hold[tockKey] = Bag() hold[tockKey].value = tymist.tock + tymth = tymist.tymen() + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + name = "pig" - # default when no hits logs box state + # Test rule "every" default, when no hits logs box state as default hits hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid) # test defaults assert hog.temp assert hog.name == name @@ -249,6 +255,11 @@ def test_hog_log(mockHelpingNowIso8601): assert Hog.Registry['log'] is Hog assert Hog.Registry['Log'] is Hog + assert hog.rule == Rules.every + assert not hog.started + assert not hog.onced + + # run hog once assert hog() == iops # default returns iops assert hog.nabe == Nabes.afdo assert hog.hold @@ -259,7 +270,10 @@ def test_hog_log(mockHelpingNowIso8601): 'active': '_boxer_BoxerTest_active', 'tock': '_boxer_BoxerTest_tock' } - + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 assert hog.rid == rid assert hog.stamp == dts assert hog.header.startswith('rid') @@ -301,12 +315,99 @@ def test_hog_log(mockHelpingNowIso8601): '_boxer_BoxerTest_tock', ] + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert hog() == iops # default returns iops + assert hog.last != hog.first + assert hog.last == tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest','pig','2021-06-27T21:26:21.233257+00:00','every','0'), + ('tyme.key', 'active.key', 'tock.key'), + ('_boxer_BoxerTest_tyme', '_boxer_BoxerTest_active', '_boxer_BoxerTest_tock'), + ('tyme.value', 'active.value', 'tock.value'), + ('0.0', 'BoxTop', '0.03125'), + ('0.03125', 'BoxTop', '0.03125') + ] + + hog.close(clear=True) assert not hog.opened assert not hog.file assert not os.path.exists(hog.path) + # Test rule "once" + name = "dog" + tymist.tyme = 0.0 + tymth = tymist.tymen() + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + hold[tymeKey].value = tymist.tyme + hold[activeKey].value = boxName + hold[tockKey].value = tymist.tock + + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, rule=Rules.once) + assert hog.rule == Rules.once + assert not hog.started + assert not hog.onced + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'active': '_boxer_BoxerTest_active', + 'tock': '_boxer_BoxerTest_tock' + } + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.rid == rid + assert hog.stamp == dts + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest','dog','2021-06-27T21:26:21.233257+00:00','once','0'), + ('tyme.key', 'active.key', 'tock.key'), + ('_boxer_BoxerTest_tyme', '_boxer_BoxerTest_active', '_boxer_BoxerTest_tock'), + ('tyme.value', 'active.value', 'tock.value'), + ('0.0', 'BoxTop', '0.03125') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert hog() == iops # default returns iops + assert hog.last == hog.first # since once does not log again + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest','dog','2021-06-27T21:26:21.233257+00:00','once','0'), + ('tyme.key', 'active.key', 'tock.key'), + ('_boxer_BoxerTest_tyme', '_boxer_BoxerTest_active', '_boxer_BoxerTest_tock'), + ('tyme.value', 'active.value', 'tock.value'), + ('0.0', 'BoxTop', '0.03125'), + ] + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) """Done Test""" From f7559e8e5b97bfb85e501f8d8e8f8b92cb87bc04 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 19:15:49 -0600 Subject: [PATCH 23/36] test Hog with vector bag --- tests/base/hier/test_hogging.py | 118 +++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 0cc83f5..7a733be 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -7,13 +7,15 @@ import platform import tempfile import inspect +from dataclasses import dataclass +from typing import Any, Type import pytest import hio from hio.base import Doist, Tymist from hio.base.hier import Nabes, Rules, Hog, openHog, HogDoer, Hold, Bag -from hio.help import TymeDom +from hio.help import TymeDom, namify, registerify from hio.help.timing import nowIso8601 # timing so pytest mock nowIso8601 works @@ -404,6 +406,120 @@ def test_hog_log(mockHelpingNowIso8601): ('0.0', 'BoxTop', '0.03125'), ] + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + # Test vector hold bag with rule "every" + name = "cat" + tymist.tyme = 0.0 + tymth = tymist.tymen() + + + @namify + @registerify + @dataclass + class LocationBag(TymeDom): + """Vector Bag dataclass + + Field Attributes: + latN (Any): latitude North fractional minutes + lonE (Any): longitude East fractional minutes + """ + latN: Any = None + lonE: Any = None + + def __hash__(self): + """Define hash so can work with ordered_set + __hash__ is not inheritable in dataclasses so must be explicitly defined + in every subclass + """ + return hash((self.__class__.__name__,) + self._astuple()) # almost same as __eq__ + + + homeKey = hold.tokey(("home", )) + hold[homeKey] = LocationBag(latN=45.0, lonE=-90.0) + + awayKey = hold.tokey(("away", )) + hold[awayKey] = LocationBag(latN=40.0, lonE=10.0) + + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + hold[tymeKey].value = tymist.tyme + hold[activeKey].value = boxName + hold[tockKey].value = tymist.tock + hold[homeKey].latN = 40.7607 + hold[homeKey].lonE = -111.8939 + hold[awayKey].latN = 39.3999 + hold[awayKey].lonE = 8.2245 + + # locationKey as hit + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, + home=homeKey, away=awayKey) + assert hog.rule == Rules.every + assert not hog.started + assert not hog.onced + assert hog.hits == {'home': 'home', 'away': 'away'} + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'home', + 'away': 'away' + } + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.rid == rid + assert hog.stamp == dts + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', + 'BoxerTest', + 'cat', + '2021-06-27T21:26:21.233257+00:00', + 'every', + '0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '39.3999', '8.2245') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 41.5020 + hold[awayKey].lonE = 9.5123 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest', 'cat', '2021-06-27T21:26:21.233257+00:00', 'every', '0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '39.3999', '8.2245'), + ('0.03125', '40.7607', '-111.8939', '41.502', '9.5123') + ] + hog.close(clear=True) assert not hog.opened assert not hog.file From 47a38135b7c9eebadd9a9c63a5758d3259a1d08c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 19:20:36 -0600 Subject: [PATCH 24/36] fixed logic for update or change rule need to test --- src/hio/base/hier/hogging.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 0dc4b17..b037d94 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -248,7 +248,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.started = False self.first = None self.last = None - self.rule = rule + self.rule = rule # maybe should make property so readonly after init self.span = span self.onced = False self.rid = rid @@ -360,21 +360,29 @@ def act(self, **iops): case Rules.update: if tyme is not None: + updated = False for key, mark in self.marks: # marked tyme holdTyme = self.hold[key]._tyme if holdTyme > mark: # updated since marked - self.file.write(self.record()) - self.last = tyme - self.marks[key].value = holdTyme + updated = True + break + if updated: + self.file.write(self.record()) + self.last = tyme + self.marks[key].value = holdTyme case Rules.change: if tyme is not None: + changed = False for key, mark in self.marks: # marked value tuple holdValue = self.hold[key]._astuple() if holdValue != mark: # changed since marked - self.file.write(self.record()) - self.last = tyme - self.marks[key].value = holdValue + changed = True + break + if changed: + self.file.write(self.record()) + self.last = tyme + self.marks[key].value = holdValue case _: pass From c0469c3776c064aeb5db0154efc426bdce7317d0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Sep 2025 21:43:20 -0600 Subject: [PATCH 25/36] minor fix to mark logic in Hog --- src/hio/base/hier/hogging.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index b037d94..2b19e62 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -340,10 +340,11 @@ def act(self, **iops): self.file.write(self.record()) if self.hits: for tag, key in self.hits.items(): - if self.rule == Rules.update: - self.marks[key].value = self.hold[key]._tyme - elif self.rule == Rules.change: - self.marks[key].value = self.hold[key]._astuple() + if tag != "tyme": # do not mark tyme hold + if self.rule == Rules.update: + self.marks[key].value = self.hold[key]._tyme + elif self.rule == Rules.change: + self.marks[key].value = self.hold[key]._astuple() self.first = tyme self.last = tyme From 7bfadade235b5761b622fa63e054b3c871e45993 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 08:48:15 -0600 Subject: [PATCH 26/36] added tests for rule update --- src/hio/base/hier/hogging.py | 19 ++- tests/base/hier/test_hogging.py | 201 ++++++++++++++++++++++++++++---- 2 files changed, 188 insertions(+), 32 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 2b19e62..e0bed96 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -341,10 +341,10 @@ def act(self, **iops): if self.hits: for tag, key in self.hits.items(): if tag != "tyme": # do not mark tyme hold - if self.rule == Rules.update: - self.marks[key].value = self.hold[key]._tyme - elif self.rule == Rules.change: - self.marks[key].value = self.hold[key]._astuple() + if self.rule == Rules.update: # create mark + self.marks[key] = self.hold[key]._tyme + elif self.rule == Rules.change: # create mark + self.marks[key] = self.hold[key]._astuple() self.first = tyme self.last = tyme @@ -362,28 +362,27 @@ def act(self, **iops): case Rules.update: if tyme is not None: updated = False - for key, mark in self.marks: # marked tyme + for key, mark in self.marks.items(): # marked tyme holdTyme = self.hold[key]._tyme if holdTyme > mark: # updated since marked + self.marks[key] = holdTyme updated = True - break if updated: self.file.write(self.record()) self.last = tyme - self.marks[key].value = holdTyme case Rules.change: if tyme is not None: changed = False - for key, mark in self.marks: # marked value tuple + for key, mark in self.marks.items(): # marked value tuple holdValue = self.hold[key]._astuple() if holdValue != mark: # changed since marked + self.marks[key] = holdValue changed = True - break if changed: self.file.write(self.record()) self.last = tyme - self.marks[key].value = holdValue + case _: pass diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 7a733be..accba18 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -189,6 +189,28 @@ def test_hog_log(mockHelpingNowIso8601): """Test Hog clas with logging""" Hog._clearall() # clear Hog.Instances for debugging + @namify + @registerify + @dataclass + class LocationBag(TymeDom): + """Vector Bag dataclass + + Field Attributes: + latN (Any): latitude North fractional minutes + lonE (Any): longitude East fractional minutes + """ + latN: Any = None + lonE: Any = None + + def __hash__(self): + """Define hash so can work with ordered_set + __hash__ is not inheritable in dataclasses so must be explicitly defined + in every subclass + """ + return hash((self.__class__.__name__,) + self._astuple()) # almost same as __eq__ + + + # at some point could create utility function here that walks the .mro # hierachy using inspect to collect all the keyword args to reserve them # and double check Hog.Reserved is correct @@ -417,27 +439,6 @@ def test_hog_log(mockHelpingNowIso8601): tymth = tymist.tymen() - @namify - @registerify - @dataclass - class LocationBag(TymeDom): - """Vector Bag dataclass - - Field Attributes: - latN (Any): latitude North fractional minutes - lonE (Any): longitude East fractional minutes - """ - latN: Any = None - lonE: Any = None - - def __hash__(self): - """Define hash so can work with ordered_set - __hash__ is not inheritable in dataclasses so must be explicitly defined - in every subclass - """ - return hash((self.__class__.__name__,) + self._astuple()) # almost same as __eq__ - - homeKey = hold.tokey(("home", )) hold[homeKey] = LocationBag(latN=45.0, lonE=-90.0) @@ -456,7 +457,7 @@ def __hash__(self): hold[awayKey].latN = 39.3999 hold[awayKey].lonE = 8.2245 - # locationKey as hit + # vector locations as hits with default rule every hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, home=homeKey, away=awayKey) assert hog.rule == Rules.every @@ -520,6 +521,162 @@ def __hash__(self): ('0.03125', '40.7607', '-111.8939', '41.502', '9.5123') ] + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + # Test rule update + name = "fox" + tymist.tyme = 0.0 + tymth = tymist.tymen() + + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + # reset given rewound tyme + hold[tymeKey].value = tymist.tyme + hold[activeKey].value = boxName + hold[tockKey].value = tymist.tock + hold[homeKey].latN = 40.7607 + hold[homeKey].lonE = -111.8939 + hold[awayKey].latN = 40.0 + hold[awayKey].lonE = 7.0 + + # vector locations as hits with rule update + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, rule=Rules.update, + home=homeKey, away=awayKey) + assert hog.rule == Rules.update + assert not hog.started + assert not hog.onced + assert hog.hits == {'home': 'home', 'away': 'away'} + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'home', + 'away': 'away' + } + assert hog.marks == {'home': 0.0, 'away': 0.0} + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.rid == rid + assert hog.stamp == dts + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '40.0', '7.0') + ] + + # run again update but not change value + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 40.0 # update without changing value + hold[awayKey].lonE = 7.0 # update without changing value + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.03125', '40.7607', '-111.8939', '40.0', '7.0') + ] + + # run again update but change value + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[homeKey].latN = 45.4545 # update change value + hold[homeKey].lonE = -112.1212 # update change value + hold[awayKey].latN = 41.0505 # update change value + hold[awayKey].lonE = 8.0222 # update change value + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', + 'BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again no update + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.09375 + + assert hog() == iops # default returns iops + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', + 'BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again update not change + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.125 + hold[homeKey].latN = 45.4545 # update do not change value + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222'), + ('0.125', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + hog.close(clear=True) assert not hog.opened assert not hog.file From 1f2e98733ec0c9d5ed947bf9ce31270737e7ba37 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 09:18:38 -0600 Subject: [PATCH 27/36] tests for chage rule --- tests/base/hier/test_hogging.py | 205 ++++++++++++++++++++++++++++---- 1 file changed, 183 insertions(+), 22 deletions(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index accba18..3b05e43 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -439,10 +439,10 @@ def __hash__(self): tymth = tymist.tymen() - homeKey = hold.tokey(("home", )) + homeKey = hold.tokey(("location", "home", )) hold[homeKey] = LocationBag(latN=45.0, lonE=-90.0) - awayKey = hold.tokey(("away", )) + awayKey = hold.tokey(("location", "away", )) hold[awayKey] = LocationBag(latN=40.0, lonE=10.0) for dom in hold.values(): # wind hold @@ -463,16 +463,17 @@ def __hash__(self): assert hog.rule == Rules.every assert not hog.started assert not hog.onced - assert hog.hits == {'home': 'home', 'away': 'away'} + assert hog.hits =={'home': 'location_home', 'away': 'location_away'} # run hog once assert hog() == iops # default returns iops assert hog.hits == \ { 'tyme': '_boxer_BoxerTest_tyme', - 'home': 'home', - 'away': 'away' + 'home': 'location_home', + 'away': 'location_away' } + assert hog.started assert hog.onced assert hog.first == 0.0 @@ -486,14 +487,9 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', - 'BoxerTest', - 'cat', - '2021-06-27T21:26:21.233257+00:00', - 'every', - '0'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cat','2021-06-27T21:26:21.233257+00:00','every','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '39.3999', '8.2245') ] @@ -515,7 +511,7 @@ def __hash__(self): ('rid', 'base', 'name', 'stamp', 'rule', 'count'), ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest', 'cat', '2021-06-27T21:26:21.233257+00:00', 'every', '0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '39.3999', '8.2245'), ('0.03125', '40.7607', '-111.8939', '41.502', '9.5123') @@ -550,17 +546,17 @@ def __hash__(self): assert hog.rule == Rules.update assert not hog.started assert not hog.onced - assert hog.hits == {'home': 'home', 'away': 'away'} + assert hog.hits == {'home': 'location_home', 'away': 'location_away'} # run hog once assert hog() == iops # default returns iops assert hog.hits == \ { 'tyme': '_boxer_BoxerTest_tyme', - 'home': 'home', - 'away': 'away' + 'home': 'location_home', + 'away': 'location_away' } - assert hog.marks == {'home': 0.0, 'away': 0.0} + assert hog.marks == {'location_home': 0.0, 'location_away': 0.0} assert hog.started assert hog.onced assert hog.first == 0.0 @@ -576,7 +572,7 @@ def __hash__(self): ('rid', 'base', 'name', 'stamp', 'rule', 'count'), ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '40.0', '7.0') ] @@ -598,7 +594,7 @@ def __hash__(self): ('rid', 'base', 'name', 'stamp', 'rule', 'count'), ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), ('0.03125', '40.7607', '-111.8939', '40.0', '7.0') @@ -623,7 +619,7 @@ def __hash__(self): ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), @@ -646,7 +642,7 @@ def __hash__(self): ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ', 'BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), @@ -669,7 +665,7 @@ def __hash__(self): ('rid', 'base', 'name', 'stamp', 'rule', 'count'), ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','fox','2021-06-27T21:26:21.233257+00:00','update','0'), ('tyme.key', 'home.key', 'away.key'), - ('_boxer_BoxerTest_tyme', 'home', 'away'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), ('0.0', '40.7607', '-111.8939', '40.0', '7.0'), ('0.03125', '40.7607', '-111.8939', '40.0', '7.0'), @@ -677,6 +673,171 @@ def __hash__(self): ('0.125', '45.4545', '-112.1212', '41.0505', '8.0222') ] + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + # Test rule change + name = "owl" + tymist.tyme = 0.0 + tymth = tymist.tymen() + + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + # reset given rewound tyme + hold[tymeKey].value = tymist.tyme + hold[activeKey].value = boxName + hold[tockKey].value = tymist.tock + hold[homeKey].latN = 40.0 + hold[homeKey].lonE = -113.0 + hold[awayKey].latN = 42.0 + hold[awayKey].lonE = 8.0 + + # vector locations as hits with rule update + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, rule=Rules.change, + home=homeKey, away=awayKey) + assert hog.rule == Rules.change + assert not hog.started + assert not hog.onced + assert hog.hits == {'home': 'location_home', 'away': 'location_away'} + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'location_home', + 'away': 'location_away' + } + assert hog.marks == \ + { + 'location_home': (40.0, -113.0), + 'location_away': (42.0, 8.0) + } + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.rid == rid + assert hog.stamp == dts + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','owl','2021-06-27T21:26:21.233257+00:00','change','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0') + ] + + # run again , update but not change value + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 42.0 # update but not change value + hold[awayKey].lonE = 8.0 # update but not change value + + assert hog() == iops # default returns iops + assert hog.last == hog.first # since once does not log again + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','owl','2021-06-27T21:26:21.233257+00:00','change','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0') + ] + + # run again change value + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[homeKey].latN = 45.4545 # update change value + hold[homeKey].lonE = -112.1212 # update change value + hold[awayKey].latN = 41.0505 # update change value + hold[awayKey].lonE = 8.0222 # update change value + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + assert hog.marks == \ + { + 'location_home': (45.4545, -112.1212), + 'location_away': (41.0505, 8.0222) + } + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','owl','2021-06-27T21:26:21.233257+00:00','change','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again no update + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.09375 + + assert hog() == iops # default returns iops + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','owl','2021-06-27T21:26:21.233257+00:00','change','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again change only one + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.125 + hold[homeKey].latN = 46.0 # change value + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + assert hog.marks == \ + { + 'location_home': (46.0, -112.1212), + 'location_away': (41.0505, 8.0222) + } + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','owl','2021-06-27T21:26:21.233257+00:00','change','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222'), + ('0.125', '46.0', '-112.1212', '41.0505', '8.0222') + ] + hog.close(clear=True) assert not hog.opened assert not hog.file From 02c29d38da6dff0a1b196659abb2cef2a02d0889 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 09:37:25 -0600 Subject: [PATCH 28/36] Tests for span rule --- tests/base/hier/test_hogging.py | 156 +++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 3b05e43..c05772e 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -696,7 +696,7 @@ def __hash__(self): hold[awayKey].latN = 42.0 hold[awayKey].lonE = 8.0 - # vector locations as hits with rule update + # vector locations as hits with rule change hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, rule=Rules.change, home=homeKey, away=awayKey) assert hog.rule == Rules.change @@ -838,6 +838,160 @@ def __hash__(self): ('0.125', '46.0', '-112.1212', '41.0505', '8.0222') ] + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + # Test rule span + name = "cow" + tymist.tyme = 0.0 + tymth = tymist.tymen() + span = tymist.tock * 2 # every other run + + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + # reset given rewound tyme + hold[tymeKey].value = tymist.tyme + hold[activeKey].value = boxName + hold[tockKey].value = tymist.tock + hold[homeKey].latN = 40.0 + hold[homeKey].lonE = -113.0 + hold[awayKey].latN = 42.0 + hold[awayKey].lonE = 8.0 + + # vector locations as hits with rule span + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, + rule=Rules.span, span=span, home=homeKey, away=awayKey) + assert hog.rule == Rules.span + assert not hog.started + assert not hog.onced + assert hog.hits == {'home': 'location_home', 'away': 'location_away'} + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'location_home', + 'away': 'location_away' + } + assert hog.marks == {} + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.rid == rid + assert hog.stamp == dts + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cow','2021-06-27T21:26:21.233257+00:00','span','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0') + ] + + # run again , update and change value but span not enough + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[homeKey].latN = 45.4545 # update change value + hold[homeKey].lonE = -112.1212 # update change value + hold[awayKey].latN = 41.0505 # update change value + hold[awayKey].lonE = 8.0222 # update change value + + assert hog() == iops # default returns iops + assert hog.last == hog.first # since once does not log again + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cow','2021-06-27T21:26:21.233257+00:00','span','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0') + ] + + # run again do not update or change but should log due to span + tymist.tick() + hold[tymeKey].value = tymist.tyme + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + assert hog.marks == {} + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cow','2021-06-27T21:26:21.233257+00:00','span','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again change but span not enough + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.09375 + hold[homeKey].latN = 47.0 # update change value + hold[homeKey].lonE = -114.0 # update change value + + assert hog() == iops # default returns iops + assert hog.last != tymist.tyme + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cow','2021-06-27T21:26:21.233257+00:00','span','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222') + ] + + # run again change different one span enough + tymist.tick() + hold[tymeKey].value = tymist.tyme + assert tymist.tyme == 0.125 + hold[awayKey].latN = 39.0 # change value + + assert hog() == iops # default returns iops + assert hog.last == tymist.tyme + assert hog.marks =={} + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','cow','2021-06-27T21:26:21.233257+00:00','span','0'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '40.0', '-113.0', '42.0', '8.0'), + ('0.0625', '45.4545', '-112.1212', '41.0505', '8.0222'), + ('0.125', '47.0', '-114.0', '39.0', '8.0222') + ] + hog.close(clear=True) assert not hog.opened assert not hog.file From 696ad28d086c072451b9efdc7f2ff0b6c53d3ce2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 10:09:47 -0600 Subject: [PATCH 29/36] added support for flush --- src/hio/base/filing.py | 9 ++++++ src/hio/base/hier/hogging.py | 58 ++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index e1c9754..003b6dc 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -479,6 +479,15 @@ def exists(self, name="", base="", headDirPath=None, clean=False, return os.path.exists(path) + def flush(self): + """ + flush self.file if not closed + """ + if self.file and not self.file.closed: + self.file.flush() + os.fsync(self.file.fileno()) + + def close(self, clear=False): """Close .file if any and if clear rm directory or file at .path diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index e0bed96..aa93ac9 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -101,12 +101,13 @@ class Hog(ActBase, Filer): realtime equiv of last = began + (last - first) rule (str): condition for log to fire one of Rules (once, every, span, update, change) - span (float): tyme span for periodic logging + span (float): tyme span seconds for periodic logging 0.0 means everytyme onced (bool): True means logged at least one (first) record False means not yet logged one (first) record header (str): header for log file(s) rid (str): universally unique run ID for given run of hog - flushSpan (float): tyme span between flushes (flush) + flushSpan (float): tyme span seconds between flushes (flush) + 0.0 means everytyme flushLast (float|None): tyme last flushed, None means not yet running cycleCount (int): number of cycled logs, 0 means do not cycle (count) cyclePaths (list[str]): paths for cycled logs @@ -146,7 +147,7 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, mode='a+', fext="hog", reuse=True, - rid=None, rule=Rules.every, span=0.0, flush=0.0, + rid=None, rule=Rules.every, span=0.0, flush=60.0, count=0, cycle=0.0, low=0, high=0, hits=None, **kwa): """Initialize instance. @@ -193,10 +194,10 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, None means create one using uuid lib rule (str|None): condition for log to fire, one of Rules default every (once, every, span, update, change) - span (float): periodic time span when rule is spanned. 0.0 means - every tyme - flush (float): flush tyme span, tyme between flushes, 0.0 means - every tyme + span (float): periodic tyme span seconds when rule is spanned + 0.0 means every tyme + flush (float): flush tyme span seconds, tyme between flushes + 0.0 means every tyme count (int): number of cycled logs, 0 means do not cycle cycle (float): cycle tyme span, tyme between log cycles low (int): minimum size in bytes required for cycling log @@ -337,7 +338,10 @@ def act(self, **iops): tyme = self.hold[self.hits["tyme"]].value if self.hits else None if not self.onced: - self.file.write(self.record()) + #self.file.write(self.record()) + #self.flush() + # always flush on first write to ensure header synced on disk + self.log(self.record(), tyme, force=True) if self.hits: for tag, key in self.hits.items(): if tag != "tyme": # do not mark tyme hold @@ -348,16 +352,19 @@ def act(self, **iops): self.first = tyme self.last = tyme + self.flushLast = tyme self.onced = True else: match self.rule: case Rules.every: - self.file.write(self.record()) - self.last = tyme + #self.file.write(self.record()) + #self.last = tyme + self.log(self.record(), tyme) case Rules.span: if tyme is not None and tyme - self.last >= self.span: - self.file.write(self.record()) - self.last = tyme + #self.file.write(self.record()) + #self.last = tyme + self.log(self.record(), tyme) case Rules.update: if tyme is not None: @@ -368,8 +375,9 @@ def act(self, **iops): self.marks[key] = holdTyme updated = True if updated: - self.file.write(self.record()) - self.last = tyme + #self.file.write(self.record()) + #self.last = tyme + self.log(self.record(), tyme) case Rules.change: if tyme is not None: @@ -380,8 +388,9 @@ def act(self, **iops): self.marks[key] = holdValue changed = True if changed: - self.file.write(self.record()) - self.last = tyme + #self.file.write(self.record()) + #self.last = tyme + self.log(self.record(), tyme) case _: @@ -389,6 +398,23 @@ def act(self, **iops): return iops + + def log(self, record, tyme, force=False): + """Write one record to file and flush when indicated + + Parameters: + record (str): one line of tab delimited newline ended values + tyme (float): tyme of log record + force (bool): True means force flush even when not flushSpan elapsed + False means do not force flush only if flushSpan elapsed + """ + self.file.write(record) + self.last = tyme + if force or (tyme - self.flushLast) >= self.flushSpan: + self.flush() + self.flushLast = tyme + + def record(self): """Generate on record line .hits values from .hold From 323707c8ac2eebcc073aa19845643915d707cf76 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 16:42:00 -0600 Subject: [PATCH 30/36] Added CloseAct for closing instances at runtime in hier box used to close Hog --- src/hio/base/filing.py | 1 + src/hio/base/hier/__init__.py | 3 +- src/hio/base/hier/acting.py | 120 ++++++++++++++++++++++++++++++--- src/hio/base/hier/boxing.py | 2 +- src/hio/base/hier/hogging.py | 40 +++++------ tests/base/hier/test_acting.py | 72 +++++++++++++++++++- tests/base/hier/test_boxing.py | 91 +++++++++++++++++++++++++ 7 files changed, 296 insertions(+), 33 deletions(-) diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index 003b6dc..128c6e1 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -495,6 +495,7 @@ def close(self, clear=False): clear (bool): True means remove dir or file at .path """ if self.file: + self.flush() # since file.close does not guarantee file sync self.file.close() self.opened = False diff --git a/src/hio/base/hier/__init__.py b/src/hio/base/hier/__init__.py index 182e0aa..466f9c0 100644 --- a/src/hio/base/hier/__init__.py +++ b/src/hio/base/hier/__init__.py @@ -8,7 +8,8 @@ Act, Goact, EndAct, Beact, Mark, LapseMark, RelapseMark, Count, Discount, BagMark, UpdateMark, ReupdateMark, - ChangeMark, RechangeMark) + ChangeMark, RechangeMark, + CloseAct) from .needing import Need from .bagging import Bag, IceBag from .canning import CanDom, Can diff --git a/src/hio/base/hier/acting.py b/src/hio/base/hier/acting.py index 0aeed6d..df72cf0 100644 --- a/src/hio/base/hier/acting.py +++ b/src/hio/base/hier/acting.py @@ -828,8 +828,8 @@ def compile(self): @register() class Mark(ActBase): - """Mark (Mine Mark) is base classubclass of ActBase whose .act marks a - box for a special need condition. + """Mark is base class that is subclass of ActBase whose .act marks a box + for a special need condition. Inherited Class Attributes: Registry (dict): subclass registry whose items are (name, cls) where: @@ -897,17 +897,14 @@ def __init__(self, nabe=Nabes.enmark, **kwa): """ super(Mark, self).__init__(nabe=nabe, **kwa) - try: - boxer = self.iops['_boxer'] # get boxer name to ensure existence - except KeyError as ex: + if '_boxer' not in self.iops: # ensure existence for subclasses raise HierError(f"Missing iops '_boxer' for '{self.name}' instance " - f"of Act self.__class__.__name__") from ex + f"of Act self.__class__.__name__") - try: - box = self.iops['_box'] # get box name to ensure existence - except KeyError as ex: + if '_box' not in self.iops: # ensure existence for subclasses raise HierError(f"Missing iops '_box' for '{self.name}' instance " - f"of Act self.__class__.__name__") from ex + f"of Act self.__class__.__name__") + def act(self, **iops): """Act called by ActBase. @@ -1467,3 +1464,106 @@ def act(self, **iops): self.hold[keys].value = bag._astuple() # bag field value tuple as mark return self.hold[keys].value + + +@register(names=('close', 'Close')) +class CloseAct(ActBase): + """CloseAct is subclass of ActBase whose .act calls .close method of target + instance provided by iops item "it", if iops item "clear" provided then + passes that value as clear parameter to it.close + + Inherited Class Attributes: + Registry (dict): subclass registry whose items are (name, cls) where: + name is unique name for subclass + cls is reference to class object + Instances (dict): instance registry whose items are (name, instance) where: + name is unique instance name and instance is instance reference + Index (int): default naming index for subclass instances. Each subclass + overrides with a subclass specific Index value to track + subclass specific instance default names. + Names (tuple[str]): tuple of aliases (names) under which this subclas + appears in .Registry. Created by @register + + Overridden Class Attributes + Index (int): default naming index for subclass instances. Each subclass + overrides with a subclass specific Index value to track + subclass specific instance default names. + + + Inherited Properties: + name (str): unique name string of instance + iops (dict): input-output-parameters for .act + nabe (str): action nabe (context) for .act + + Inherited Attributes: + hold (Hold): data shared by boxwork + + Attributes: + it (Any): instance with Callable attribute .close + clear (bool|None): clear parameter for .close method + + Used iops: + it (Any): instance with .close method + clear (bool|None): when not None passes value to .close method + + Hidden + _name (str|None): unique name of instance + _iops (dict): input-output-parameters for .act + _nabe (str): action nabe (context) for .act + + """ + Index = 0 # naming index for default names of this subclasses instances + + def __init__(self, nabe=Nabes.exdo, **kwa): + """Initialization method for instance. + + Inherited Parameters: + name (str|None): unique name of this instance. When None then + generate name from .Index + iops (dict|None): input-output-parameters for .act. When None then + set to empty dict. + nabe (str): action nabe (context) for .act + mine (None|Mine): ephemeral bags in mine (in memory) shared by boxwork + dock (None|Dock): durable bags in dock (on disc) shared by boxwork + + Parameters: + + Used iops: + it (Any): instance with .close method + clear (bool|None): when not None passes value to .close method + + + """ + super(CloseAct, self).__init__(nabe=nabe, **kwa) + + try: + it = self.iops['it'] # get instance to close + except KeyError as ex: + raise HierError(f"Missing iops 'it' for '{self.name}' instance " + f"of Act self.__class__.__name__") from ex + if not (hasattr(it, 'close') and isinstance(it.close, Callable)): + raise HierError(f"No close method of iops {it=} for '{self.name}'" + f" instance of Act self.__class__.__name__") + + clear = self.iops.get("clear", None) # get clear parameter if any + if clear not in (None, True, False): + raise HierError(f"Invalid value of iops {clear=} for '{self.name}'" + f" instance of Act self.__class__.__name__") + + self.it = it + self.clear = clear + + + def act(self, **iops): + """Act called by ActBase. + + Parameters: + iops (dict): input/output parms, same as self.iops. Puts **iops in + local scope in case act compliles exec/eval str + + """ + parms = {} + if self.clear is not None: + parms["clear"] = self.clear + + self.it.close(**parms) diff --git a/src/hio/base/hier/boxing.py b/src/hio/base/hier/boxing.py index 21a887d..785091f 100644 --- a/src/hio/base/hier/boxing.py +++ b/src/hio/base/hier/boxing.py @@ -916,7 +916,7 @@ def on(self, cond: None|str=None, key: None|str=None, expr: None|str=None, *, mods: WorkDom|None=None, **iops)->Need: """Make a Need with support for special Need conditions and return it. Use inside go verb as need argument for special need condition - Use inside do verb as deed argument for preact or anact + Use inside do verb as deed argument for preact Returns: need (Need): newly created special need diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index aa93ac9..75d28d0 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -138,6 +138,25 @@ class Hog(ActBase, Filer): _nabe (str): action nabe (context) for .act for .nabe property + As ActBase subclass that is managed inside boxwork, the Hog is created and + inited by Boxer.make which is run in the enter context of the BoxerDoer. + So opening file during init is compatible as its in the enter context of the + its Doer even though its not in the endo context of its box. + It still needs to be closed. Unlike the hold subery the Boxer doesn't know + about any Hogs runing as acts inside its boxes so can't close in its BoxerDoer + exit context. So much be closed inside with a close do act + + Since filed it should reopen without truncating so does not overwrite + existing log file of same name so uses mode = 'a+'. + Need to test reopen logic + Always rewrites header on first log after restart which may have changed + log behavior so header demarcation enables recognition of log parameters. + Log header includes rid (unique run id) and datatime stamp so can always + uniquely match a given run data to header even when reusing same log file. + This is most important when rotating logs. + Because each individual log record after header always starts with tyme as + floating point decimal, processor can find header demarcations interior to + log file not merely at start. """ ReservedTags = dict(name=True, iops=True, nabe=True, hold=True, base=True, temp=True, headDirPath=True, perm=True, reopen=True, @@ -212,25 +231,6 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, When made (created and inited) by boxer.do then have "_boxer" and "_box" keys in self.iops = dict(_boxer=self.name, _box=m.box.name, **iops) - - since filed it should reopen without truncating so does not overwrite - existing log file of same name. use mode 'a+'. Otherwise when reopening - would need to seek to end of file as default 'r+' goes to beginning. - using mode "a+" don't need to seek to end - self.file.seek(0, os.SEEK_END) # seek to end of file - Need to test reopen logic - Always init by writing header so - even if change logs the header demarcation allows recovery of logged - data that includes different sets of logs. - header should include UUID and date time stamp so even when process - quits and restarts have known uniqueness of data. This includes matching - up cycle sets of logs where the cycle count changes so there may be - stale cycle logs from old bigger set but same name. - Because each log record of data always starts with tyme as float, - which starts with numeric characters any consumer of log file can find restart demarcations of restart - header interior (not at start) because header starts with non numeric - characters This way rotating logs - """ if not base: if iops and "_boxer" in iops: # '_boxer' in iops when made by boxer @@ -241,6 +241,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, base=base, filed=filed, extensioned=extensioned, + mode=mode, fext=fext, reuse=reuse, **kwa) @@ -257,6 +258,7 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.header = '' # need to init self.flushSpan = flush self.flushLast = None + self.cycleCount = count self.cyclePaths = [] # need to init self.cycleSpan = cycle diff --git a/tests/base/hier/test_acting.py b/tests/base/hier/test_acting.py index 97c2962..310162b 100644 --- a/tests/base/hier/test_acting.py +++ b/tests/base/hier/test_acting.py @@ -6,16 +6,19 @@ import pytest +import os from collections.abc import Callable from hio import Mixin, HierError from hio.base import Tymist -from hio.base.hier import Nabes, Need, Box, Boxer, Bag, Hold +from hio.base.hier import Nabes, Need, Box, Boxer, Bag, Hold, Hog from hio.base.hier import (ActBase, actify, Act, Goact, EndAct, Beact, Mark, LapseMark, RelapseMark, BagMark, UpdateMark, ReupdateMark, ChangeMark, RechangeMark, - Count, Discount) + Count, Discount, + CloseAct) +from hio.help import TymeDom def test_actbase(): @@ -1057,7 +1060,71 @@ def test_rechange_mark_basic(): """Done Test""" +def test_closeact_basic(): + """Test CloseAct class""" + CloseAct._clearall() # clear instances for debugging + + assert "CloseAct" in CloseAct.Registry + assert CloseAct.Registry["CloseAct"] == CloseAct + assert CloseAct.Names == ("close", "Close") + for name in CloseAct.Names: + assert name in CloseAct.Registry + assert CloseAct.Registry[name] == CloseAct + + with pytest.raises(HierError): + cact = CloseAct() # requires iops with me + + tymist = Tymist() + + boxerName = "BoxerTest" + boxName = "BoxTop" + iops = dict(_boxer=boxerName, _box=boxName) + + hold = Hold() + + tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) + hold[tymeKey] = Bag() + hold[tymeKey].value = tymist.tyme + + activeKey = hold.tokey(("", "boxer", boxerName, "active")) + hold[activeKey] = Bag() + hold[activeKey].value = boxName + + tockKey = hold.tokey(("", "boxer", boxerName, "tock")) + hold[tockKey] = Bag() + hold[tockKey].value = tymist.tock + + tymth = tymist.tymen() + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + name = "elk" + + + hog = Hog(name=name, iops=iops, hold=hold, temp=True) + assert hog.opened + assert hog.file + assert not hog.file.closed + + ciops = dict(it=hog, clear=True, **iops) + + cact = CloseAct(iops=ciops, hold=hold) + assert cact.name == 'CloseAct1' + assert cact.iops == ciops + assert cact.nabe == Nabes.exdo + assert cact.hold == hold + assert cact.Index == 2 + assert cact.Instances[cact.name] == cact + + cact() + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + + + """Done Test""" if __name__ == "__main__": @@ -1078,4 +1145,5 @@ def test_rechange_mark_basic(): test_reupdate_mark_basic() test_change_mark_basic() test_rechange_mark_basic() + test_closeact_basic() diff --git a/tests/base/hier/test_boxing.py b/tests/base/hier/test_boxing.py index 646c6d2..9565ecf 100644 --- a/tests/base/hier/test_boxing.py +++ b/tests/base/hier/test_boxing.py @@ -1322,6 +1322,96 @@ def fun(H, bx, go, do, on, at, be, *pa): ] """Done Test""" +def test_boxer_doer_hold_log(): + """Test BoxerDoer with hold log Hog""" + + def fun(H, bx, go, do, on, at, be, *pa): + H.test = Can(value=True) + H.buf = Durq() + H.buf.push(Bag(value=True)) + H.puf = Dusq() + H.puf.push(Bag(value=False)) + bx(name='top') + hog = do("log") # logs boxer state default afdo + do("close", it=hog, clear=True) # closes and clears log default exdo + bx('mid', 'top') + at('redo') + do("count") + at("exdo") + do("discount") + go('done', on("count >= 5")) + bx('bot0', 'mid', first=True) + go("next", on("lapse >= 2.0")) + bx('bot1') # over defaults to same as prev box + go("next", on("lapse >= 2.0")) + bx('bot2') # over defaults to same as prev box + go("bot0") + bx(name='done', over=None) + do('end') + + tock = 1.0 + doist = Doist(tock=tock, temp=True) + assert doist.tyme == 0.0 # on next cycle + assert doist.tock == tock == 1.0 + assert doist.real == False + assert doist.limit == None + assert doist.doers == [] + + hold = Hold() + boxer = Boxer(hold=hold, fun=fun, durable=True) + assert boxer.fun == fun + assert boxer.boxes == {} + + doer = BoxerDoer(boxer=boxer, tock=tock) + assert doer.boxer == boxer + assert doer.tock == tock + + doers = [doer] + + ticks = 10 + limit = tock * ticks + assert limit == 10.0 + doist.do(doers=doers, limit=limit) # doist.do sets all doer.tymth to its tymth + assert doist.tyme == 8.0 # redoer exits before limit + + # sdb does not exist anymore since temp clears at close of doer + assert boxer.hold.test.value == True + assert not os.path.exists(boxer.hold.subery.path) + assert not boxer.hold.subery.opened + + assert len(boxer.hold.buf) == 1 + assert boxer.hold.buf.pull() == Bag(value=True) + + assert len(boxer.hold.puf) == 1 + assert boxer.hold.puf.pull() == Bag(value=False) + + assert hold._boxer_boxer_active.value == None + assert hold._boxer_boxer_tock.value == 1.0 + assert hold._boxer_boxer_end.value == True + assert hold._boxer_boxer_box_mid_count.value == None + assert hold._boxer_boxer_box_bot0_lapse.value == 5.0 + assert hold._boxer_boxer_box_bot0_lapse._tyme == 5.0 + assert hold._boxer_boxer_box_bot0_lapse._now == 8.0 + assert hold._boxer_boxer_box_bot1_lapse.value == 2.0 + assert hold._boxer_boxer_box_bot1_lapse._tyme == 2.0 + assert hold._boxer_boxer_box_bot1_lapse._now == 8.0 + + assert list(boxer.hold.keys()) == \ + [ + '_hold_subery', + 'test', + 'buf', + 'puf', + '_boxer_boxer_box_mid_count', + '_boxer_boxer_box_bot0_lapse', + '_boxer_boxer_box_bot1_lapse', + '_boxer_boxer_end', + '_boxer_boxer_tyme', + '_boxer_boxer_active', + '_boxer_boxer_tock', + ] + """Done Test""" + def test_boxery_basic(): @@ -1655,6 +1745,7 @@ def bx(name: None|str=None, over: None|str|Box="")->Box: test_boxer_run_lapse() test_boxer_run_relapse() test_boxer_doer() + test_boxer_doer_hold_log() test_boxery_basic() test_concept_bx_nonlocal() test_concept_bx_global() From e03febff26bd2bb9ab4d0dfcdf8cdab8821fffac Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 18:39:56 -0600 Subject: [PATCH 31/36] added code for Hog log cycling need to test --- src/hio/base/hier/hogging.py | 117 +++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 31 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 75d28d0..7c8c04d 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -12,15 +12,18 @@ import uuid from contextlib import contextmanager import inspect -from collections import namedtuple +from collections import namedtuple, deque from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 +from ...hioing import HierError +from ...help import timing # import timing to pytest mock of nowIso8601 works +from ...help.helping import ocfn from ..doing import Doer from ..filing import Filer from .hiering import Nabes from .acting import ActBase, register -from ...help import timing # import timing to pytest mock of nowIso8601 works + Ruleage = namedtuple("Rules", 'once every span update change') Rules = Ruleage(once='once', every='every', span='span', update='update', change='change') @@ -110,16 +113,14 @@ class Hog(ActBase, Filer): 0.0 means everytyme flushLast (float|None): tyme last flushed, None means not yet running cycleCount (int): number of cycled logs, 0 means do not cycle (count) - cyclePaths (list[str]): paths for cycled logs - cycleSpan (float): min tyme span for cycling logs (cycle). 0.0 means + cyclePeriod (float): min tyme span for cycling logs (cycle). 0.0 means cycle based on cycleHigh not tyme span. One of cycleSpan or cycleHigh must be non zero - cycleLast (float|None): tyme last cycled. None means not yet running - cycleLow (int): minimum size in bytes required for cycling log based on - cycleSpan. 0 means no minimum - cycleHigh (int): maximum size in bytes allowed for each cycled log + cycleSize (int): maximum size in bytes allowed for each cycled log 0 means no maximum. One of cycleSpan or cycleHigh must be non zero + cyclePaths (list[str]): paths for cycled logs + cycleLast (float|None): tyme last cycled. None means not yet running hits (dict): hold items to log. Item label is log header tag Item value is hold key that provides value to log marks (dict): tyme or value tuples marks of hold items logged with @@ -167,7 +168,7 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, mode='a+', fext="hog", reuse=True, rid=None, rule=Rules.every, span=0.0, flush=60.0, - count=0, cycle=0.0, low=0, high=0, hits=None, **kwa): + count=0, period=0.0, size=0, hits=None, **kwa): """Initialize instance. Inherited Parameters: @@ -218,10 +219,8 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, flush (float): flush tyme span seconds, tyme between flushes 0.0 means every tyme count (int): number of cycled logs, 0 means do not cycle - cycle (float): cycle tyme span, tyme between log cycles - low (int): minimum size in bytes required for cycling log - 0 means no minimum - high (int): maximum size in bytes allowed for each cycled log + period (float): cycle tyme period, tyme between log cycles + size (int): maximum size in bytes allowed for each cycled log 0 means no maximum hits (None|dict): hold items to log. Item label is log header tag Item value is hold key that provides value to log @@ -259,12 +258,15 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.flushSpan = flush self.flushLast = None - self.cycleCount = count + if count and period == 0.0 and size == 0: + raise HierError(f"For non-zero count one of {period=} or {size=} " + f"must be non-zero") + + self.cycleCount = max(min(count, 99), 0) + self.cyclePeriod = period + self.cycleSize = size self.cyclePaths = [] # need to init - self.cycleSpan = cycle self.cycleLast = None - self.cycleLow = low - self.cycleHigh = high self.activeKey = None self.tockKey = None @@ -284,6 +286,20 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.hits = hits self.marks = {} + if self.cycleCount > 0: + self.cyclePaths = [] + for k in range(1, self.cycleCount + 1): + root, fext = os.path.splitext(self.path) + path = f"{root}_{k:02}.{fext}" + self.cyclePaths.append(path) + # trial open to ensure can make + try: + file = ocfn(path, 'r') # do not truncate in case reusing + except OSError as ex: + raise HierError("Failed making cycle paths") from ex + + file.close() + def act(self, **iops): """Act called by ActBase. @@ -340,8 +356,8 @@ def act(self, **iops): tyme = self.hold[self.hits["tyme"]].value if self.hits else None if not self.onced: - #self.file.write(self.record()) - #self.flush() + self.first = tyme + self.cycleLast = tyme # always flush on first write to ensure header synced on disk self.log(self.record(), tyme, force=True) if self.hits: @@ -352,20 +368,15 @@ def act(self, **iops): elif self.rule == Rules.change: # create mark self.marks[key] = self.hold[key]._astuple() - self.first = tyme - self.last = tyme - self.flushLast = tyme + + self.onced = True else: match self.rule: case Rules.every: - #self.file.write(self.record()) - #self.last = tyme self.log(self.record(), tyme) case Rules.span: if tyme is not None and tyme - self.last >= self.span: - #self.file.write(self.record()) - #self.last = tyme self.log(self.record(), tyme) case Rules.update: @@ -377,8 +388,6 @@ def act(self, **iops): self.marks[key] = holdTyme updated = True if updated: - #self.file.write(self.record()) - #self.last = tyme self.log(self.record(), tyme) case Rules.change: @@ -390,11 +399,8 @@ def act(self, **iops): self.marks[key] = holdValue changed = True if changed: - #self.file.write(self.record()) - #self.last = tyme self.log(self.record(), tyme) - case _: pass @@ -412,10 +418,27 @@ def log(self, record, tyme, force=False): """ self.file.write(record) self.last = tyme + if force or (tyme - self.flushLast) >= self.flushSpan: self.flush() self.flushLast = tyme + if self.cycleCount: + cycled = False + if self.cyclePeriod: + if (tyme - self.cycleLast) >= self.cyclePeriod: + self.cycle(tyme=tyme) + cycled = True + + if self.cycleSize and not cycled: + try: + size = os.path.getsize(self.path) + except OSError as ex: + pass + else: + if size >= self.cycleSize: + self.cycle(tyme=tyme) + def record(self): """Generate on record line .hits values from .hold @@ -436,6 +459,38 @@ def record(self): return "" + def cycle(self, tyme): + """Cycle log files + + Parameters: + tyme (float): current tyme of cycle + """ + self.flush + cycles = deque([self.path]) + cycles.extend(self.cyclePaths) + + cycled = False # if cycling is successful + new = cycles.pop() + while cycles: + old = cycles.pop() + try: + os.replace(old, new) # rename old to new thereby clobbering old + cycled = True + except OSError as ex: + cycled = False # failed to cycle so do not clobber self.file + break + new = old # old path is now free to use + + if not cycled: # reopen cleanly just in case + self.reopen(reuse=True) # reopen reuse, .mode is "a+" so saves + else: # all cycled so recreate self.file + self.reopen() # not reuse so recreates empty + + self.file.write(self.header) # rewrite header + self.flush() + self.flushLast = tyme + self.cycleLast = tyme + @contextmanager def openHog(cls=None, name=None, temp=True, reopen=True, clear=False, **kwa): From 491ca527fe46ab9664255a1612f4e16d6b1e9c95 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 21:13:14 -0600 Subject: [PATCH 32/36] tests for cycling (rotating) logs now working --- src/hio/base/hier/hogging.py | 53 ++-- tests/base/hier/test_hogging.py | 543 +++++++++++++++++++++++++++++++- 2 files changed, 574 insertions(+), 22 deletions(-) diff --git a/src/hio/base/hier/hogging.py b/src/hio/base/hier/hogging.py index 7c8c04d..4cc8aba 100644 --- a/src/hio/base/hier/hogging.py +++ b/src/hio/base/hier/hogging.py @@ -111,6 +111,8 @@ class Hog(ActBase, Filer): rid (str): universally unique run ID for given run of hog flushSpan (float): tyme span seconds between flushes (flush) 0.0 means everytyme + flushForce (bool): True means force flush on every log + False means only flush at appropriate times flushLast (float|None): tyme last flushed, None means not yet running cycleCount (int): number of cycled logs, 0 means do not cycle (count) cyclePeriod (float): min tyme span for cycling logs (cycle). 0.0 means @@ -167,8 +169,10 @@ class Hog(ActBase, Filer): def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, extensioned=True, mode='a+', fext="hog", reuse=True, - rid=None, rule=Rules.every, span=0.0, flush=60.0, - count=0, period=0.0, size=0, hits=None, **kwa): + rid=None, rule=Rules.every, span=0.0, + flushSpan=60.0, flushForce=False, + cycleCount=0, cycleSpan=0.0, cycleSize=0, + hits=None, **kwa): """Initialize instance. Inherited Parameters: @@ -216,11 +220,13 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, (once, every, span, update, change) span (float): periodic tyme span seconds when rule is spanned 0.0 means every tyme - flush (float): flush tyme span seconds, tyme between flushes + flushSpan (float): flush tyme span seconds, tyme between flushes 0.0 means every tyme - count (int): number of cycled logs, 0 means do not cycle - period (float): cycle tyme period, tyme between log cycles - size (int): maximum size in bytes allowed for each cycled log + flushForce (bool): True means force flush on every log + False means only flush at appropriate times + cycleCount (int): number of cycled logs, 0 means do not cycle + cycleSapn (float): cycle tyme span, tyme between log cycles + cycleSize (int): maximum size in bytes allowed for each cycled log 0 means no maximum hits (None|dict): hold items to log. Item label is log header tag Item value is hold key that provides value to log @@ -255,16 +261,17 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, self.rid = rid self.stamp = '' # need to init self.header = '' # need to init - self.flushSpan = flush + self.flushSpan = flushSpan + self.flushForce = flushForce self.flushLast = None - if count and period == 0.0 and size == 0: - raise HierError(f"For non-zero count one of {period=} or {size=} " - f"must be non-zero") + if cycleCount and cycleSpan == 0.0 and cycleSize == 0: + raise HierError(f"For non-zero count one of {cycleSpan=} or " + f"{cycleSize=} must be non-zero") - self.cycleCount = max(min(count, 99), 0) - self.cyclePeriod = period - self.cycleSize = size + self.cycleCount = max(min(cycleCount, 99), 0) + self.cycleSpan = cycleSpan + self.cycleSize = cycleSize self.cyclePaths = [] # need to init self.cycleLast = None @@ -289,8 +296,8 @@ def __init__(self, iops=None, nabe=Nabes.afdo, base="", filed=True, if self.cycleCount > 0: self.cyclePaths = [] for k in range(1, self.cycleCount + 1): - root, fext = os.path.splitext(self.path) - path = f"{root}_{k:02}.{fext}" + root, ext = os.path.splitext(self.path) + path = f"{root}_{k:02}{ext}" # ext includes leading dot self.cyclePaths.append(path) # trial open to ensure can make try: @@ -357,7 +364,6 @@ def act(self, **iops): if not self.onced: self.first = tyme - self.cycleLast = tyme # always flush on first write to ensure header synced on disk self.log(self.record(), tyme, force=True) if self.hits: @@ -368,8 +374,6 @@ def act(self, **iops): elif self.rule == Rules.change: # create mark self.marks[key] = self.hold[key]._astuple() - - self.onced = True else: match self.rule: @@ -419,14 +423,19 @@ def log(self, record, tyme, force=False): self.file.write(record) self.last = tyme - if force or (tyme - self.flushLast) >= self.flushSpan: + if force or self.flushForce or (tyme - self.flushLast) >= self.flushSpan: self.flush() self.flushLast = tyme if self.cycleCount: cycled = False - if self.cyclePeriod: - if (tyme - self.cycleLast) >= self.cyclePeriod: + if self.cycleSpan: + if self.cycleLast is None: + delta = tyme - self.first + else: + delta = tyme - self.cycleLast + + if delta >= self.cycleSpan: self.cycle(tyme=tyme) cycled = True @@ -474,6 +483,8 @@ def cycle(self, tyme): while cycles: old = cycles.pop() try: + if old == self.path: + self.close() os.replace(old, new) # rename old to new thereby clobbering old cycled = True except OSError as ex: diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index c05772e..2602b89 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -186,7 +186,7 @@ def test_hog_doer(): def test_hog_log(mockHelpingNowIso8601): - """Test Hog clas with logging""" + """Test Hog class with logging rules""" Hog._clearall() # clear Hog.Instances for debugging @namify @@ -999,6 +999,547 @@ def __hash__(self): """Done Test""" +def test_hog_cycle_size(mockHelpingNowIso8601): + """Test Hog class with cycle size (rotated logs) logging""" + Hog._clearall() # clear Hog.Instances for debugging + + @namify + @registerify + @dataclass + class LocationBag(TymeDom): + """Vector Bag dataclass + + Field Attributes: + latN (Any): latitude North fractional minutes + lonE (Any): longitude East fractional minutes + """ + latN: Any = None + lonE: Any = None + + def __hash__(self): + """Define hash so can work with ordered_set + __hash__ is not inheritable in dataclasses so must be explicitly defined + in every subclass + """ + return hash((self.__class__.__name__,) + self._astuple()) # almost same as __eq__ + + + tymist = Tymist() + + dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked + assert dts == '2021-06-27T21:26:21.233257+00:00' + + boxerName = "BoxerTest" + boxName = "BoxTop" + iops = dict(_boxer=boxerName, _box=boxName) + + uid = 'KQzSlod5EfC1TvKsr0VvkQ' # for test + rid = f"{boxerName}_{uid}" + + hold = Hold() + + tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) + hold[tymeKey] = Bag() + hold[tymeKey].value = tymist.tyme + + activeKey = hold.tokey(("", "boxer", boxerName, "active")) + hold[activeKey] = Bag() + hold[activeKey].value = boxName + + tockKey = hold.tokey(("", "boxer", boxerName, "tock")) + hold[tockKey] = Bag() + hold[tockKey].value = tymist.tock + + homeKey = hold.tokey(("location", "home", )) + hold[homeKey] = LocationBag(latN=45.0, lonE=-90.0) + + awayKey = hold.tokey(("location", "away", )) + hold[awayKey] = LocationBag(latN=40.0, lonE=10.0) + + tymth = tymist.tymen() + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + # Test vector hold bag with rule "every" + name = "bat" + count = 2 + period = tymist.tock * 2 + size = 300 + + # vector locations as hits with default rule every + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, flushForce=True, + cycleCount=count, cycleSpan=period, cycleSize=size, + home=homeKey, away=awayKey) + assert hog.rule == Rules.every + assert not hog.started + assert not hog.onced + assert hog.flushForce == True + assert hog.cycleCount == count + assert hog.cycleSpan == period == 0.0625 + assert hog.cycleSize == size + + assert hog.hits == {'home': 'location_home', 'away': 'location_away'} + assert len(hog.cyclePaths) == count + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'location_home', + 'away': 'location_away' + } + assert hog.marks == {} + + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.flushLast == 0.0 + assert hog.cycleLast == None + assert hog.rid == rid + assert hog.stamp == dts + + assert os.path.getsize(hog.path) == 272 + for path in hog.cyclePaths: + assert os.path.getsize(path) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + assert len(lines[-1]) == 25 + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 41.0 + hold[awayKey].lonE = 11.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme + assert hog.cycleLast == tymist.tyme # cycled due to size + + assert os.path.getsize(hog.path) == 247 # only header + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') + ] + + cycle1 = open(hog.cyclePaths[0], "r") + lines = cycle1.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0'), + ('0.03125', '45.0', '-90.0', '41.0', '11.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 42.0 + hold[awayKey].lonE = 12.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme == 0.0625 + assert hog.cycleLast == 0.03125 + + assert os.path.getsize(hog.path) == 275 + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0625', '45.0', '-90.0', '42.0', '12.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 43.0 + hold[awayKey].lonE = 13.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme == 0.09375 + assert hog.cycleLast == 0.09375 + + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 304 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 301 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') + ] + + cycle1 = open(hog.cyclePaths[0], "r") + lines = cycle1.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0625', '45.0', '-90.0', '42.0', '12.0'), + ('0.09375', '45.0', '-90.0', '43.0', '13.0') + ] + + + cycle2 = open(hog.cyclePaths[1], "r") + lines = cycle2.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0'), + ('0.03125', '45.0', '-90.0', '41.0', '11.0') + ] + + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + """Done Test""" + +def test_hog_cycle_span(mockHelpingNowIso8601): + """Test Hog class with cycle span (rotated logs) logging""" + Hog._clearall() # clear Hog.Instances for debugging + + @namify + @registerify + @dataclass + class LocationBag(TymeDom): + """Vector Bag dataclass + + Field Attributes: + latN (Any): latitude North fractional minutes + lonE (Any): longitude East fractional minutes + """ + latN: Any = None + lonE: Any = None + + def __hash__(self): + """Define hash so can work with ordered_set + __hash__ is not inheritable in dataclasses so must be explicitly defined + in every subclass + """ + return hash((self.__class__.__name__,) + self._astuple()) # almost same as __eq__ + + + tymist = Tymist() + + dts = hio.help.timing.nowIso8601() # mocked version testing that mocking worked + assert dts == '2021-06-27T21:26:21.233257+00:00' + + boxerName = "BoxerTest" + boxName = "BoxTop" + iops = dict(_boxer=boxerName, _box=boxName) + + uid = 'KQzSlod5EfC1TvKsr0VvkQ' # for test + rid = f"{boxerName}_{uid}" + + hold = Hold() + + tymeKey = hold.tokey(("", "boxer", boxerName, "tyme")) + hold[tymeKey] = Bag() + hold[tymeKey].value = tymist.tyme + + activeKey = hold.tokey(("", "boxer", boxerName, "active")) + hold[activeKey] = Bag() + hold[activeKey].value = boxName + + tockKey = hold.tokey(("", "boxer", boxerName, "tock")) + hold[tockKey] = Bag() + hold[tockKey].value = tymist.tock + + homeKey = hold.tokey(("location", "home", )) + hold[homeKey] = LocationBag(latN=45.0, lonE=-90.0) + + awayKey = hold.tokey(("location", "away", )) + hold[awayKey] = LocationBag(latN=40.0, lonE=10.0) + + tymth = tymist.tymen() + for dom in hold.values(): # wind hold + if isinstance(dom, TymeDom): + dom._wind(tymth=tymth) + + # Test vector hold bag with rule "every" + name = "bat" + count = 2 + period = tymist.tock * 2 + size = 2048 + + # vector locations as hits with default rule every + hog = Hog(name=name, iops=iops, hold=hold, temp=True, rid=rid, flushForce=True, + cycleCount=count, cycleSpan=period, cycleSize=size, + home=homeKey, away=awayKey) + assert hog.rule == Rules.every + assert not hog.started + assert not hog.onced + assert hog.flushForce == True + assert hog.cycleCount == count + assert hog.cycleSpan == period == 0.0625 + assert hog.cycleSize == size + + assert hog.hits == {'home': 'location_home', 'away': 'location_away'} + assert len(hog.cyclePaths) == count + + # run hog once + assert hog() == iops # default returns iops + assert hog.hits == \ + { + 'tyme': '_boxer_BoxerTest_tyme', + 'home': 'location_home', + 'away': 'location_away' + } + assert hog.marks == {} + + assert hog.started + assert hog.onced + assert hog.first == 0.0 + assert hog.last == 0.0 + assert hog.flushLast == 0.0 + assert hog.cycleLast == None + assert hog.rid == rid + assert hog.stamp == dts + + assert os.path.getsize(hog.path) == 272 + for path in hog.cyclePaths: + assert os.path.getsize(path) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + assert len(lines[-1]) == 25 + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 41.0 + hold[awayKey].lonE = 11.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme + assert hog.cycleLast == None # not cycled yet + + assert os.path.getsize(hog.path) == 301 + assert os.path.getsize(hog.cyclePaths[0]) == 0 + assert os.path.getsize(hog.cyclePaths[1]) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0'), + ('0.03125', '45.0', '-90.0', '41.0', '11.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 42.0 + hold[awayKey].lonE = 12.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme == 0.0625 + assert hog.cycleLast == 0.0625 == hog.cycleSpan # cycled due to time + + assert os.path.getsize(hog.path) == 247 # just header + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') + ] + + cycle1 = open(hog.cyclePaths[0], "r") + lines = cycle1.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0'), + ('0.03125', '45.0', '-90.0', '41.0', '11.0'), + ('0.0625', '45.0', '-90.0', '42.0', '12.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 43.0 + hold[awayKey].lonE = 13.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme == 0.09375 + assert hog.cycleLast == 0.0625 + + assert os.path.getsize(hog.path) == 276 + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.09375', '45.0', '-90.0', '43.0', '13.0') + ] + + # run again + tymist.tick() + hold[tymeKey].value = tymist.tyme + hold[awayKey].latN = 44.0 + hold[awayKey].lonE = 14.0 + + assert hog() == iops # default returns iops + assert hog.last != hog.first # since once does not log again + assert hog.last == tymist.tyme + assert hog.flushLast == tymist.tyme == 0.125 + assert hog.cycleLast == 0.125 + + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 303 + assert os.path.getsize(hog.cyclePaths[1]) == 329 + + hog.file.seek(0, os.SEEK_SET) # seek to beginning of file + lines = hog.file.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') + ] + + cycle1 = open(hog.cyclePaths[0], "r") + lines = cycle1.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.09375', '45.0', '-90.0', '43.0', '13.0'), + ('0.125', '45.0', '-90.0', '44.0', '14.0') + ] + + + cycle2 = open(hog.cyclePaths[1], "r") + lines = cycle2.readlines() + lines = [tuple(line.rstrip('\n').split('\t')) for line in lines] + assert lines == \ + [ + ('rid', 'base', 'name', 'stamp', 'rule', 'count'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('tyme.key', 'home.key', 'away.key'), + ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), + ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), + ('0.0', '45.0', '-90.0', '40.0', '10.0'), + ('0.03125', '45.0', '-90.0', '41.0', '11.0'), + ('0.0625', '45.0', '-90.0', '42.0', '12.0') + ] + + hog.close(clear=True) + assert not hog.opened + assert not hog.file + assert not os.path.exists(hog.path) + """Done Test""" + + if __name__ == "__main__": test_hog_basic() test_open_hog() From 9c06b4f2548027cb211439d09183a4fdebdcb887 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 21:21:37 -0600 Subject: [PATCH 33/36] fixed bug with registerify --- src/hio/help/doming.py | 1 + tests/base/hier/test_hogging.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/hio/help/doming.py b/src/hio/help/doming.py index 177ea9d..d0de563 100644 --- a/src/hio/help/doming.py +++ b/src/hio/help/doming.py @@ -15,6 +15,7 @@ from collections.abc import Callable from dataclasses import dataclass, astuple, asdict, fields, field, InitVar +from ..hioing import HierError from .helping import NonStringIterable # DOM Utilities dataclass utility classes diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 2602b89..2f40611 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -1004,7 +1004,6 @@ def test_hog_cycle_size(mockHelpingNowIso8601): Hog._clearall() # clear Hog.Instances for debugging @namify - @registerify @dataclass class LocationBag(TymeDom): """Vector Bag dataclass @@ -1062,7 +1061,7 @@ def __hash__(self): dom._wind(tymth=tymth) # Test vector hold bag with rule "every" - name = "bat" + name = "rat" count = 2 period = tymist.tock * 2 size = 300 @@ -1112,7 +1111,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), @@ -1141,7 +1140,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') @@ -1153,7 +1152,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), @@ -1183,7 +1182,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), @@ -1212,7 +1211,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE') @@ -1224,7 +1223,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), @@ -1239,7 +1238,7 @@ def __hash__(self): assert lines == \ [ ('rid', 'base', 'name', 'stamp', 'rule', 'count'), - ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','bat','2021-06-27T21:26:21.233257+00:00','every','2'), + ('BoxerTest_KQzSlod5EfC1TvKsr0VvkQ','BoxerTest','rat','2021-06-27T21:26:21.233257+00:00','every','2'), ('tyme.key', 'home.key', 'away.key'), ('_boxer_BoxerTest_tyme', 'location_home', 'location_away'), ('tyme.value', 'home.latN', 'home.lonE', 'away.latN', 'away.lonE'), @@ -1258,7 +1257,6 @@ def test_hog_cycle_span(mockHelpingNowIso8601): Hog._clearall() # clear Hog.Instances for debugging @namify - @registerify @dataclass class LocationBag(TymeDom): """Vector Bag dataclass From 9109cf9f8151fcbd75d341537e3698c129515114 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 21:38:13 -0600 Subject: [PATCH 34/36] file sizes not same on windows because of /r/n versus /n --- tests/base/hier/test_hogging.py | 61 ++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index 2f40611..aa25afe 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -1100,7 +1100,11 @@ def __hash__(self): assert hog.rid == rid assert hog.stamp == dts - assert os.path.getsize(hog.path) == 272 + if platform.system() == 'Windows': + assert os.path.getsize(hog.path) == 278 # /r/n + else: + assert os.path.getsize(hog.path) == 272 + for path in hog.cyclePaths: assert os.path.getsize(path) == 0 @@ -1130,9 +1134,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme assert hog.cycleLast == tymist.tyme # cycled due to size - assert os.path.getsize(hog.path) == 247 # only header - assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 0 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 247 # only header + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1172,9 +1177,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.0625 assert hog.cycleLast == 0.03125 - assert os.path.getsize(hog.path) == 275 - assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 0 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 275 + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1201,9 +1207,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.09375 assert hog.cycleLast == 0.09375 - assert os.path.getsize(hog.path) == 247 - assert os.path.getsize(hog.cyclePaths[0]) == 304 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 301 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 304 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 301 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1353,7 +1360,11 @@ def __hash__(self): assert hog.rid == rid assert hog.stamp == dts - assert os.path.getsize(hog.path) == 272 + if platform.system() == 'Windows': + assert os.path.getsize(hog.path) == 278 # /r/n + else: + assert os.path.getsize(hog.path) == 272 + for path in hog.cyclePaths: assert os.path.getsize(path) == 0 @@ -1383,9 +1394,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme assert hog.cycleLast == None # not cycled yet - assert os.path.getsize(hog.path) == 301 - assert os.path.getsize(hog.cyclePaths[0]) == 0 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 301 + assert os.path.getsize(hog.cyclePaths[0]) == 0 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1413,9 +1425,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.0625 assert hog.cycleLast == 0.0625 == hog.cycleSpan # cycled due to time - assert os.path.getsize(hog.path) == 247 # just header - assert os.path.getsize(hog.cyclePaths[0]) == 329 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 247 # just header + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1456,9 +1469,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.09375 assert hog.cycleLast == 0.0625 - assert os.path.getsize(hog.path) == 276 - assert os.path.getsize(hog.cyclePaths[0]) == 329 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 276 + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1485,9 +1499,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.125 assert hog.cycleLast == 0.125 - assert os.path.getsize(hog.path) == 247 - assert os.path.getsize(hog.cyclePaths[0]) == 303 - assert os.path.getsize(hog.cyclePaths[1]) == 329 + if platform.system() != 'Windows': + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 303 + assert os.path.getsize(hog.cyclePaths[1]) == 329 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() From badf26f65f575cc75e25f19e5963fa7f94d46747 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 21:46:59 -0600 Subject: [PATCH 35/36] do not run log cycle tests on windows as all file size are different due to windows /r/n vs /n so unit tests need to adapt file sizes and also when rotation occurs --- tests/base/hier/test_hogging.py | 68 +++++++++++++++------------------ 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index aa25afe..a0b847e 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -1001,6 +1001,9 @@ def __hash__(self): def test_hog_cycle_size(mockHelpingNowIso8601): """Test Hog class with cycle size (rotated logs) logging""" + if platform.system() != 'Windows': + return + Hog._clearall() # clear Hog.Instances for debugging @namify @@ -1100,11 +1103,7 @@ def __hash__(self): assert hog.rid == rid assert hog.stamp == dts - if platform.system() == 'Windows': - assert os.path.getsize(hog.path) == 278 # /r/n - else: - assert os.path.getsize(hog.path) == 272 - + assert os.path.getsize(hog.path) == 272 for path in hog.cyclePaths: assert os.path.getsize(path) == 0 @@ -1134,10 +1133,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme assert hog.cycleLast == tymist.tyme # cycled due to size - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 247 # only header - assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 0 + assert os.path.getsize(hog.path) == 247 # only header + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1177,10 +1175,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.0625 assert hog.cycleLast == 0.03125 - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 275 - assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 0 + assert os.path.getsize(hog.path) == 275 + assert os.path.getsize(hog.cyclePaths[0]) == 301 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1207,10 +1204,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.09375 assert hog.cycleLast == 0.09375 - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 247 - assert os.path.getsize(hog.cyclePaths[0]) == 304 # over cycleSize - assert os.path.getsize(hog.cyclePaths[1]) == 301 + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 304 # over cycleSize + assert os.path.getsize(hog.cyclePaths[1]) == 301 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1261,6 +1257,9 @@ def __hash__(self): def test_hog_cycle_span(mockHelpingNowIso8601): """Test Hog class with cycle span (rotated logs) logging""" + if platform.system() != 'Windows': + return + Hog._clearall() # clear Hog.Instances for debugging @namify @@ -1360,11 +1359,7 @@ def __hash__(self): assert hog.rid == rid assert hog.stamp == dts - if platform.system() == 'Windows': - assert os.path.getsize(hog.path) == 278 # /r/n - else: - assert os.path.getsize(hog.path) == 272 - + assert os.path.getsize(hog.path) == 272 for path in hog.cyclePaths: assert os.path.getsize(path) == 0 @@ -1394,10 +1389,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme assert hog.cycleLast == None # not cycled yet - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 301 - assert os.path.getsize(hog.cyclePaths[0]) == 0 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + assert os.path.getsize(hog.path) == 301 + assert os.path.getsize(hog.cyclePaths[0]) == 0 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1425,10 +1419,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.0625 assert hog.cycleLast == 0.0625 == hog.cycleSpan # cycled due to time - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 247 # just header - assert os.path.getsize(hog.cyclePaths[0]) == 329 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + assert os.path.getsize(hog.path) == 247 # just header + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1469,10 +1462,9 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.09375 assert hog.cycleLast == 0.0625 - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 276 - assert os.path.getsize(hog.cyclePaths[0]) == 329 - assert os.path.getsize(hog.cyclePaths[1]) == 0 + assert os.path.getsize(hog.path) == 276 + assert os.path.getsize(hog.cyclePaths[0]) == 329 + assert os.path.getsize(hog.cyclePaths[1]) == 0 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() @@ -1499,10 +1491,10 @@ def __hash__(self): assert hog.flushLast == tymist.tyme == 0.125 assert hog.cycleLast == 0.125 - if platform.system() != 'Windows': - assert os.path.getsize(hog.path) == 247 - assert os.path.getsize(hog.cyclePaths[0]) == 303 - assert os.path.getsize(hog.cyclePaths[1]) == 329 + + assert os.path.getsize(hog.path) == 247 + assert os.path.getsize(hog.cyclePaths[0]) == 303 + assert os.path.getsize(hog.cyclePaths[1]) == 329 hog.file.seek(0, os.SEEK_SET) # seek to beginning of file lines = hog.file.readlines() From c802d4b56420560eaad1a966aafbe96db2cd3763 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Sep 2025 21:52:37 -0600 Subject: [PATCH 36/36] fixed logic mess up --- tests/base/hier/test_hogging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base/hier/test_hogging.py b/tests/base/hier/test_hogging.py index a0b847e..b788d97 100644 --- a/tests/base/hier/test_hogging.py +++ b/tests/base/hier/test_hogging.py @@ -1001,7 +1001,7 @@ def __hash__(self): def test_hog_cycle_size(mockHelpingNowIso8601): """Test Hog class with cycle size (rotated logs) logging""" - if platform.system() != 'Windows': + if platform.system() == 'Windows': return Hog._clearall() # clear Hog.Instances for debugging @@ -1257,7 +1257,7 @@ def __hash__(self): def test_hog_cycle_span(mockHelpingNowIso8601): """Test Hog class with cycle span (rotated logs) logging""" - if platform.system() != 'Windows': + if platform.system() == 'Windows': return Hog._clearall() # clear Hog.Instances for debugging