From c2f85189fa126253312b3f4a4d72042490df2fb8 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 15 Sep 2021 17:48:40 +0200 Subject: [PATCH 1/7] Fix open in binary mode. --- test_pytracing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_pytracing.py b/test_pytracing.py index aa114d3..b3a333e 100755 --- a/test_pytracing.py +++ b/test_pytracing.py @@ -28,7 +28,7 @@ def main(): if __name__ == '__main__': - with io.open('./trace.out', mode='w', encoding='utf-8') as fh: + with io.open('./trace.out', mode='wb') as fh: tp = TraceProfiler(output=fh) tp.install() main() From 7f778745da287fca5231be3bc632987cc0ea1559 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 15 Sep 2021 17:53:57 +0200 Subject: [PATCH 2/7] Use default encoding for json. --- pytracing/pytracing.py | 11 ++++++++--- test_pytracing.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pytracing/pytracing.py b/pytracing/pytracing.py index 607f532..9cead8a 100644 --- a/pytracing/pytracing.py +++ b/pytracing/pytracing.py @@ -9,8 +9,11 @@ import json import time import threading +import logging from contextlib import contextmanager +logger = logging.getLogger(__name__) + try: from queue import Queue except ImportError: @@ -32,17 +35,19 @@ def __init__(self, terminator, input_queue, output_stream): def _open_collection(self): """Write the opening of a JSON array to the output.""" - self.output.write(b'[') + logger.debug('_open_collection') + self.output.write('[') def _close_collection(self): """Write the closing of a JSON array to the output.""" - self.output.write(b'{}]') # empty {} so the final entry doesn't end with a comma + logger.debug('_close_collection') + self.output.write('{}]') # empty {} so the final entry doesn't end with a comma def run(self): self._open_collection() while not self.terminator.is_set() or not self.input.empty(): item = self.input.get() - self.output.write((json.dumps(item) + ',\n').encode('ascii')) + self.output.write(json.dumps(item) + ',\n') self._close_collection() diff --git a/test_pytracing.py b/test_pytracing.py index b3a333e..aa114d3 100755 --- a/test_pytracing.py +++ b/test_pytracing.py @@ -28,7 +28,7 @@ def main(): if __name__ == '__main__': - with io.open('./trace.out', mode='wb') as fh: + with io.open('./trace.out', mode='w', encoding='utf-8') as fh: tp = TraceProfiler(output=fh) tp.install() main() From b6c3267f7a01a9e11605e6327e15866ffbba855b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 16 Sep 2021 05:18:54 -0700 Subject: [PATCH 3/7] Add GH actions CI, lint and test. --- .github/workflows/python-package.yml | 34 +++++ LICENSE | 1 - pytracing/__init__.py | 2 +- pytracing/pytracing.py | 209 +++++++++++++-------------- setup.cfg | 15 ++ setup.py | 15 +- test_pytracing.py | 33 ++--- 7 files changed, 178 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..88b0cba --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,34 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[test] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint + run: | + pre-commit run -v -a + flake8 . --config=setup.cfg + - name: Test with pytest + run: | + #pytest + ./test_pytracing.py diff --git a/LICENSE b/LICENSE index 5cc10a6..b83c351 100644 --- a/LICENSE +++ b/LICENSE @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/pytracing/__init__.py b/pytracing/__init__.py index aa0ec66..dc7dc34 100644 --- a/pytracing/__init__.py +++ b/pytracing/__init__.py @@ -6,4 +6,4 @@ from .pytracing import TraceProfiler -__all__ = ['TraceProfiler'] +__all__ = ["TraceProfiler"] diff --git a/pytracing/pytracing.py b/pytracing/pytracing.py index 9cead8a..b00fcec 100644 --- a/pytracing/pytracing.py +++ b/pytracing/pytracing.py @@ -15,119 +15,118 @@ logger = logging.getLogger(__name__) try: - from queue import Queue + from queue import Queue except ImportError: - from Queue import Queue + from Queue import Queue def to_microseconds(s): - return 1000000 * float(s) + return 1000000 * float(s) class TraceWriter(threading.Thread): - - def __init__(self, terminator, input_queue, output_stream): - threading.Thread.__init__(self) - self.daemon = True - self.terminator = terminator - self.input = input_queue - self.output = output_stream - - def _open_collection(self): - """Write the opening of a JSON array to the output.""" - logger.debug('_open_collection') - self.output.write('[') - - def _close_collection(self): - """Write the closing of a JSON array to the output.""" - logger.debug('_close_collection') - self.output.write('{}]') # empty {} so the final entry doesn't end with a comma - - def run(self): - self._open_collection() - while not self.terminator.is_set() or not self.input.empty(): - item = self.input.get() - self.output.write(json.dumps(item) + ',\n') - self._close_collection() + def __init__(self, terminator, input_queue, output_stream): + threading.Thread.__init__(self) + self.daemon = True + self.terminator = terminator + self.input = input_queue + self.output = output_stream + + def _open_collection(self): + """Write the opening of a JSON array to the output.""" + logger.debug("_open_collection") + self.output.write("[") + + def _close_collection(self): + """Write the closing of a JSON array to the output.""" + logger.debug("_close_collection") + self.output.write("{}]") # empty {} so the final entry doesn't end with a comma + + def run(self): + self._open_collection() + while not self.terminator.is_set() or not self.input.empty(): + item = self.input.get() + self.output.write(json.dumps(item) + ",\n") + self._close_collection() class TraceProfiler(object): - """A python trace profiler that outputs Chrome Trace-Viewer format (about://tracing). - - Usage: - - from pytracing import TraceProfiler - tp = TraceProfiler(output=open('/tmp/trace.out', 'wb')) - with tp.traced(): - ... - - """ - TYPES = {'call': 'B', 'return': 'E'} - - def __init__(self, output, clock=None): - self.output = output - self.clock = clock or time.time - self.pid = os.getpid() - self.queue = Queue() - self.terminator = threading.Event() - self.writer = TraceWriter(self.terminator, self.queue, self.output) - - @property - def thread_id(self): - return threading.current_thread().name - - @contextmanager - def traced(self): - """Context manager for install/shutdown in a with block.""" - self.install() - try: - yield - finally: - self.shutdown() - - def install(self): - """Install the trace function and open the JSON output stream.""" - self.writer.start() # Start the writer thread. - sys.setprofile(self.tracer) # Set the trace/profile function. - threading.setprofile(self.tracer) # Set the trace/profile function for threads. - - def shutdown(self): - sys.setprofile(None) # Clear the trace/profile function. - threading.setprofile(None) # Clear the trace/profile function for threads. - self.terminator.set() # Stop the writer thread. - self.writer.join() # Join the writer thread. - - def fire_event(self, event_type, func_name, func_filename, func_line_no, - caller_filename, caller_line_no): - """Write a trace event to the output stream.""" - timestamp = to_microseconds(self.clock()) - # https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview - - event = dict( - name=func_name, # Event Name. - cat=func_filename, # Event Category. - tid=self.thread_id, # Thread ID. - ph=self.TYPES[event_type], # Event Type. - pid=self.pid, # Process ID. - ts=timestamp, # Timestamp. - args=dict( - function=':'.join([str(x) for x in (func_filename, func_line_no, func_name)]), - caller=':'.join([str(x) for x in (caller_filename, caller_line_no)]), - ) - ) - self.queue.put(event) - - def tracer(self, frame, event_type, arg): - """Bound tracer function for sys.settrace().""" - try: - if event_type in self.TYPES.keys() and frame.f_code.co_name != 'write': - self.fire_event( - event_type=event_type, - func_name=frame.f_code.co_name, - func_filename=frame.f_code.co_filename, - func_line_no=frame.f_lineno, - caller_filename=frame.f_back.f_code.co_filename, - caller_line_no=frame.f_back.f_lineno, + """A python trace profiler that outputs Chrome Trace-Viewer format (about://tracing). + + Usage: + + from pytracing import TraceProfiler + tp = TraceProfiler(output=open('/tmp/trace.out', 'wb')) + with tp.traced(): + ... + + """ + + TYPES = {"call": "B", "return": "E"} + + def __init__(self, output, clock=None): + self.output = output + self.clock = clock or time.time + self.pid = os.getpid() + self.queue = Queue() + self.terminator = threading.Event() + self.writer = TraceWriter(self.terminator, self.queue, self.output) + + @property + def thread_id(self): + return threading.current_thread().name + + @contextmanager + def traced(self): + """Context manager for install/shutdown in a with block.""" + self.install() + try: + yield + finally: + self.shutdown() + + def install(self): + """Install the trace function and open the JSON output stream.""" + self.writer.start() # Start the writer thread. + sys.setprofile(self.tracer) # Set the trace/profile function. + threading.setprofile(self.tracer) # Set the trace/profile function for threads. + + def shutdown(self): + sys.setprofile(None) # Clear the trace/profile function. + threading.setprofile(None) # Clear the trace/profile function for threads. + self.terminator.set() # Stop the writer thread. + self.writer.join() # Join the writer thread. + + def fire_event(self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no): + """Write a trace event to the output stream.""" + timestamp = to_microseconds(self.clock()) + # https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview + + event = dict( + name=func_name, # Event Name. + cat=func_filename, # Event Category. + tid=self.thread_id, # Thread ID. + ph=self.TYPES[event_type], # Event Type. + pid=self.pid, # Process ID. + ts=timestamp, # Timestamp. + args=dict( + function=":".join([str(x) for x in (func_filename, func_line_no, func_name)]), + caller=":".join([str(x) for x in (caller_filename, caller_line_no)]), + ), ) - except Exception: - pass # Don't disturb execution if we can't log the trace. + self.queue.put(event) + + def tracer(self, frame, event_type, arg): + """Bound tracer function for sys.settrace().""" + try: + if event_type in self.TYPES.keys() and frame.f_code.co_name != "write": + self.fire_event( + event_type=event_type, + func_name=frame.f_code.co_name, + func_filename=frame.f_code.co_filename, + func_line_no=frame.f_lineno, + caller_filename=frame.f_back.f_code.co_filename, + caller_line_no=frame.f_back.f_lineno, + ) + except Exception: + pass # Don't disturb execution if we can't log the trace. diff --git a/setup.cfg b/setup.cfg index b88034e..15bb1fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,17 @@ [metadata] description-file = README.md + +[flake8] +exclude = + venv*, + .pybuilder, + build, + dist +max-line-length = 120 +select = E9,F63,F7,F82 +max-complexity = 10 +verbose = false +jobs = auto +count = true +show-source = true +statistics = true diff --git a/setup.py b/setup.py index 250b8c2..0c0516e 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,12 @@ from setuptools import setup setup( - name='pytracing', - version='0.4', - description='a python trace profiler that outputs to chrome trace-viewer format (about://tracing).', - author='Kris Wilson', - author_email='kwilson@twitter.com', - url='https://www.github.com/kwlzn/pytracing', - packages=['pytracing'] + name="pytracing", + version="0.4", + description="a python trace profiler that outputs to chrome trace-viewer format (about://tracing).", + author="Kris Wilson", + author_email="kwilson@twitter.com", + extras_require={"test": ["flake8==3.9.2", "pre-commit"]}, + url="https://www.github.com/kwlzn/pytracing", + packages=["pytracing"], ) diff --git a/test_pytracing.py b/test_pytracing.py index aa114d3..255676d 100755 --- a/test_pytracing.py +++ b/test_pytracing.py @@ -13,29 +13,28 @@ def function_a(x): - print('sleeping {}'.format(x)) - time.sleep(x) - return + print("sleeping {}".format(x)) + time.sleep(x) + return def function_b(x): - function_a(x) + function_a(x) def main(): - function_a(1) - function_b(2) + function_a(1) + function_b(2) -if __name__ == '__main__': - with io.open('./trace.out', mode='w', encoding='utf-8') as fh: - tp = TraceProfiler(output=fh) - tp.install() - main() - tp.shutdown() - print('wrote trace.out') - - # ensure the output is at least valid JSON - with io.open('./trace.out', encoding='utf-8') as fh: - json.load(fh) +if __name__ == "__main__": + with io.open("./trace.out", mode="w", encoding="utf-8") as fh: + tp = TraceProfiler(output=fh) + tp.install() + main() + tp.shutdown() + print("wrote trace.out") + # ensure the output is at least valid JSON + with io.open("./trace.out", encoding="utf-8") as fh: + json.load(fh) From 87798a156afcf2b11857802e113cb73faa7dec88 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 16 Sep 2021 05:20:32 -0700 Subject: [PATCH 4/7] Add pre-commit-config --- .pre-commit-config.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..91680cd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-aws-credentials + args: [--allow-missing-credentials] + +- repo: https://github.com/humitos/mirrors-autoflake.git + rev: v1.3 + hooks: + - id: autoflake + args: ['--in-place', '--expand-star-imports', '--ignore-init-module-imports', '--remove-all-unused-imports'] + +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + args: [--line-length=120] + + #- repo: https://github.com/pre-commit/mirrors-isort + # rev: v4.3.21 + # hooks: + # - id: isort From d0d372ef0bb1c890da84f4f88ebb8c371b859b5b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 16 Sep 2021 05:38:29 -0700 Subject: [PATCH 5/7] Use a lint.sh script. --- .github/workflows/python-package.yml | 3 +-- lint.sh | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100755 lint.sh diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 88b0cba..3fbbc71 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,8 +26,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint run: | - pre-commit run -v -a - flake8 . --config=setup.cfg + ./lint.sh - name: Test with pytest run: | #pytest diff --git a/lint.sh b/lint.sh new file mode 100755 index 0000000..c71099a --- /dev/null +++ b/lint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +pre-commit run -v -a +flake8 . --config=setup.cfg From 8fb50d2dfba2085477ec25c4011703fb659c671b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 16 Sep 2021 05:39:01 -0700 Subject: [PATCH 6/7] Add a callback array so other tracers can be easily installed. --- pytracing/pytracing.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pytracing/pytracing.py b/pytracing/pytracing.py index b00fcec..57f551a 100644 --- a/pytracing/pytracing.py +++ b/pytracing/pytracing.py @@ -11,6 +11,7 @@ import threading import logging from contextlib import contextmanager +from typing import Dict, Any logger = logging.getLogger(__name__) @@ -71,6 +72,7 @@ def __init__(self, output, clock=None): self.queue = Queue() self.terminator = threading.Event() self.writer = TraceWriter(self.terminator, self.queue, self.output) + self.event_callbacks = [self._chrome_tracing_event] @property def thread_id(self): @@ -97,11 +99,14 @@ def shutdown(self): self.terminator.set() # Stop the writer thread. self.writer.join() # Join the writer thread. - def fire_event(self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no): - """Write a trace event to the output stream.""" + def _chrome_tracing_event( + self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no + ) -> Dict[str, Any]: + """ + Format a Chrome tracing event that can be encoded to JSON + https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview + """ timestamp = to_microseconds(self.clock()) - # https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview - event = dict( name=func_name, # Event Name. cat=func_filename, # Event Category. @@ -114,7 +119,13 @@ def fire_event(self, event_type, func_name, func_filename, func_line_no, caller_ caller=":".join([str(x) for x in (caller_filename, caller_line_no)]), ), ) - self.queue.put(event) + return event + + def fire_event(self, event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no): + """Trigger event callbacks.""" + for cb in self.event_callbacks: + event = cb(event_type, func_name, func_filename, func_line_no, caller_filename, caller_line_no) + self.queue.put(event) def tracer(self, frame, event_type, arg): """Bound tracer function for sys.settrace().""" From 34d0a69921cdf263787724bcff6d2c1a4dcdf15b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 16 Sep 2021 05:40:31 -0700 Subject: [PATCH 7/7] Add python 3.9 to CI. --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3fbbc71..94988fb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2