diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7684a5e --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,46 @@ +name: Lint + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: latest + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run Ruff Linter + run: poetry run ruff check . + + - name: Run Ruff Formatter Check + run: poetry run ruff format --check . \ No newline at end of file diff --git a/.gitignore b/.gitignore index 63c2e41..c54181b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc .vscode/ -.vscode/settings.json \ No newline at end of file +.vscode/settings.json +poetry.lock +__pycache__/ \ No newline at end of file diff --git a/README.md b/README.md index 8da3bb1..d450c07 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ simple micro service for converting your RRD's to web services ### getting started - ensure you have ```rrdtool``` installed and you can access the rrd files from the server - git clone the project ``` git clone https://github.com/tbotnz/RRDReST && cd RRDReST ``` -- install the requirements ```pip3 install -r requirements.txt``` -- run the app with uvicorn ```uvicorn rrdrest:rrd_rest --host "0.0.0.0" --port 9000``` +- install dependencies ```poetry install``` +- run the app with poetry ```poetry run uvicorn rrdrest:rrd_rest --host "0.0.0.0" --port 9000``` - access the swagger documentation via ```http://127.0.0.1:9000/docs``` ### examples diff --git a/__version__.py b/__version__.py new file mode 100644 index 0000000..e7c2a5b --- /dev/null +++ b/__version__.py @@ -0,0 +1,5 @@ +__version__ = "0.3.0" + + +def get_version(): + return __version__ diff --git a/backend/RRD_parse.py b/backend/RRD_parse.py index bc0827e..19948aa 100644 --- a/backend/RRD_parse.py +++ b/backend/RRD_parse.py @@ -8,7 +8,6 @@ class RRD_parser: - def __init__(self, rrd_file=None, start_time=None, end_time=None): self.rrd_file = rrd_file self.ds = None @@ -19,23 +18,21 @@ def __init__(self, rrd_file=None, start_time=None, end_time=None): self.end_time = end_time def check_dependc(self): - result = subprocess.check_output( - "rrdtool --version", - shell=True - ).decode('utf-8') + result = subprocess.check_output("rrdtool --version", shell=True).decode( + "utf-8" + ) if "RRDtool 1." not in result: raise Exception("RRDtool version not found, check rrdtool installed") def get_data_source(self): - """ gets datasources from rrd tool """ + """gets datasources from rrd tool""" STEP_VAL = None DS_VALS = [] result = subprocess.check_output( - f"rrdtool info {self.rrd_file}", - shell=True - ).decode('utf-8') + f"rrdtool info {self.rrd_file}", shell=True + ).decode("utf-8") temp_arr = result.split("\n") @@ -48,7 +45,7 @@ def get_data_source(self): STEP_VAL = raw_val if ("ds[" in raw_key) and ("]." in raw_key): - match_obj = re.match(r'^ds\[(.*)\]', raw_key) + match_obj = re.match(r"^ds\[(.*)\]", raw_key) if match_obj: ds_val = match_obj.group(1) if ds_val not in DS_VALS: @@ -57,47 +54,47 @@ def get_data_source(self): self.ds = DS_VALS def get_rrd_json(self, ds): - """ gets RRD json from rrd tool """ - + """gets RRD json from rrd tool""" + rrd_xport_command = f"rrdtool xport --step {self.step} DEF:data={self.rrd_file}:{ds}:AVERAGE XPORT:data:{ds} --showtime" if self.start_time: rrd_xport_command = f"rrdtool xport DEF:data={self.rrd_file}:{ds}:AVERAGE XPORT:data:{ds} --showtime --start {self.start_time} --end {self.end_time}" - result = subprocess.check_output( - rrd_xport_command, - shell=True - ).decode('utf-8') + result = subprocess.check_output(rrd_xport_command, shell=True).decode("utf-8") json_result = json.dumps(xmltodict.parse(result), indent=4) # replace rrdtool v key with the ds - replace_val = "\""+ds.lower()+"\": " - temp_result_one = re.sub("\"v\": ", replace_val, json_result) + replace_val = '"' + ds.lower() + '": ' + temp_result_one = re.sub('"v": ', replace_val, json_result) return json.loads(temp_result_one) def cleanup_payload(self, payload): - """ cleans up / transforms response payload """ + """cleans up / transforms response payload""" # convert timezones and floats for count, temp_obj in enumerate(payload["data"]): epoch_time = temp_obj["t"] - utc_time = datetime.datetime.fromtimestamp( - int(epoch_time) - ).strftime(self.time_format) + utc_time = datetime.datetime.fromtimestamp(int(epoch_time)).strftime( + self.time_format + ) payload["data"][count]["t"] = utc_time for key in payload["data"][count]: temp_val = "" - if "e+" in payload["data"][count][key] or "e-" in payload["data"][count][key]: + if ( + "e+" in payload["data"][count][key] + or "e-" in payload["data"][count][key] + ): temp_val = payload["data"][count][key] payload["data"][count][key] = float(temp_val) pl = json.dumps(payload) # convert ints, floats - pl = re.sub(r'\"(\d+)\"', r'\1', f"{pl}") - pl = re.sub(r'\"(\d+\.\d+)\"', r'\1', f"{pl}") + pl = re.sub(r"\"(\d+)\"", r"\1", f"{pl}") + pl = re.sub(r"\"(\d+\.\d+)\"", r"\1", f"{pl}") # convert NaN to null - pl = re.sub(r'\"NaN\"', "null", f"{pl}") + pl = re.sub(r"\"NaN\"", "null", f"{pl}") # replace "t" with time - pl = re.sub(r'\"t\"', r'"time"', f"{pl}") + pl = re.sub(r"\"t\"", r'"time"', f"{pl}") # return response as JSON obj return json.loads(pl) @@ -111,10 +108,9 @@ def compile_result(self): "step": "", "end": "", "rows": "", - "data_sources": [] + "data_sources": [], }, "data": [], - } collector = defaultdict(dict) @@ -123,19 +119,17 @@ def compile_result(self): r = self.get_rrd_json(ds=d) master_result["meta"]["start"] = datetime.datetime.fromtimestamp( int(r["xport"]["meta"]["start"]) - ).strftime(self.time_format) + ).strftime(self.time_format) master_result["meta"]["step"] = r["xport"]["meta"]["step"] master_result["meta"]["end"] = datetime.datetime.fromtimestamp( int(r["xport"]["meta"]["end"]) - ).strftime(self.time_format) + ).strftime(self.time_format) master_result["meta"]["rows"] = 0 master_result["meta"]["data_sources"].append( r["xport"]["meta"]["legend"]["entry"] - ) + ) - for collectible in chain( - master_result["data"], r["xport"]["data"]["row"] - ): + for collectible in chain(master_result["data"], r["xport"]["data"]["row"]): collector[collectible["t"]].update(collectible.items()) # combine objs, add row_count diff --git a/backend/security.py b/backend/security.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..26c7cc3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[tool.poetry] +name = "rrdrest" +version = "0.0.0" # Will be replaced by dynamic versioning +description = "Makes RRD files API-able" +authors = ["tbotnz "] +readme = "README.md" +packages = [{include = "rrdrest.py"}, {include = "backend"}, {include = "__version__.py"}] + +[tool.poetry.dependencies] +python = "^3.11" +fastapi = "^0.119.1" +xmltodict = "^1.0.2" +pydantic = "^2.12.3" +uvicorn = "^0.38.0" +uvloop = "^0.22.1" +gunicorn = "^23.0.0" +httptools = "^0.7.1" +ruff = "^0.14.1" + +[tool.poetry-dynamic-versioning] +enable = true +vcs = "any" +style = "pep440" +metadata = false +format-jinja = "{{serialize_pep440(bump_version(base))}}" +# Read version from __version__.py +format = "{base}" +pattern = "default-unprefixed" + +[tool.poetry-dynamic-versioning.substitution] +files = ["__version__.py"] + +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0", "poetry-dynamic-versioning"] +build-backend = "poetry_dynamic_versioning.backend" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4bee7f9..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -fastapi==0.59.0 -xmltodict==0.12.0 -pydantic==1.6.1 -uvicorn==0.13.4 -uvloop==0.15.2 -gunicorn==20.0.4 -httptools==0.1.1 \ No newline at end of file diff --git a/rrdrest.py b/rrdrest.py index 1d7f6e7..c31ec59 100644 --- a/rrdrest.py +++ b/rrdrest.py @@ -1,34 +1,75 @@ from fastapi import FastAPI, HTTPException +from pydantic import Field from backend.RRD_parse import RRD_parser +from __version__ import __version__ -from typing import Optional - +from typing import Optional, Annotated import os + rrd_rest = FastAPI( title="RRDReST", description="Makes RRD files API-able", - version="0.2", + version=__version__, ) -@rrd_rest.get( - "/", - summary="Get the data from a RRD file, takes in a rrd file path" - ) -async def get_rrd(rrd_path: str, epoch_start_time: Optional[int] = None, epoch_end_time: Optional[int] = None): - is_file = os.path.isfile(rrd_path) - if is_file: - if (epoch_start_time and not epoch_end_time) or (epoch_end_time and not epoch_start_time): - raise HTTPException(status_code=500, detail="If epoch start or end time is specified both start and end time MUST be specified") - try: - rr = RRD_parser( - rrd_file=rrd_path, - start_time=epoch_start_time, - end_time=epoch_end_time - ) - r = rr.compile_result() - return r - except Exception as e: - HTTPException(status_code=500, detail=f"{e}") - raise HTTPException(status_code=404, detail="RRD not found") +def _validate_time_params( + epoch_start_time: Optional[int], epoch_end_time: Optional[int] +): + if (epoch_start_time and not epoch_end_time) or ( + epoch_end_time and not epoch_start_time + ): + raise HTTPException( + status_code=400, + detail="If epoch start or end time is specified both start and end time MUST be specified", + ) + + +def _parse_rrd_file( + rrd_path: Annotated[ + str, Field(description="RRD file path", examples=["/path/to/file.rrd"]) + ], + epoch_start_time: Optional[int], + epoch_end_time: Optional[int], +): + if not os.path.isfile(rrd_path): + raise HTTPException(status_code=404, detail=f"RRD file not found: {rrd_path}") + + _validate_time_params(epoch_start_time, epoch_end_time) + + try: + rr = RRD_parser( + rrd_file=rrd_path, start_time=epoch_start_time, end_time=epoch_end_time + ) + return rr.compile_result() + except Exception as e: + raise HTTPException(status_code=500, detail=f"Error parsing RRD file: {e}") + + +@rrd_rest.get("/", summary="Get data from a RRD file, takes in a rrd file path") +async def get_rrd( + rrd_path: str, + epoch_start_time: Optional[int] = None, + epoch_end_time: Optional[int] = None, +): + return _parse_rrd_file(rrd_path, epoch_start_time, epoch_end_time) + + +@rrd_rest.post("/", summary="Get multiple RRD files, takes in a list of rrd file paths") +async def get_rrd_multiple( + rrd_paths: list[ + Annotated[ + str, Field(description="RRD file path", examples=["/path/to/file.rrd"]) + ] + ], + epoch_start_time: Optional[int] = None, + epoch_end_time: Optional[int] = None, +): + results = {} + for rrd_path in rrd_paths: + result = _parse_rrd_file(rrd_path, epoch_start_time, epoch_end_time) + results[rrd_path] = result + if not results: + raise HTTPException(status_code=404, detail="No RRD Data Found") + return results