From 0b6f556fdd3444c11137a4db47d6f2807a37a11d Mon Sep 17 00:00:00 2001 From: Ryan Wallner Date: Wed, 10 Sep 2025 12:20:00 -0400 Subject: [PATCH 1/2] fixes for CVEs, reduce image size, base in on alpine --- README.md | 46 +- base-fiotools/Dockerfile | 75 ++- buildimages.sh | 90 +++- fio-genplots/Dockerfile | 15 +- fio-genplots/lib/fio2gnuplot | 319 ++++++------ fio-genplots/lib/graph2D.gpm | 55 ++ fio-genplots/lib/graph3D.gpm | 95 ++++ fio-genplots/lib/math.gpm | 42 ++ fio-genplots/plot.sh | 99 ++++ fio-plotserve/Dockerfile | 12 +- fio-tool/Dockerfile | 13 +- fio-tool/run.sh | 12 +- fiotools-aio/Dockerfile | 38 +- fiotools-aio/check.sh | 4 +- fiotools-aio/lib/fio2gnuplot | 319 ++++++------ fiotools-aio/lib/fio2gnuplot.bak | 527 ++++++++++++++++++++ fiotools-aio/lib/fio2gnuplot.bak.2 | 527 ++++++++++++++++++++ fiotools-aio/lib/graph2D.gpm | 55 ++ fiotools-aio/lib/graph3D.gpm | 95 ++++ fiotools-aio/lib/math.gpm | 42 ++ fiotools-aio/plot.sh | 102 +++- fiotools-aio/run.sh | 16 +- fiotools-aio/runall.sh | 8 +- kubernetes/fiotools-aio-portworx-read.yaml | 18 +- kubernetes/fiotools-aio-portworx-write.yaml | 20 +- 25 files changed, 2211 insertions(+), 433 deletions(-) create mode 100644 fio-genplots/lib/graph2D.gpm create mode 100644 fio-genplots/lib/graph3D.gpm create mode 100644 fio-genplots/lib/math.gpm create mode 100644 fio-genplots/plot.sh mode change 100644 => 100755 fiotools-aio/lib/fio2gnuplot create mode 100644 fiotools-aio/lib/fio2gnuplot.bak create mode 100755 fiotools-aio/lib/fio2gnuplot.bak.2 create mode 100644 fiotools-aio/lib/graph2D.gpm create mode 100644 fiotools-aio/lib/graph3D.gpm create mode 100644 fiotools-aio/lib/math.gpm diff --git a/README.md b/README.md index 4b82259..8e9bd64 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ``` docker run -v /tmp/fio-data:/tmp/fio-data \ -e JOBFILES= \ - portworx/fio-tool + wallnerryan/fio-tool ``` If your file is a remote raw text file, you can use REMOTEFILES @@ -24,8 +24,7 @@ If your file is a remote raw text file, you can use REMOTEFILES 4. Run the `fio-genplots` script ``` - docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots \ - + docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots ``` 5. Serve your Graph Images and Log Files @@ -98,25 +97,21 @@ env: ##### To use with docker and docker volumes ``` docker run \ --e REMOTEFILES="https://gist.githubusercontent.com/wallnerryan/fd0146ee3122278d7b5f/raw/cdd8de476abbecb5fb5c56239ab9b6eb3cec3ed5/job.fio" \ --v /tmp/fio-data:/tmp/fio-data \ ---volume-driver flocker \ +-p 8000:8000 \ +-v /tmp/fio-data \ +-e REMOTEFILES="https://gist.githubusercontent.com/wallnerryan/6bcfec794cbaef9a86569d5553b156b3/raw/8a6f5a6cb924f493a095b5077ed402d71a333b52/realworld.fio" \ +-e JOBFILES=realworld.fio \ +-e PLOTNAME=MyTest \ -v myvol1:/myvol \ --e JOBFILES=job.fio wallnerryan/fio-tool +-d \ +--name MyTest wallnerryan/fiotools-aio ``` -To produce graphs, run the genplots container, `-p ` +To produce graphs, run the genplots container -*Produce Bandwidth Graphs* +*Produce IOPS/Bandwidth Graphs* ``` -docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots \ --t My16kAWSRandomReadTest -b -g -p *_bw* -``` - -*Produce IOPS graphs* -``` -docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots \ --t My16kAWSRandomReadTest -i -g -p *_iops* +docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots ``` Simply serve them on port 8000 @@ -156,14 +151,13 @@ wallnerryan/fio-tool If you have a directory that already has them in it ``` docker run -v /Users/wallnerryan/Desktop/fio:/tmp/fio-data \ --e JOBFILES=job.fio wallnerryan/fio-tool +-e JOBFILES=job.fio fiotools/fio-tool ``` -To produce graphs, run the genplots container, `-p ` +To produce graphs, run the genplots container ``` docker run \ --v /Users/wallnerryan/Desktop/fio:/tmp/fio-data wallnerryan/fio-genplots \ --t My16kAWSRandomReadTest -b -g -p *_bw* +-v /Users/wallnerryan/Desktop/fio:/tmp/fio-data wallnerryan/fio-genplots ``` Simply serve them on port 8000 @@ -191,10 +185,9 @@ docker run -v /tmp/fio-data:/tmp/fio-data \ -e JOBFILES= wallnerryan/fio-tool ``` -To produce graphs, run the genplots container, `-p ` +To produce graphs, run the genplots container ``` -docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots \ --t My16kAWSRandomReadTest -b -g -p *_bw* +docker run -v /tmp/fio-data:/tmp/fio-data wallnerryan/fio-genplots ``` Simply serve them on port 8000 @@ -213,3 +206,8 @@ wallnerryan/fio-plotserve **bw= and BW= explained** - https://www.spinics.net/lists/fio/msg05517.html + + +**build** + +`env bash ./buildimages.sh` \ No newline at end of file diff --git a/base-fiotools/Dockerfile b/base-fiotools/Dockerfile index 4c0b915..634e1ed 100644 --- a/base-fiotools/Dockerfile +++ b/base-fiotools/Dockerfile @@ -1,13 +1,62 @@ -FROM ubuntu:18.10 -MAINTAINER - -RUN apt-get -y update -RUN apt-get -y install fio \ - wget \ - libqt5gui5 \ - gnuplot \ - python2.7 - -# Bug "gnuplot: error while loading shared libraries: libQt5Core.so.5: cannot open shared object file: No such file or directory" -# https://askubuntu.com/questions/1041588/virtualbox-not-launching-on-ubuntu-18-04-qt-lib-problem -RUN apt-get remove -y libqt5core5a && apt-get install -y libqt5core5a \ No newline at end of file +# ---------- Stage 1: build a minimal gnuplot (no cairo/pango/harfbuzz) ---------- +ARG ALPINE_VERSION=3.20 +ARG GNUPLOT_VERSION=5.4.10 +FROM alpine:${ALPINE_VERSION} AS gnuplot-build +ARG GNUPLOT_VERSION + +# Build gnuplot to avoid CVEs in upstreadm 3.20 alpine +# two criticals are in glib and harfbuzz pulled in by Alpine’s gnuplot stack (via cairo/pango). +RUN set -eux; \ + apk add --no-cache \ + build-base wget tar \ + gd-dev libpng-dev freetype-dev libjpeg-turbo-dev zlib-dev readline-dev; \ + mkdir -p /tmp/build && cd /tmp/build; \ + wget -O gnuplot.tar.gz \ + "https://downloads.sourceforge.net/project/gnuplot/gnuplot/${GNUPLOT_VERSION}/gnuplot-${GNUPLOT_VERSION}.tar.gz"; \ + tar -xzf gnuplot.tar.gz; \ + cd gnuplot-${GNUPLOT_VERSION}; \ + ./configure \ + --prefix=/usr \ + --without-x \ + --without-qt \ + --disable-wxwidgets \ + --without-aquaterm \ + --without-cairo \ + --without-lua \ + --with-gd; \ + make -j"$(nproc)"; \ + make install-strip + +# ---------- Stage 2: runtime ---------- +ARG ALPINE_VERSION=3.20 +FROM alpine:${ALPINE_VERSION} +LABEL maintainer="Ryan Wallner " + +# runtime deps (no cairo/pango/harfbuzz) +RUN set -eux; \ + apk add --no-cache \ + fio jq wget ca-certificates tini python3 py3-six \ + gd libpng freetype libjpeg-turbo zlib readline ncurses-libs ttf-dejavu + +# bring in the self-built gnuplot + its shared data +COPY --from=gnuplot-build /usr/bin/gnuplot /usr/bin/gnuplot +COPY --from=gnuplot-build /usr/share/gnuplot /usr/share/gnuplot + +# Make fio’s gnuplot templates work headlessly: +RUN set -eux; \ + sed -i 's/^set term .*/set term png size 1280,720/' /usr/share/fio/*.gpm; \ + sed -i 's|font "arial"|font "/usr/share/fonts/TTF/DejaVuSans.ttf"|' /usr/share/fio/*.gpm; \ + sed -i 's/^set pm3d at .*/set pm3d at s; set hidden3d; set pm3d scansbackward/' /usr/share/fio/graph3D.gpm + +# OCP/K8s random-UID friendly perms +RUN set -eux; \ + mkdir -p /opt /tmp/fio-data; \ + chgrp -R 0 /opt /tmp/fio-data; \ + chmod -R g=u /opt /tmp/fio-data + +VOLUME ["/tmp/fio-data"] +WORKDIR /tmp/fio-data + +ENTRYPOINT ["/sbin/tini","--"] +CMD ["/bin/sh"] + diff --git a/buildimages.sh b/buildimages.sh index df74dbf..9a7e955 100755 --- a/buildimages.sh +++ b/buildimages.sh @@ -1,15 +1,75 @@ -#!/bin/bash - -echo "Enter your Dockerhub Username or a prefix for the images..." -DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME:-myfiotools} - -docker build -t base-fiotools base-fiotools/ -docker build -t fiotools/fio-genplots fio-genplots/ -docker build -t fiotools/fio-tool fio-tool/ -docker build -t fiotools/fio-plotserve fio-plotserve/ -docker build -t fiotools/fiotools-aio fiotools-aio -docker tag base-fiotools $DOCKERHUB_USERNAME/base-fiotools -docker tag fiotools/fio-genplots $DOCKERHUB_USERNAME/fio-genplots -docker tag fiotools/fio-tool $DOCKERHUB_USERNAME/fio-tool -docker tag fiotools/fio-plotserve $DOCKERHUB_USERNAME/fio-plotserve -docker tag fiotools/fiotools-aio $DOCKERHUB_USERNAME/fiotools-aio +#!/usr/bin/env bash +set -euo pipefail + +NS="${FIOTOOLS_DOCKERHUB_USERNAME:-fiotools}" +TAG="${FIOTOOLS_DOCKERHUB_TAG:-tag}" +PUSH="${FIOTOOLS_DOCKERHUB_PUSH:-false}" +PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}" + +echo "Using repo: $NS tag: $TAG platforms: $PLATFORMS push: $PUSH" + +# ---- helpers --------------------------------------------------------------- + +is_true() { + case "${1,,}" in + 1|true|yes|y) return 0 ;; + *) return 1 ;; + esac +} + +host_load_platform() { + # choose the platform we can --load into the host daemon + case "$(uname -m)" in + x86_64) echo "linux/amd64" ;; + aarch64|arm64) echo "linux/arm64" ;; + *) echo "linux/amd64" ;; + esac +} + +ensure_builder() { + local name="$1" driver="$2" + if ! docker buildx inspect "$name" >/dev/null 2>&1; then + docker buildx create --name "$name" --driver "$driver" --use >/dev/null + else + docker buildx use "$name" >/dev/null + fi +} + +# ---- builder selection ----------------------------------------------------- + +if is_true "$PUSH"; then + # multi-arch + push -> container driver + ensure_builder fiotools docker-container + # enable qemu emulation for cross-builds (idempotent) + docker run --privileged --rm tonistiigi/binfmt --install all >/dev/null + BUILD_EXTRA=(--push --platform "$PLATFORMS") +else + # local single-arch build that can see host images -> docker driver + ensure_builder fiotools-local docker + LOAD_PLAT="$(host_load_platform)" + BUILD_EXTRA=(--load --platform "$LOAD_PLAT") + echo "Local load platform: $LOAD_PLAT" +fi + +build() { + local img="$1" dir="$2" + shift 2 || true + echo "==> Building $img from $dir" + docker buildx build "$dir" -t "$img" "${BUILD_EXTRA[@]}" "$@" +} + +# ---- build order ----------------------------------------------------------- + +# 1) base (must be first) +BASE_REF="${NS}/base-fiotools:${TAG}" +GNUPLOT_VERSION=5.4.10 +build "$BASE_REF" base-fiotools/ --build-arg GNUPLOT_VERSION=$GNUPLOT_VERSION + +# 2) dependents (pass the base via ARG) +build "${NS}/fio-genplots:${TAG}" fio-genplots/ --build-arg BASE_IMAGE="$BASE_REF" +build "${NS}/fio-tool:${TAG}" fio-tool/ --build-arg BASE_IMAGE="$BASE_REF" +build "${NS}/fio-plotserve:${TAG}" fio-plotserve/ --build-arg BASE_IMAGE="$BASE_REF" +build "${NS}/fiotools-aio:${TAG}" fiotools-aio/ --build-arg BASE_IMAGE="$BASE_REF" + +echo "Done." + diff --git a/fio-genplots/Dockerfile b/fio-genplots/Dockerfile index 5dd227c..46a7ed7 100644 --- a/fio-genplots/Dockerfile +++ b/fio-genplots/Dockerfile @@ -1,9 +1,12 @@ -FROM base-fiotools -MAINTAINER +# syntax=docker/dockerfile:1.6 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +LABEL maintainer="Ryan Wallner " -# Add modified script to y axis is set accurately -ADD lib/fio2gnuplot /usr/bin/fio2gnuplot -RUN chmod +x /usr/bin/fio2gnuplot +# Uses fio2gnuplot to build graphs from *_bw* and *_iops* logs +# We reuse the project's plot.sh to keep behavior identical. +ADD plot.sh /opt/plot.sh +RUN chmod +x /opt/plot.sh WORKDIR /tmp/fio-data -ENTRYPOINT ["/usr/bin/fio2gnuplot"] +CMD ["/opt/plot.sh"] \ No newline at end of file diff --git a/fio-genplots/lib/fio2gnuplot b/fio-genplots/lib/fio2gnuplot index 50e9d29..ce3ca2c 100644 --- a/fio-genplots/lib/fio2gnuplot +++ b/fio-genplots/lib/fio2gnuplot @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# Note: this script is python2 and python3 compatible. # # Copyright (C) 2013 eNovance SAS # Author: Erwan Velu @@ -17,8 +18,10 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import absolute_import +from __future__ import print_function import os import fnmatch import sys @@ -26,27 +29,29 @@ import getopt import re import math import shutil +from six.moves import map +from six.moves import range def find_file(path, pattern): fio_data_file=[] # For all the local files for file in os.listdir(path): - # If the file math the regexp - if fnmatch.fnmatch(file, pattern): - # Let's consider this file - fio_data_file.append(file) + # If the file matches the glob + if fnmatch.fnmatch(file, pattern): + # Let's consider this file + fio_data_file.append(file) return fio_data_file def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): - if verbose: print "Generating rendering scripts" + if verbose: print("Generating rendering scripts") filename=gnuplot_output_dir+'mygraph' temporary_files.append(filename) f=open(filename,'w') # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: - f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) + f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) # Setting up the compare files that will be plot later compare=open(gnuplot_output_dir + 'compare.gnuplot','w') @@ -88,10 +93,10 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); - pos=0 - # Let's create a temporary file for each selected fio file - for file in fio_data_file: - tmp_filename = "gnuplot_temp_file.%d" % pos + pos=0 + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "gnuplot_temp_file.%d" % pos # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: @@ -101,12 +106,12 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos])) png_file=file.replace('.log','') - raw_filename = "%s-2Draw" % (png_file) - smooth_filename = "%s-2Dsmooth" % (png_file) - trend_filename = "%s-2Dtrend" % (png_file) - avg = average(disk_perf[pos]) - f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) - pos = pos +1 + raw_filename = "%s-2Draw" % (png_file) + smooth_filename = "%s-2Dsmooth" % (png_file) + trend_filename = "%s-2Dtrend" % (png_file) + avg = average(disk_perf[pos]) + f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) + pos = pos +1 # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: @@ -120,11 +125,11 @@ def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnup filename=gnuplot_output_dir+'mymath'; temporary_files.append(filename) f=open(filename,'a') - f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) + f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) f.close() def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir): - if verbose: print "Processing data file 2/2" + if verbose: print("Processing data file 2/2") temp_files=[] pos=0 @@ -152,7 +157,7 @@ def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_ end_time=max_time if end_time == -1: end_time="infinite" - if verbose: print "Processing data file 1/2 with %s 1: - if verbose: print " |-> Rendering comparing traces" + if verbose: print(" |-> Rendering comparing traces") os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir) - if verbose: print " |-> Rendering math traces" + if verbose: print(" |-> Rendering math traces") os.system("cd %s; gnuplot mymath" % gnuplot_output_dir) - if verbose: print " |-> Rendering 2D & 3D traces" + if verbose: print(" |-> Rendering 2D & 3D traces") os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir) name_of_directory="the current" if gnuplot_output_dir != "./": name_of_directory=gnuplot_output_dir - print "\nRendering traces are available in %s directory" % name_of_directory + print("\nRendering traces are available in %s directory" % name_of_directory) global keep_temp_files keep_temp_files=False except: - print "Could not run gnuplot on mymath or mygraph !\n" + print("Could not run gnuplot on mymath or mygraph !\n") sys.exit(1); def print_help(): - print 'fio2gnuplot -ghbiodvk -t -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>' - print - print '-h --help : Print this help' - print '-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files' - print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files' - print '-i or --iops : A predefined pattern for selecting *_iops.log files' - print '-g or --gnuplot : Render gnuplot traces before exiting' - print '-o or --outputfile <file> : The basename for gnuplot traces' - print ' - Basename is set with the pattern if defined' - print '-d or --outputdir <dir> : The directory where gnuplot shall render files' - print '-t or --title <title> : The title of the gnuplot traces' - print ' - Title is set with the block size detected in fio traces' - print '-G or --Global <type> : Search for <type> in .global files match by a pattern' - print ' - Available types are : min, max, avg, stddev' - print ' - The .global extension is added automatically to the pattern' - print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)' - print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)' - print '-v or --verbose : Increasing verbosity' - print '-k or --keep : Keep all temporary files from gnuplot\'s output dir' + print('fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>') + print() + print('-h --help : Print this help') + print('-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files') + print('-b or --bandwidth : A predefined pattern for selecting *_bw.log files') + print('-i or --iops : A predefined pattern for selecting *_iops.log files') + print('-g or --gnuplot : Render gnuplot traces before exiting') + print('-o or --outputfile <file> : The basename for gnuplot traces') + print(' - Basename is set with the pattern if defined') + print('-d or --outputdir <dir> : The directory where gnuplot shall render files') + print('-t or --title <title> : The title of the gnuplot traces') + print(' - Title is set with the block size detected in fio traces') + print('-G or --Global <type> : Search for <type> in .global files match by a pattern') + print(' - Available types are : min, max, avg, stddev') + print(' - The .global extension is added automatically to the pattern') + print('-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)') + print('-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)') + print('-v or --verbose : Increasing verbosity') + print('-k or --keep : Keep all temporary files from gnuplot\'s output dir') def main(argv): mode='unknown' @@ -401,126 +406,126 @@ def main(argv): force_keep_temp_files=False if not os.path.isfile(gpm_dir+'math.gpm'): - gpm_dir="/usr/local/share/fio/" - if not os.path.isfile(gpm_dir+'math.gpm'): - print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n" - sys.exit(3) + gpm_dir="/usr/local/share/fio/" + if not os.path.isfile(gpm_dir+'math.gpm'): + print("Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n") + sys.exit(3) try: - opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) + opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) except getopt.GetoptError: - print "Error: One of the options passed to the cmdline was not supported" - print "Please fix your command line or read the help (-h option)" - sys.exit(2) + print("Error: One of the options passed to the cmdline was not supported") + print("Please fix your command line or read the help (-h option)") + sys.exit(2) for opt, arg in opts: - if opt in ("-b", "--bandwidth"): - pattern='*_bw.log' - elif opt in ("-i", "--iops"): - pattern='*_iops.log' - elif opt in ("-v", "--verbose"): - verbose=True - elif opt in ("-k", "--keep"): - #User really wants to keep the temporary files - force_keep_temp_files=True - elif opt in ("-p", "--pattern"): - pattern_set_by_user=True - pattern=arg - pattern=pattern.replace('\\','') - elif opt in ("-o", "--outputfile"): - gnuplot_output_filename=arg - elif opt in ("-d", "--outputdir"): - gnuplot_output_dir=arg - if not gnuplot_output_dir.endswith('/'): - gnuplot_output_dir=gnuplot_output_dir+'/' - if not os.path.exists(gnuplot_output_dir): - os.makedirs(gnuplot_output_dir) - elif opt in ("-t", "--title"): - title=arg - elif opt in ("-m", "--min_time"): - min_time=arg - elif opt in ("-M", "--max_time"): - max_time=arg - elif opt in ("-g", "--gnuplot"): - run_gnuplot=True - elif opt in ("-G", "--Global"): - parse_global=True - global_search=arg - elif opt in ("-h", "--help"): - print_help() - sys.exit(1) + if opt in ("-b", "--bandwidth"): + pattern='*_bw.log' + elif opt in ("-i", "--iops"): + pattern='*_iops.log' + elif opt in ("-v", "--verbose"): + verbose=True + elif opt in ("-k", "--keep"): + #User really wants to keep the temporary files + force_keep_temp_files=True + elif opt in ("-p", "--pattern"): + pattern_set_by_user=True + pattern=arg + pattern=pattern.replace('\\','') + elif opt in ("-o", "--outputfile"): + gnuplot_output_filename=arg + elif opt in ("-d", "--outputdir"): + gnuplot_output_dir=arg + if not gnuplot_output_dir.endswith('/'): + gnuplot_output_dir=gnuplot_output_dir+'/' + if not os.path.exists(gnuplot_output_dir): + os.makedirs(gnuplot_output_dir) + elif opt in ("-t", "--title"): + title=arg + elif opt in ("-m", "--min_time"): + min_time=arg + elif opt in ("-M", "--max_time"): + max_time=arg + elif opt in ("-g", "--gnuplot"): + run_gnuplot=True + elif opt in ("-G", "--Global"): + parse_global=True + global_search=arg + elif opt in ("-h", "--help"): + print_help() + sys.exit(1) # Adding .global extension to the file if parse_global==True: - if not gnuplot_output_filename.endswith('.global'): - pattern = pattern+'.global' + if not gnuplot_output_filename.endswith('.global'): + pattern = pattern+'.global' fio_data_file=find_file('.',pattern) if len(fio_data_file) == 0: - print "No log file found with pattern %s!" % pattern - # Try numjob log file format if per_numjob_logs=1 - if (pattern == '*_bw.log'): - fio_data_file=find_file('.','*_bw.*.log') - if (pattern == '*_iops.log'): - fio_data_file=find_file('.','*_iops.*.log') - if len(fio_data_file) == 0: - sys.exit(1) - else: - print "Using log file per job format instead" + print("No log file found with pattern %s!" % pattern) + # Try numjob log file format if per_numjob_logs=1 + if (pattern == '*_bw.log'): + fio_data_file=find_file('.','*_bw.*.log') + if (pattern == '*_iops.log'): + fio_data_file=find_file('.','*_iops.*.log') + if len(fio_data_file) == 0: + sys.exit(1) + else: + print("Using log file per job format instead") else: - print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern) + print("%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)) fio_data_file=sorted(fio_data_file, key=str.lower) for file in fio_data_file: - print ' |-> %s' % file - if "_bw." in file : - mode="Bandwidth (KB/sec)" - if "_iops." in file : - mode="IO per Seconds (IO/sec)" + print(' |-> %s' % file) + if "_bw.log" in file : + mode="Bandwidth (KB/sec)" + if "_iops.log" in file : + mode="IO per Seconds (IO/sec)" if (title == 'No title') and (mode != 'unknown'): - if "Bandwidth" in mode: - title='Bandwidth benchmark with %d fio results' % len(fio_data_file) - if "IO" in mode: - title='IO benchmark with %d fio results' % len(fio_data_file) + if "Bandwidth" in mode: + title='Bandwidth benchmark with %d fio results' % len(fio_data_file) + if "IO" in mode: + title='IO benchmark with %d fio results' % len(fio_data_file) - print + print() #We need to adjust the output filename regarding the pattern required by the user if (pattern_set_by_user == True): - gnuplot_output_filename=pattern - # As we do have some regexp in the pattern, let's make this simpliest - # We do remove the simpliest parts of the expression to get a clear file name - gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') - gnuplot_output_filename=gnuplot_output_filename.replace('*','-') - gnuplot_output_filename=gnuplot_output_filename.replace('--','-') - gnuplot_output_filename=gnuplot_output_filename.replace('.log','') - # Insure that we don't have any starting or trailing dash to the filename - gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename - gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename - if (gnuplot_output_filename == ''): - gnuplot_output_filename='default' + gnuplot_output_filename=pattern + # As we do have some glob in the pattern, let's make this simplest + # We do remove the simplest parts of the expression to get a clear file name + gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') + gnuplot_output_filename=gnuplot_output_filename.replace('*','-') + gnuplot_output_filename=gnuplot_output_filename.replace('--','-') + gnuplot_output_filename=gnuplot_output_filename.replace('.log','') + # Insure that we don't have any starting or trailing dash to the filename + gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename + gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename + if (gnuplot_output_filename == ''): + gnuplot_output_filename='default' if parse_global==True: - parse_global_files(fio_data_file, global_search) + parse_global_files(fio_data_file, global_search) else: - blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) - title="%s @ Blocksize = %dK" % (title,blk_size/1024) - compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) - compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) - generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) - - if (run_gnuplot==True): - render_gnuplot(fio_data_file, gnuplot_output_dir) - - # Shall we clean the temporary files ? - if keep_temp_files==False and force_keep_temp_files==False: - # Cleaning temporary files - if verbose: print "Cleaning temporary files" - for f in enumerate(temporary_files): - if verbose: print " -> %s"%f[1] - try: - os.remove(f[1]) - except: - True + blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) + title="%s @ Blocksize = %dK" % (title,blk_size/1024) + compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) + compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + + if (run_gnuplot==True): + render_gnuplot(fio_data_file, gnuplot_output_dir) + + # Shall we clean the temporary files ? + if keep_temp_files==False and force_keep_temp_files==False: + # Cleaning temporary files + if verbose: print("Cleaning temporary files") + for f in enumerate(temporary_files): + if verbose: print(" -> %s"%f[1]) + try: + os.remove(f[1]) + except: + True #Main if __name__ == "__main__": diff --git a/fio-genplots/lib/graph2D.gpm b/fio-genplots/lib/graph2D.gpm new file mode 100644 index 0000000..769b754 --- /dev/null +++ b/fio-genplots/lib/graph2D.gpm @@ -0,0 +1,55 @@ +# This Gnuplot file has been generated by eNovance + +needed_args = 8 +if (exists("ARGC") && ARGC >= needed_args) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" >= needed_args) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ + ARG6 = "$5"; \ + ARG7 = "$6"; \ + ARG8 = "$7"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +avg_num = ARG8 + 0 +avg_str = sprintf("%g", avg_num) + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' +#set terminal x11 + +#Preparing Axes +#set logscale x +set ytics axis out auto +#set data style lines +set key top left reverse +set xlabel "Time (Seconds)" +set ylabel ARG5 +set xrange [0:] +set yrange [0:] + +#Set Color style +#set palette rgbformulae 22,9,23 +#set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" + +plot ARG2 using 2:3 with linespoints title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' + +set output ARG6 . '.png' +plot ARG2 using 2:3 smooth csplines title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' + +set output ARG7 . '.png' +plot ARG2 using 2:3 smooth bezier title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str .')' + +#pause -1 +#The End diff --git a/fio-genplots/lib/graph3D.gpm b/fio-genplots/lib/graph3D.gpm new file mode 100644 index 0000000..ac2cdf6 --- /dev/null +++ b/fio-genplots/lib/graph3D.gpm @@ -0,0 +1,95 @@ +# This Gnuplot file has been generated by eNovance + +needed_args = 5 +if (exists("ARGC") && ARGC >= needed_args) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" >= needed_args) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' +#set terminal x11 +#3D Config +set isosamples 30 +set hidden3d +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder + +#Preparing Axes +#set logscale x +set ytics axis out 0,1 +#set data style lines +set grid back +set key top left reverse +set ylabel "Disk" +set xlabel "Time (Seconds)" +set zlabel ARG5 +set cbrange [0:] +set zrange [0:] + +#Set Color style +#set palette rgbformulae 22,9,23 +set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 + +#Multiploting +set multiplot + +#Top Left View +set size 0.5,0.5 +set view 64,216 +set origin 0,0.5 +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Top Right View +set size 0.5,0.5 +set origin 0.5,0.5 +set view 90,0 +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Bottom Right View +set size 0.5,0.5 +set origin 0.5,0 +set view 63,161 +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Bottom Left View +set size 0.5,0.5 +set origin 0,0 +set pm3d map +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Unsetting multiplotting +unset multiplot +#pause -1 + +#Preparing 3D Interactive view +set mouse +set terminal png size 1024,768 +set output ARG4 . '-3D.png' + +#set term x11 +set view 64,216 +set origin 0,0 +set size 1,1 +set pm3d at bs solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#pause -1 +#The End diff --git a/fio-genplots/lib/math.gpm b/fio-genplots/lib/math.gpm new file mode 100644 index 0000000..0a2aff5 --- /dev/null +++ b/fio-genplots/lib/math.gpm @@ -0,0 +1,42 @@ +# This Gnuplot file has been generated by eNovance +if (exists("ARGC") && ARGC > 5) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" > 5) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ + ARG6 = "$5"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +avg_num = ARG6 + 0 +avg_str = sprintf("%g", avg_num) + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' + +set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 +set style fill transparent solid 0.9 noborder +set auto x +set ylabel ARG5 +set xlabel "Disk" +set yrange [0:] +set style data histogram +set style histogram cluster gap 1 +set style fill solid border -1 +set boxwidth 2 +#set xtic rotate by -10 scale 10 font ",8" +set bmargin 3 +set xtics axis out +set xtic rotate by 45 scale 0 font ",8" autojustify +set xtics offset 0,-1 border -5,1,5 +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" +plot ARG2 using 2:xtic(1) ti col, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' diff --git a/fio-genplots/plot.sh b/fio-genplots/plot.sh new file mode 100644 index 0000000..0190d22 --- /dev/null +++ b/fio-genplots/plot.sh @@ -0,0 +1,99 @@ +#!/bin/sh +set -eu + +echo "Plotting data..." +PLOTNAME=${PLOTNAME:-FIO} +OUTDIR=${OUTDIR:-plots} +mkdir -p "$OUTDIR" + +# Expand globs in POSIX and ignore non-matches +find_logs() { + for pat in "$@"; do + # unquoted to allow globbing + for f in $pat; do + case "$f" in + *'*'*|*'?'*|*'['*']'*) + : ;; # still a pattern -> no matches + *) + [ -f "$f" ] && printf '%s\n' "$f" + ;; + esac + done + done +} + +BW_LOGS="$(find_logs '*_bw.log' '*_bw.*.log')" +IOPS_LOGS="$(find_logs '*_iops.log' '*_iops.*.log')" +LAT_LOGS="$(find_logs '*_lat*.log' '*_clat*.log' '*_slat*.log')" + +WORK=".norm" +rm -rf "$WORK"; mkdir "$WORK" + +# Normalize time to integer milliseconds from t0 +normalize() { + in="$1"; out="$WORK/$(basename "$1")" + # first numeric timestamp + first="$(awk 'NF{print $1; exit}' "$in" 2>/dev/null || echo 0)" + awk -v t0="$first" 'BEGIN{OFS="\t"; mode=0} + NR==1{ + if (t0>=1000000000000) mode=1; # epoch ms + else if (t0>=1000000000) mode=2; # epoch s + else mode=3; # relative seconds + } + { + t=$1 + if (mode==1) t = t - t0; # already ms + else t = (t - t0) * 1000; # seconds -> ms + $1 = int(t + 0.5) # force integer ms + print + }' "$in" > "$out" +} + +# Normalize all logs we found +for f in $BW_LOGS $IOPS_LOGS $LAT_LOGS; do + [ -n "${f:-}" ] && normalize "$f" || : +done + +# best-effort bs hint for titles +BS_HINT="" +if [ -n "${JOBFILES:-}" ] && [ -f "$JOBFILES" ]; then + BS_HINT="$(awk -F= '$0 ~ /(^|[,;[:space:]])bs[[:space:]]*=/ {print $2; exit}' "$JOBFILES" 2>/dev/null || true)" +fi +[ -z "$BS_HINT" ] && [ -f fio.output ] && BS_HINT="$( + awk -F'[= ,]' '$0 ~ / bs=/ {for(i=1;i<=NF;i++) if($i=="bs"){print $(i+1); exit}}' fio.output 2>/dev/null || true +)" + +cd "$WORK" + +# detect optional flags +HAS_NO3D="" +if fio2gnuplot -h 2>&1 | grep -Eq '(^|[[:space:]])--no-3d([[:space:]]|,|$)'; then + HAS_NO3D="--no-3d" +fi +HAS_LAT="" +if fio2gnuplot -h 2>&1 | grep -Eq '(^|[[:space:]])-l([[:space:]]|,|$)'; then + HAS_LAT="-l" +fi + +GP_OPTS="-g" +[ -n "$HAS_NO3D" ] && GP_OPTS="$GP_OPTS $HAS_NO3D" + +ttl_bs="" +[ -n "$BS_HINT" ] && ttl_bs=" (bs=$BS_HINT)" + +if [ -n "$BW_LOGS" ]; then + echo "$(printf '%s\n' $BW_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_bw*'" + fio2gnuplot $GP_OPTS -b -t "${PLOTNAME} - Bandwidth${ttl_bs}" -d "../$OUTDIR" || true +fi + +if [ -n "$IOPS_LOGS" ]; then + echo "$(printf '%s\n' $IOPS_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_iops*'" + fio2gnuplot $GP_OPTS -i -t "${PLOTNAME} - IOPS${ttl_bs}" -d "../$OUTDIR" || true +fi + +if [ -n "$LAT_LOGS" ] && [ -n "$HAS_LAT" ]; then + echo "$(printf '%s\n' $LAT_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_lat|clat|slat*'" + fio2gnuplot $GP_OPTS $HAS_LAT -t "${PLOTNAME} - Latency${ttl_bs}" -d "../$OUTDIR" || true +fi + +cd .. diff --git a/fio-plotserve/Dockerfile b/fio-plotserve/Dockerfile index db15acb..046d955 100644 --- a/fio-plotserve/Dockerfile +++ b/fio-plotserve/Dockerfile @@ -1,8 +1,10 @@ -FROM python:2.7 -MAINTAINER <Ryan Wallner ryan.wallner@portworx.com> +# syntax=docker/dockerfile:1.6 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +LABEL maintainer="Ryan Wallner <ryan.wallner@portworx.com>" -VOLUME /tmp/fio-data -WORKDIR /tmp/fio-data +# Static file server for /tmp/fio-data on port 8000 EXPOSE 8000 -CMD ["python", "-m", "SimpleHTTPServer", "8000"] +WORKDIR /tmp/fio-data +CMD ["python3","-m","http.server","8000"] diff --git a/fio-tool/Dockerfile b/fio-tool/Dockerfile index 91a1228..92f7612 100644 --- a/fio-tool/Dockerfile +++ b/fio-tool/Dockerfile @@ -1,8 +1,13 @@ -FROM base-fiotools -MAINTAINER <Ryan Wallner ryan.wallner@portworx.com> +# syntax=docker/dockerfile:1.6 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +LABEL maintainer="Ryan Wallner <ryan.wallner@portworx.com>" -VOLUME /tmp/fio-data +# Runs the user-provided job(s) +# (Existing entry script expects JOBFILES, and writes fio.output) +# See run.sh in repo: runs "fio --output=fio.output $JOBFILES" ADD run.sh /opt/run.sh RUN chmod +x /opt/run.sh + WORKDIR /tmp/fio-data -CMD ["/opt/run.sh"] +CMD ["/opt/run.sh"] diff --git a/fio-tool/run.sh b/fio-tool/run.sh index d330a4a..66f0ba5 100644 --- a/fio-tool/run.sh +++ b/fio-tool/run.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/sh [ -z "$JOBFILES" ] && echo "Need to set JOBFILES" && exit 1; -echo "Running $JOBFILES" +echo "Found jobs: $JOBFILES" # We really want no old data in here except the fio script mv /tmp/fio-data/*.fio /tmp/ @@ -9,8 +9,6 @@ rm -rf /tmp/fio-data/* mv /tmp/*fio /tmp/fio-data/ if [ ! -z "$REMOTEFILES" ]; then - # We really want no old data in here - rm -rf /tmp/fio-data/* IFS=' ' echo "Gathering remote files..." for file in $REMOTEFILES; do @@ -18,4 +16,8 @@ if [ ! -z "$REMOTEFILES" ]; then done fi -fio --output=fio.output $JOBFILES +#!/bin/sh + +echo "Running FIO job $JOBFILES" +fio $JOBFILES 2>&1 | tee fio.output + diff --git a/fiotools-aio/Dockerfile b/fiotools-aio/Dockerfile index e8b2c82..a39943c 100644 --- a/fiotools-aio/Dockerfile +++ b/fiotools-aio/Dockerfile @@ -1,27 +1,23 @@ -FROM base-fiotools -MAINTAINER <Ryan Wallner ryan.wallner@portworx.com> +# syntax=docker/dockerfile:1.6 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +LABEL maintainer="Ryan Wallner <ryan.wallner@portworx.com>" -# Check ENV vars -ADD check.sh /opt/check.sh -RUN chmod +x /opt/check.sh -# Run the FIO Job -VOLUME /tmp/fio-data -ADD run.sh /opt/run.sh -RUN chmod +x /opt/run.sh -WORKDIR /tmp/fio-data - -# Generate plots -ADD plot.sh /opt/plot.sh -RUN chmod +x /opt/plot.sh +# All-in-one: check -> run fio -> plot -> serve on 8000 +# set owner and mode when copying +COPY run.sh /opt/run.sh +COPY plot.sh /opt/plot.sh +COPY check.sh /opt/check.sh +COPY runall.sh /opt/runall.sh -# Add all-in-one script -ADD runall.sh /opt/runall.sh -RUN chmod +x /opt/runall.sh +RUN chmod +x /opt/*.sh -# Add modified script to y axis is set accurately -ADD lib/fio2gnuplot /usr/bin/fio2gnuplot -RUN chmod +x /usr/bin/fio2gnuplot +# Fio2gnuplot helper (already in repo) +# If this file is located at the repo root as before, this copies it in. +COPY lib/fio2gnuplot /usr/local/bin/fio2gnuplot +RUN chmod +x /usr/local/bin/fio2gnuplot || true EXPOSE 8000 -CMD ["/opt/runall.sh"] +WORKDIR /tmp/fio-data +CMD ["/opt/runall.sh"] diff --git a/fiotools-aio/check.sh b/fiotools-aio/check.sh index e507e66..ab5ecd6 100644 --- a/fiotools-aio/check.sh +++ b/fiotools-aio/check.sh @@ -1,10 +1,10 @@ -#!/bin/bash +#!/bin/sh [ -z "$JOBFILES" ] && echo "Need to set JOBFILES" && exit 1; echo "Found jobs: $JOBFILES" [ -z "$PLOTNAME" ] && echo "Need to set PLOTOPTS" && exit 1; -echo "Received $PLOTNAME" +echo "Received Plot Name: $PLOTNAME" # We really want no old data in here rm -rf /tmp/fio-data/* diff --git a/fiotools-aio/lib/fio2gnuplot b/fiotools-aio/lib/fio2gnuplot old mode 100644 new mode 100755 index 50e9d29..ce3ca2c --- a/fiotools-aio/lib/fio2gnuplot +++ b/fiotools-aio/lib/fio2gnuplot @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# Note: this script is python2 and python3 compatible. # # Copyright (C) 2013 eNovance SAS <licensing@enovance.com> # Author: Erwan Velu <erwan@enovance.com> @@ -17,8 +18,10 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import absolute_import +from __future__ import print_function import os import fnmatch import sys @@ -26,27 +29,29 @@ import getopt import re import math import shutil +from six.moves import map +from six.moves import range def find_file(path, pattern): fio_data_file=[] # For all the local files for file in os.listdir(path): - # If the file math the regexp - if fnmatch.fnmatch(file, pattern): - # Let's consider this file - fio_data_file.append(file) + # If the file matches the glob + if fnmatch.fnmatch(file, pattern): + # Let's consider this file + fio_data_file.append(file) return fio_data_file def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): - if verbose: print "Generating rendering scripts" + if verbose: print("Generating rendering scripts") filename=gnuplot_output_dir+'mygraph' temporary_files.append(filename) f=open(filename,'w') # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: - f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) + f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) # Setting up the compare files that will be plot later compare=open(gnuplot_output_dir + 'compare.gnuplot','w') @@ -88,10 +93,10 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); - pos=0 - # Let's create a temporary file for each selected fio file - for file in fio_data_file: - tmp_filename = "gnuplot_temp_file.%d" % pos + pos=0 + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "gnuplot_temp_file.%d" % pos # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: @@ -101,12 +106,12 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos])) png_file=file.replace('.log','') - raw_filename = "%s-2Draw" % (png_file) - smooth_filename = "%s-2Dsmooth" % (png_file) - trend_filename = "%s-2Dtrend" % (png_file) - avg = average(disk_perf[pos]) - f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) - pos = pos +1 + raw_filename = "%s-2Draw" % (png_file) + smooth_filename = "%s-2Dsmooth" % (png_file) + trend_filename = "%s-2Dtrend" % (png_file) + avg = average(disk_perf[pos]) + f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) + pos = pos +1 # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1: @@ -120,11 +125,11 @@ def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnup filename=gnuplot_output_dir+'mymath'; temporary_files.append(filename) f=open(filename,'a') - f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) + f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) f.close() def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir): - if verbose: print "Processing data file 2/2" + if verbose: print("Processing data file 2/2") temp_files=[] pos=0 @@ -152,7 +157,7 @@ def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_ end_time=max_time if end_time == -1: end_time="infinite" - if verbose: print "Processing data file 1/2 with %s<time<%s" % (min_time,end_time) + if verbose: print("Processing data file 1/2 with %s<time<%s" % (min_time,end_time)) files=[] temp_outfile=[] blk_size=0 @@ -193,13 +198,13 @@ def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_ # Index will be used to remember what file was featuring what value index=index+1 - time, perf, x, block_size = line[1] + time, perf, x, block_size = line[1][:4] if (blk_size == 0): try: blk_size=int(block_size) except: - print "Error while reading the following line :" - print line + print("Error while reading the following line :") + print(line) sys.exit(1); # We ignore the first 500msec as it doesn't seems to be part of the real benchmark @@ -225,7 +230,7 @@ def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_ return blk_size def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): - if verbose: print "Computing Maths" + if verbose: print("Computing Maths") global_min=[] global_max=[] average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w') @@ -243,14 +248,14 @@ def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir max_file.write('DiskName %s\n'% mode) average_file.write('DiskName %s\n'% mode) stddev_file.write('DiskName %s\n'% mode ) - for disk in xrange(len(fio_data_file)): + for disk in range(len(fio_data_file)): # print disk_perf[disk] - min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) - max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) - average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) - stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) avg = average(disk_perf[disk]) - variance = map(lambda x: (x - avg)**2, disk_perf[disk]) + variance = [(x - avg)**2 for x in disk_perf[disk]] standard_deviation = math.sqrt(average(variance)) # print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation) average_file.write('%d %d\n' % (disk, avg)) @@ -264,7 +269,7 @@ def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir global_disk_perf = sum(disk_perf, []) avg = average(global_disk_perf) - variance = map(lambda x: (x - avg)**2, global_disk_perf) + variance = [(x - avg)**2 for x in global_disk_perf] standard_deviation = math.sqrt(average(variance)) global_file.write('min=%.2f\n' % min(global_disk_perf)) @@ -331,52 +336,52 @@ def parse_global_files(fio_data_file, global_search): max_file=file # Let's print the avg output if global_search == "avg": - print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file) + print("Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)) else: - print "Global search %s is not yet implemented\n" % global_search + print("Global search %s is not yet implemented\n" % global_search) def render_gnuplot(fio_data_file, gnuplot_output_dir): - print "Running gnuplot Rendering" + print("Running gnuplot Rendering") try: # Let's render all the compared files if some if len(fio_data_file) > 1: - if verbose: print " |-> Rendering comparing traces" + if verbose: print(" |-> Rendering comparing traces") os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir) - if verbose: print " |-> Rendering math traces" + if verbose: print(" |-> Rendering math traces") os.system("cd %s; gnuplot mymath" % gnuplot_output_dir) - if verbose: print " |-> Rendering 2D & 3D traces" + if verbose: print(" |-> Rendering 2D & 3D traces") os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir) name_of_directory="the current" if gnuplot_output_dir != "./": name_of_directory=gnuplot_output_dir - print "\nRendering traces are available in %s directory" % name_of_directory + print("\nRendering traces are available in %s directory" % name_of_directory) global keep_temp_files keep_temp_files=False except: - print "Could not run gnuplot on mymath or mygraph !\n" + print("Could not run gnuplot on mymath or mygraph !\n") sys.exit(1); def print_help(): - print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>' - print - print '-h --help : Print this help' - print '-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files' - print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files' - print '-i or --iops : A predefined pattern for selecting *_iops.log files' - print '-g or --gnuplot : Render gnuplot traces before exiting' - print '-o or --outputfile <file> : The basename for gnuplot traces' - print ' - Basename is set with the pattern if defined' - print '-d or --outputdir <dir> : The directory where gnuplot shall render files' - print '-t or --title <title> : The title of the gnuplot traces' - print ' - Title is set with the block size detected in fio traces' - print '-G or --Global <type> : Search for <type> in .global files match by a pattern' - print ' - Available types are : min, max, avg, stddev' - print ' - The .global extension is added automatically to the pattern' - print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)' - print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)' - print '-v or --verbose : Increasing verbosity' - print '-k or --keep : Keep all temporary files from gnuplot\'s output dir' + print('fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>') + print() + print('-h --help : Print this help') + print('-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files') + print('-b or --bandwidth : A predefined pattern for selecting *_bw.log files') + print('-i or --iops : A predefined pattern for selecting *_iops.log files') + print('-g or --gnuplot : Render gnuplot traces before exiting') + print('-o or --outputfile <file> : The basename for gnuplot traces') + print(' - Basename is set with the pattern if defined') + print('-d or --outputdir <dir> : The directory where gnuplot shall render files') + print('-t or --title <title> : The title of the gnuplot traces') + print(' - Title is set with the block size detected in fio traces') + print('-G or --Global <type> : Search for <type> in .global files match by a pattern') + print(' - Available types are : min, max, avg, stddev') + print(' - The .global extension is added automatically to the pattern') + print('-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)') + print('-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)') + print('-v or --verbose : Increasing verbosity') + print('-k or --keep : Keep all temporary files from gnuplot\'s output dir') def main(argv): mode='unknown' @@ -401,126 +406,126 @@ def main(argv): force_keep_temp_files=False if not os.path.isfile(gpm_dir+'math.gpm'): - gpm_dir="/usr/local/share/fio/" - if not os.path.isfile(gpm_dir+'math.gpm'): - print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n" - sys.exit(3) + gpm_dir="/usr/local/share/fio/" + if not os.path.isfile(gpm_dir+'math.gpm'): + print("Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n") + sys.exit(3) try: - opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) + opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) except getopt.GetoptError: - print "Error: One of the options passed to the cmdline was not supported" - print "Please fix your command line or read the help (-h option)" - sys.exit(2) + print("Error: One of the options passed to the cmdline was not supported") + print("Please fix your command line or read the help (-h option)") + sys.exit(2) for opt, arg in opts: - if opt in ("-b", "--bandwidth"): - pattern='*_bw.log' - elif opt in ("-i", "--iops"): - pattern='*_iops.log' - elif opt in ("-v", "--verbose"): - verbose=True - elif opt in ("-k", "--keep"): - #User really wants to keep the temporary files - force_keep_temp_files=True - elif opt in ("-p", "--pattern"): - pattern_set_by_user=True - pattern=arg - pattern=pattern.replace('\\','') - elif opt in ("-o", "--outputfile"): - gnuplot_output_filename=arg - elif opt in ("-d", "--outputdir"): - gnuplot_output_dir=arg - if not gnuplot_output_dir.endswith('/'): - gnuplot_output_dir=gnuplot_output_dir+'/' - if not os.path.exists(gnuplot_output_dir): - os.makedirs(gnuplot_output_dir) - elif opt in ("-t", "--title"): - title=arg - elif opt in ("-m", "--min_time"): - min_time=arg - elif opt in ("-M", "--max_time"): - max_time=arg - elif opt in ("-g", "--gnuplot"): - run_gnuplot=True - elif opt in ("-G", "--Global"): - parse_global=True - global_search=arg - elif opt in ("-h", "--help"): - print_help() - sys.exit(1) + if opt in ("-b", "--bandwidth"): + pattern='*_bw.log' + elif opt in ("-i", "--iops"): + pattern='*_iops.log' + elif opt in ("-v", "--verbose"): + verbose=True + elif opt in ("-k", "--keep"): + #User really wants to keep the temporary files + force_keep_temp_files=True + elif opt in ("-p", "--pattern"): + pattern_set_by_user=True + pattern=arg + pattern=pattern.replace('\\','') + elif opt in ("-o", "--outputfile"): + gnuplot_output_filename=arg + elif opt in ("-d", "--outputdir"): + gnuplot_output_dir=arg + if not gnuplot_output_dir.endswith('/'): + gnuplot_output_dir=gnuplot_output_dir+'/' + if not os.path.exists(gnuplot_output_dir): + os.makedirs(gnuplot_output_dir) + elif opt in ("-t", "--title"): + title=arg + elif opt in ("-m", "--min_time"): + min_time=arg + elif opt in ("-M", "--max_time"): + max_time=arg + elif opt in ("-g", "--gnuplot"): + run_gnuplot=True + elif opt in ("-G", "--Global"): + parse_global=True + global_search=arg + elif opt in ("-h", "--help"): + print_help() + sys.exit(1) # Adding .global extension to the file if parse_global==True: - if not gnuplot_output_filename.endswith('.global'): - pattern = pattern+'.global' + if not gnuplot_output_filename.endswith('.global'): + pattern = pattern+'.global' fio_data_file=find_file('.',pattern) if len(fio_data_file) == 0: - print "No log file found with pattern %s!" % pattern - # Try numjob log file format if per_numjob_logs=1 - if (pattern == '*_bw.log'): - fio_data_file=find_file('.','*_bw.*.log') - if (pattern == '*_iops.log'): - fio_data_file=find_file('.','*_iops.*.log') - if len(fio_data_file) == 0: - sys.exit(1) - else: - print "Using log file per job format instead" + print("No log file found with pattern %s!" % pattern) + # Try numjob log file format if per_numjob_logs=1 + if (pattern == '*_bw.log'): + fio_data_file=find_file('.','*_bw.*.log') + if (pattern == '*_iops.log'): + fio_data_file=find_file('.','*_iops.*.log') + if len(fio_data_file) == 0: + sys.exit(1) + else: + print("Using log file per job format instead") else: - print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern) + print("%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)) fio_data_file=sorted(fio_data_file, key=str.lower) for file in fio_data_file: - print ' |-> %s' % file - if "_bw." in file : - mode="Bandwidth (KB/sec)" - if "_iops." in file : - mode="IO per Seconds (IO/sec)" + print(' |-> %s' % file) + if "_bw.log" in file : + mode="Bandwidth (KB/sec)" + if "_iops.log" in file : + mode="IO per Seconds (IO/sec)" if (title == 'No title') and (mode != 'unknown'): - if "Bandwidth" in mode: - title='Bandwidth benchmark with %d fio results' % len(fio_data_file) - if "IO" in mode: - title='IO benchmark with %d fio results' % len(fio_data_file) + if "Bandwidth" in mode: + title='Bandwidth benchmark with %d fio results' % len(fio_data_file) + if "IO" in mode: + title='IO benchmark with %d fio results' % len(fio_data_file) - print + print() #We need to adjust the output filename regarding the pattern required by the user if (pattern_set_by_user == True): - gnuplot_output_filename=pattern - # As we do have some regexp in the pattern, let's make this simpliest - # We do remove the simpliest parts of the expression to get a clear file name - gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') - gnuplot_output_filename=gnuplot_output_filename.replace('*','-') - gnuplot_output_filename=gnuplot_output_filename.replace('--','-') - gnuplot_output_filename=gnuplot_output_filename.replace('.log','') - # Insure that we don't have any starting or trailing dash to the filename - gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename - gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename - if (gnuplot_output_filename == ''): - gnuplot_output_filename='default' + gnuplot_output_filename=pattern + # As we do have some glob in the pattern, let's make this simplest + # We do remove the simplest parts of the expression to get a clear file name + gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') + gnuplot_output_filename=gnuplot_output_filename.replace('*','-') + gnuplot_output_filename=gnuplot_output_filename.replace('--','-') + gnuplot_output_filename=gnuplot_output_filename.replace('.log','') + # Insure that we don't have any starting or trailing dash to the filename + gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename + gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename + if (gnuplot_output_filename == ''): + gnuplot_output_filename='default' if parse_global==True: - parse_global_files(fio_data_file, global_search) + parse_global_files(fio_data_file, global_search) else: - blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) - title="%s @ Blocksize = %dK" % (title,blk_size/1024) - compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) - compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) - generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) - - if (run_gnuplot==True): - render_gnuplot(fio_data_file, gnuplot_output_dir) - - # Shall we clean the temporary files ? - if keep_temp_files==False and force_keep_temp_files==False: - # Cleaning temporary files - if verbose: print "Cleaning temporary files" - for f in enumerate(temporary_files): - if verbose: print " -> %s"%f[1] - try: - os.remove(f[1]) - except: - True + blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) + title="%s @ Blocksize = %dK" % (title,blk_size/1024) + compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) + compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + + if (run_gnuplot==True): + render_gnuplot(fio_data_file, gnuplot_output_dir) + + # Shall we clean the temporary files ? + if keep_temp_files==False and force_keep_temp_files==False: + # Cleaning temporary files + if verbose: print("Cleaning temporary files") + for f in enumerate(temporary_files): + if verbose: print(" -> %s"%f[1]) + try: + os.remove(f[1]) + except: + True #Main if __name__ == "__main__": diff --git a/fiotools-aio/lib/fio2gnuplot.bak b/fiotools-aio/lib/fio2gnuplot.bak new file mode 100644 index 0000000..50e9d29 --- /dev/null +++ b/fiotools-aio/lib/fio2gnuplot.bak @@ -0,0 +1,527 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013 eNovance SAS <licensing@enovance.com> +# Author: Erwan Velu <erwan@enovance.com> +# +# The license below covers all files distributed with fio unless otherwise +# noted in the file itself. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import fnmatch +import sys +import getopt +import re +import math +import shutil + +def find_file(path, pattern): + fio_data_file=[] + # For all the local files + for file in os.listdir(path): + # If the file math the regexp + if fnmatch.fnmatch(file, pattern): + # Let's consider this file + fio_data_file.append(file) + + return fio_data_file + +def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): + if verbose: print "Generating rendering scripts" + filename=gnuplot_output_dir+'mygraph' + temporary_files.append(filename) + f=open(filename,'w') + + # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) + + # Setting up the compare files that will be plot later + compare=open(gnuplot_output_dir + 'compare.gnuplot','w') + compare.write(''' +set title '%s' +set terminal png size 1280,1024 +set ytics axis out auto +set key top left reverse +set xlabel "Time (Seconds)" +set ylabel '%s' +set yrange [0:] +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" +'''% (title,mode)) + compare.close() + #Copying the common file for all kind of graph (raw/smooth/trend) + compare_raw_filename="compare-%s-2Draw" % (gnuplot_output_filename) + compare_smooth_filename="compare-%s-2Dsmooth" % (gnuplot_output_filename) + compare_trend_filename="compare-%s-2Dtrend" % (gnuplot_output_filename) + + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_raw_filename+".gnuplot") + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_smooth_filename+".gnuplot") + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_trend_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_raw_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_smooth_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_trend_filename+".gnuplot") + + #Setting up a different output filename for each kind of graph + compare_raw=open(gnuplot_output_dir+compare_raw_filename + ".gnuplot",'a') + compare_raw.write("set output '%s.png'\n" % compare_raw_filename) + compare_smooth=open(gnuplot_output_dir+compare_smooth_filename+".gnuplot",'a') + compare_smooth.write("set output '%s.png'\n" % compare_smooth_filename) + compare_trend=open(gnuplot_output_dir+compare_trend_filename+".gnuplot",'a') + compare_trend.write("set output '%s.png'\n" % compare_trend_filename) + + # Let's plot the average value for all the traces + global_disk_perf = sum(disk_perf, []) + global_avg = average(global_disk_perf) + compare_raw.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + + pos=0 + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "gnuplot_temp_file.%d" % pos + + # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + # Adding the plot instruction for each kind of comparing graphs + compare_raw.write(",\\\n'%s' using 2:3 with linespoints title '%s'" % (tmp_filename,fio_data_file[pos])) + compare_smooth.write(",\\\n'%s' using 2:3 smooth csplines title '%s'" % (tmp_filename,fio_data_file[pos])) + compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos])) + + png_file=file.replace('.log','') + raw_filename = "%s-2Draw" % (png_file) + smooth_filename = "%s-2Dsmooth" % (png_file) + trend_filename = "%s-2Dtrend" % (png_file) + avg = average(disk_perf[pos]) + f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) + pos = pos +1 + + # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + os.remove(gnuplot_output_dir+"compare.gnuplot") + compare_raw.close() + compare_smooth.close() + compare_trend.close() + f.close() + +def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnuplot_output_dir,gpm_dir): + filename=gnuplot_output_dir+'mymath'; + temporary_files.append(filename) + f=open(filename,'a') + f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) + f.close() + +def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir): + if verbose: print "Processing data file 2/2" + temp_files=[] + pos=0 + + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir, pos) + temp_files.append(open(tmp_filename,'r')) + pos = pos +1 + + f = open(gnuplot_output_dir+gnuplot_output_filename, "w") + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename) + index=0 + # Let's add some information + for tempfile in temp_files: + f.write("# Disk%d was coming from %s\n" % (index,fio_data_file[index])) + f.write(tempfile.read()) + f.write("\n") + tempfile.close() + index = index + 1 + f.close() + +def average(s): return sum(s) * 1.0 / len(s) + +def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_time): + end_time=max_time + if end_time == -1: + end_time="infinite" + if verbose: print "Processing data file 1/2 with %s<time<%s" % (min_time,end_time) + files=[] + temp_outfile=[] + blk_size=0 + for file in fio_data_file: + files.append(open(file)) + pos = len(files) - 1 + tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir,pos) + temporary_files.append(tmp_filename) + gnuplot_file=open(tmp_filename,'w') + temp_outfile.append(gnuplot_file) + gnuplot_file.write("#Temporary file based on file %s\n" % file) + disk_perf.append([]) + + shall_break = False + while True: + current_line=[] + nb_empty_files=0 + nb_files=len(files) + for myfile in files: + s=myfile.readline().replace(',',' ').split() + if not s: + nb_empty_files+=1 + s="-1, 0, 0, 0".replace(',',' ').split() + + if (nb_empty_files == nb_files): + shall_break=True + break; + + current_line.append(s); + + if shall_break == True: + break + + last_time = -1 + index=-1 + perfs=[] + for line in enumerate(current_line): + # Index will be used to remember what file was featuring what value + index=index+1 + + time, perf, x, block_size = line[1] + if (blk_size == 0): + try: + blk_size=int(block_size) + except: + print "Error while reading the following line :" + print line + sys.exit(1); + + # We ignore the first 500msec as it doesn't seems to be part of the real benchmark + # Time < 500 usually reports BW=0 breaking the min computing + if (min_time == 0): + min_time==0.5 + + # Then we estimate if the data we got is part of the time range we want to plot + if ((float(time)>(float(min_time)*1000)) and ((int(time) < (int(max_time)*1000)) or max_time==-1)): + disk_perf[index].append(int(perf)) + perfs.append("%d %s %s"% (index, time, perf)) + + # If we reach this point, it means that all the traces are coherent + for p in enumerate(perfs): + index, perf_time,perf = p[1].split() + temp_outfile[int(index)].write("%s %.2f %s\n" % (index, float(float(perf_time)/1000), perf)) + + + for file in files: + file.close() + for file in temp_outfile: + file.close() + return blk_size + +def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): + if verbose: print "Computing Maths" + global_min=[] + global_max=[] + average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w') + min_file=open(gnuplot_output_dir+gnuplot_output_filename+'.min', 'w') + max_file=open(gnuplot_output_dir+gnuplot_output_filename+'.max', 'w') + stddev_file=open(gnuplot_output_dir+gnuplot_output_filename+'.stddev', 'w') + global_file=open(gnuplot_output_dir+gnuplot_output_filename+'.global','w') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.average') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.min') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.max') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.stddev') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.global') + + min_file.write('DiskName %s\n' % mode) + max_file.write('DiskName %s\n'% mode) + average_file.write('DiskName %s\n'% mode) + stddev_file.write('DiskName %s\n'% mode ) + for disk in xrange(len(fio_data_file)): +# print disk_perf[disk] + min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + avg = average(disk_perf[disk]) + variance = map(lambda x: (x - avg)**2, disk_perf[disk]) + standard_deviation = math.sqrt(average(variance)) +# print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation) + average_file.write('%d %d\n' % (disk, avg)) + stddev_file.write('%d %d\n' % (disk, standard_deviation)) + local_min=min(disk_perf[disk]) + local_max=max(disk_perf[disk]) + min_file.write('%d %d\n' % (disk, local_min)) + max_file.write('%d %d\n' % (disk, local_max)) + global_min.append(int(local_min)) + global_max.append(int(local_max)) + + global_disk_perf = sum(disk_perf, []) + avg = average(global_disk_perf) + variance = map(lambda x: (x - avg)**2, global_disk_perf) + standard_deviation = math.sqrt(average(variance)) + + global_file.write('min=%.2f\n' % min(global_disk_perf)) + global_file.write('max=%.2f\n' % max(global_disk_perf)) + global_file.write('avg=%.2f\n' % avg) + global_file.write('stddev=%.2f\n' % standard_deviation) + global_file.write('values_count=%d\n' % len(global_disk_perf)) + global_file.write('disks_count=%d\n' % len(fio_data_file)) + #print "Global [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (min(global_disk_perf),max(global_disk_perf),avg, standard_deviation) + + average_file.close() + min_file.close() + max_file.close() + stddev_file.close() + global_file.close() + try: + os.remove(gnuplot_output_dir+'mymath') + except: + True + + generate_gnuplot_math_script("Average values of "+title,gnuplot_output_filename+'.average',mode,int(avg),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Min values of "+title,gnuplot_output_filename+'.min',mode,average(global_min),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Max values of "+title,gnuplot_output_filename+'.max',mode,average(global_max),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Standard Deviation of "+title,gnuplot_output_filename+'.stddev',mode,int(standard_deviation),gnuplot_output_dir,gpm_dir) + +def parse_global_files(fio_data_file, global_search): + max_result=0 + max_file='' + for file in fio_data_file: + f=open(file) + disk_count=0 + search_value=-1 + + # Let's read the complete file + while True: + try: + # We do split the name from the value + name,value=f.readline().split("=") + except: + f.close() + break + # If we ended the file + if not name: + # Let's process what we have + f.close() + break + else: + # disks_count is not global_search item + # As we need it for some computation, let's save it + if name=="disks_count": + disks_count=int(value) + + # Let's catch the searched item + if global_search in name: + search_value=float(value) + + # Let's process the avg value by estimated the global bandwidth per file + # We keep the biggest in memory for reporting + if global_search == "avg": + if (disks_count > 0) and (search_value != -1): + result=disks_count*search_value + if (result > max_result): + max_result=result + max_file=file + # Let's print the avg output + if global_search == "avg": + print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file) + else: + print "Global search %s is not yet implemented\n" % global_search + +def render_gnuplot(fio_data_file, gnuplot_output_dir): + print "Running gnuplot Rendering" + try: + # Let's render all the compared files if some + if len(fio_data_file) > 1: + if verbose: print " |-> Rendering comparing traces" + os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir) + if verbose: print " |-> Rendering math traces" + os.system("cd %s; gnuplot mymath" % gnuplot_output_dir) + if verbose: print " |-> Rendering 2D & 3D traces" + os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir) + + name_of_directory="the current" + if gnuplot_output_dir != "./": + name_of_directory=gnuplot_output_dir + print "\nRendering traces are available in %s directory" % name_of_directory + global keep_temp_files + keep_temp_files=False + except: + print "Could not run gnuplot on mymath or mygraph !\n" + sys.exit(1); + +def print_help(): + print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>' + print + print '-h --help : Print this help' + print '-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files' + print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files' + print '-i or --iops : A predefined pattern for selecting *_iops.log files' + print '-g or --gnuplot : Render gnuplot traces before exiting' + print '-o or --outputfile <file> : The basename for gnuplot traces' + print ' - Basename is set with the pattern if defined' + print '-d or --outputdir <dir> : The directory where gnuplot shall render files' + print '-t or --title <title> : The title of the gnuplot traces' + print ' - Title is set with the block size detected in fio traces' + print '-G or --Global <type> : Search for <type> in .global files match by a pattern' + print ' - Available types are : min, max, avg, stddev' + print ' - The .global extension is added automatically to the pattern' + print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)' + print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)' + print '-v or --verbose : Increasing verbosity' + print '-k or --keep : Keep all temporary files from gnuplot\'s output dir' + +def main(argv): + mode='unknown' + pattern='' + pattern_set_by_user=False + title='No title' + gnuplot_output_filename='result' + gnuplot_output_dir='./' + gpm_dir="/usr/share/fio/" + disk_perf=[] + run_gnuplot=False + parse_global=False + global_search='' + min_time=0 + max_time=-1 + global verbose + verbose=False + global temporary_files + temporary_files=[] + global keep_temp_files + keep_temp_files=True + force_keep_temp_files=False + + if not os.path.isfile(gpm_dir+'math.gpm'): + gpm_dir="/usr/local/share/fio/" + if not os.path.isfile(gpm_dir+'math.gpm'): + print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n" + sys.exit(3) + + try: + opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) + except getopt.GetoptError: + print "Error: One of the options passed to the cmdline was not supported" + print "Please fix your command line or read the help (-h option)" + sys.exit(2) + + for opt, arg in opts: + if opt in ("-b", "--bandwidth"): + pattern='*_bw.log' + elif opt in ("-i", "--iops"): + pattern='*_iops.log' + elif opt in ("-v", "--verbose"): + verbose=True + elif opt in ("-k", "--keep"): + #User really wants to keep the temporary files + force_keep_temp_files=True + elif opt in ("-p", "--pattern"): + pattern_set_by_user=True + pattern=arg + pattern=pattern.replace('\\','') + elif opt in ("-o", "--outputfile"): + gnuplot_output_filename=arg + elif opt in ("-d", "--outputdir"): + gnuplot_output_dir=arg + if not gnuplot_output_dir.endswith('/'): + gnuplot_output_dir=gnuplot_output_dir+'/' + if not os.path.exists(gnuplot_output_dir): + os.makedirs(gnuplot_output_dir) + elif opt in ("-t", "--title"): + title=arg + elif opt in ("-m", "--min_time"): + min_time=arg + elif opt in ("-M", "--max_time"): + max_time=arg + elif opt in ("-g", "--gnuplot"): + run_gnuplot=True + elif opt in ("-G", "--Global"): + parse_global=True + global_search=arg + elif opt in ("-h", "--help"): + print_help() + sys.exit(1) + + # Adding .global extension to the file + if parse_global==True: + if not gnuplot_output_filename.endswith('.global'): + pattern = pattern+'.global' + + fio_data_file=find_file('.',pattern) + if len(fio_data_file) == 0: + print "No log file found with pattern %s!" % pattern + # Try numjob log file format if per_numjob_logs=1 + if (pattern == '*_bw.log'): + fio_data_file=find_file('.','*_bw.*.log') + if (pattern == '*_iops.log'): + fio_data_file=find_file('.','*_iops.*.log') + if len(fio_data_file) == 0: + sys.exit(1) + else: + print "Using log file per job format instead" + else: + print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern) + + fio_data_file=sorted(fio_data_file, key=str.lower) + for file in fio_data_file: + print ' |-> %s' % file + if "_bw." in file : + mode="Bandwidth (KB/sec)" + if "_iops." in file : + mode="IO per Seconds (IO/sec)" + if (title == 'No title') and (mode != 'unknown'): + if "Bandwidth" in mode: + title='Bandwidth benchmark with %d fio results' % len(fio_data_file) + if "IO" in mode: + title='IO benchmark with %d fio results' % len(fio_data_file) + + print + #We need to adjust the output filename regarding the pattern required by the user + if (pattern_set_by_user == True): + gnuplot_output_filename=pattern + # As we do have some regexp in the pattern, let's make this simpliest + # We do remove the simpliest parts of the expression to get a clear file name + gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') + gnuplot_output_filename=gnuplot_output_filename.replace('*','-') + gnuplot_output_filename=gnuplot_output_filename.replace('--','-') + gnuplot_output_filename=gnuplot_output_filename.replace('.log','') + # Insure that we don't have any starting or trailing dash to the filename + gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename + gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename + if (gnuplot_output_filename == ''): + gnuplot_output_filename='default' + + if parse_global==True: + parse_global_files(fio_data_file, global_search) + else: + blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) + title="%s @ Blocksize = %dK" % (title,blk_size/1024) + compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) + compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + + if (run_gnuplot==True): + render_gnuplot(fio_data_file, gnuplot_output_dir) + + # Shall we clean the temporary files ? + if keep_temp_files==False and force_keep_temp_files==False: + # Cleaning temporary files + if verbose: print "Cleaning temporary files" + for f in enumerate(temporary_files): + if verbose: print " -> %s"%f[1] + try: + os.remove(f[1]) + except: + True + +#Main +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/fiotools-aio/lib/fio2gnuplot.bak.2 b/fiotools-aio/lib/fio2gnuplot.bak.2 new file mode 100755 index 0000000..a5b6a3c --- /dev/null +++ b/fiotools-aio/lib/fio2gnuplot.bak.2 @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2013 eNovance SAS <licensing@enovance.com> +# Author: Erwan Velu <erwan@enovance.com> +# +# The license below covers all files distributed with fio unless otherwise +# noted in the file itself. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import fnmatch +import sys +import getopt +import re +import math +import shutil + +def find_file(path, pattern): + fio_data_file=[] + # For all the local files + for file in os.listdir(path): + # If the file math the regexp + if fnmatch.fnmatch(file, pattern): + # Let's consider this file + fio_data_file.append(file) + + return fio_data_file + +def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): + if verbose: print("Generating rendering scripts") + filename=gnuplot_output_dir+'mygraph' + temporary_files.append(filename) + f=open(filename,'w') + + # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) + + # Setting up the compare files that will be plot later + compare=open(gnuplot_output_dir + 'compare.gnuplot','w') + compare.write(''' +set title '%s' +set terminal png size 1280,1024 +set ytics axis out auto +set key top left reverse +set xlabel "Time (Seconds)" +set ylabel '%s' +set yrange [0:] +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" +'''% (title,mode)) + compare.close() + #Copying the common file for all kind of graph (raw/smooth/trend) + compare_raw_filename="compare-%s-2Draw" % (gnuplot_output_filename) + compare_smooth_filename="compare-%s-2Dsmooth" % (gnuplot_output_filename) + compare_trend_filename="compare-%s-2Dtrend" % (gnuplot_output_filename) + + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_raw_filename+".gnuplot") + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_smooth_filename+".gnuplot") + shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_trend_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_raw_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_smooth_filename+".gnuplot") + temporary_files.append(gnuplot_output_dir+compare_trend_filename+".gnuplot") + + #Setting up a different output filename for each kind of graph + compare_raw=open(gnuplot_output_dir+compare_raw_filename + ".gnuplot",'a') + compare_raw.write("set output '%s.png'\n" % compare_raw_filename) + compare_smooth=open(gnuplot_output_dir+compare_smooth_filename+".gnuplot",'a') + compare_smooth.write("set output '%s.png'\n" % compare_smooth_filename) + compare_trend=open(gnuplot_output_dir+compare_trend_filename+".gnuplot",'a') + compare_trend.write("set output '%s.png'\n" % compare_trend_filename) + + # Let's plot the average value for all the traces + global_disk_perf = sum(disk_perf, []) + global_avg = average(global_disk_perf) + compare_raw.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); + + pos=0 + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "gnuplot_temp_file.%d" % pos + + # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + # Adding the plot instruction for each kind of comparing graphs + compare_raw.write(",\\\n'%s' using 2:3 with linespoints title '%s'" % (tmp_filename,fio_data_file[pos])) + compare_smooth.write(",\\\n'%s' using 2:3 smooth csplines title '%s'" % (tmp_filename,fio_data_file[pos])) + compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos])) + + png_file=file.replace('.log','') + raw_filename = "%s-2Draw" % (png_file) + smooth_filename = "%s-2Dsmooth" % (png_file) + trend_filename = "%s-2Dtrend" % (png_file) + avg = average(disk_perf[pos]) + f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) + pos = pos +1 + + # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces + if len(fio_data_file) > 1: + os.remove(gnuplot_output_dir+"compare.gnuplot") + compare_raw.close() + compare_smooth.close() + compare_trend.close() + f.close() + +def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnuplot_output_dir,gpm_dir): + filename=gnuplot_output_dir+'mymath'; + temporary_files.append(filename) + f=open(filename,'a') + f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) + f.close() + +def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir): + if verbose: print("Processing data file 2/2") + temp_files=[] + pos=0 + + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir, pos) + temp_files.append(open(tmp_filename,'r')) + pos = pos +1 + + f = open(gnuplot_output_dir+gnuplot_output_filename, "w") + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename) + index=0 + # Let's add some information + for tempfile in temp_files: + f.write("# Disk%d was coming from %s\n" % (index,fio_data_file[index])) + f.write(tempfile.read()) + f.write("\n") + tempfile.close() + index = index + 1 + f.close() + +def average(s): return sum(s) * 1.0 / len(s) + +def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_time): + end_time=max_time + if end_time == -1: + end_time="infinite" + if verbose: print("Processing data file 1/2 with %s<time<%s" % (min_time,end_time)) + files=[] + temp_outfile=[] + blk_size=0 + for file in fio_data_file: + files.append(open(file)) + pos = len(files) - 1 + tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir,pos) + temporary_files.append(tmp_filename) + gnuplot_file=open(tmp_filename,'w') + temp_outfile.append(gnuplot_file) + gnuplot_file.write("#Temporary file based on file %s\n" % file) + disk_perf.append([]) + + shall_break = False + while True: + current_line=[] + nb_empty_files=0 + nb_files=len(files) + for myfile in files: + s=myfile.readline().replace(',',' ').split() + if not s: + nb_empty_files+=1 + s="-1, 0, 0, 0".replace(',',' ').split() + + if (nb_empty_files == nb_files): + shall_break=True + break; + + current_line.append(s); + + if shall_break == True: + break + + last_time = -1 + index=-1 + perfs=[] + for line in enumerate(current_line): + # Index will be used to remember what file was featuring what value + index=index+1 + + time, perf, x, block_size = line[1] + if (blk_size == 0): + try: + blk_size=int(block_size) + except: + print("Error while reading the following line :") + print(line) + sys.exit(1); + + # We ignore the first 500msec as it doesn't seems to be part of the real benchmark + # Time < 500 usually reports BW=0 breaking the min computing + if (min_time == 0): + min_time==0.5 + + # Then we estimate if the data we got is part of the time range we want to plot + if ((float(time)>(float(min_time)*1000)) and ((int(time) < (int(max_time)*1000)) or max_time==-1)): + disk_perf[index].append(int(perf)) + perfs.append("%d %s %s"% (index, time, perf)) + + # If we reach this point, it means that all the traces are coherent + for p in enumerate(perfs): + index, perf_time,perf = p[1].split() + temp_outfile[int(index)].write("%s %.2f %s\n" % (index, float(float(perf_time)/1000), perf)) + + + for file in files: + file.close() + for file in temp_outfile: + file.close() + return blk_size + +def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): + if verbose: print("Computing Maths") + global_min=[] + global_max=[] + average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w') + min_file=open(gnuplot_output_dir+gnuplot_output_filename+'.min', 'w') + max_file=open(gnuplot_output_dir+gnuplot_output_filename+'.max', 'w') + stddev_file=open(gnuplot_output_dir+gnuplot_output_filename+'.stddev', 'w') + global_file=open(gnuplot_output_dir+gnuplot_output_filename+'.global','w') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.average') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.min') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.max') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.stddev') + temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.global') + + min_file.write('DiskName %s\n' % mode) + max_file.write('DiskName %s\n'% mode) + average_file.write('DiskName %s\n'% mode) + stddev_file.write('DiskName %s\n'% mode ) + for disk in range(len(fio_data_file)): +# print disk_perf[disk] + min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) + avg = average(disk_perf[disk]) + variance = [(x - avg)**2 for x in disk_perf[disk]] + standard_deviation = math.sqrt(average(variance)) +# print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation) + average_file.write('%d %d\n' % (disk, avg)) + stddev_file.write('%d %d\n' % (disk, standard_deviation)) + local_min=min(disk_perf[disk]) + local_max=max(disk_perf[disk]) + min_file.write('%d %d\n' % (disk, local_min)) + max_file.write('%d %d\n' % (disk, local_max)) + global_min.append(int(local_min)) + global_max.append(int(local_max)) + + global_disk_perf = sum(disk_perf, []) + avg = average(global_disk_perf) + variance = [(x - avg)**2 for x in global_disk_perf] + standard_deviation = math.sqrt(average(variance)) + + global_file.write('min=%.2f\n' % min(global_disk_perf)) + global_file.write('max=%.2f\n' % max(global_disk_perf)) + global_file.write('avg=%.2f\n' % avg) + global_file.write('stddev=%.2f\n' % standard_deviation) + global_file.write('values_count=%d\n' % len(global_disk_perf)) + global_file.write('disks_count=%d\n' % len(fio_data_file)) + #print "Global [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (min(global_disk_perf),max(global_disk_perf),avg, standard_deviation) + + average_file.close() + min_file.close() + max_file.close() + stddev_file.close() + global_file.close() + try: + os.remove(gnuplot_output_dir+'mymath') + except: + True + + generate_gnuplot_math_script("Average values of "+title,gnuplot_output_filename+'.average',mode,int(avg),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Min values of "+title,gnuplot_output_filename+'.min',mode,average(global_min),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Max values of "+title,gnuplot_output_filename+'.max',mode,average(global_max),gnuplot_output_dir,gpm_dir) + generate_gnuplot_math_script("Standard Deviation of "+title,gnuplot_output_filename+'.stddev',mode,int(standard_deviation),gnuplot_output_dir,gpm_dir) + +def parse_global_files(fio_data_file, global_search): + max_result=0 + max_file='' + for file in fio_data_file: + f=open(file) + disk_count=0 + search_value=-1 + + # Let's read the complete file + while True: + try: + # We do split the name from the value + name,value=f.readline().split("=") + except: + f.close() + break + # If we ended the file + if not name: + # Let's process what we have + f.close() + break + else: + # disks_count is not global_search item + # As we need it for some computation, let's save it + if name=="disks_count": + disks_count=int(value) + + # Let's catch the searched item + if global_search in name: + search_value=float(value) + + # Let's process the avg value by estimated the global bandwidth per file + # We keep the biggest in memory for reporting + if global_search == "avg": + if (disks_count > 0) and (search_value != -1): + result=disks_count*search_value + if (result > max_result): + max_result=result + max_file=file + # Let's print the avg output + if global_search == "avg": + print("Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)) + else: + print("Global search %s is not yet implemented\n" % global_search) + +def render_gnuplot(fio_data_file, gnuplot_output_dir): + print("Running gnuplot Rendering") + try: + # Let's render all the compared files if some + if len(fio_data_file) > 1: + if verbose: print(" |-> Rendering comparing traces") + os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir) + if verbose: print(" |-> Rendering math traces") + os.system("cd %s; gnuplot mymath" % gnuplot_output_dir) + if verbose: print(" |-> Rendering 2D & 3D traces") + os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir) + + name_of_directory="the current" + if gnuplot_output_dir != "./": + name_of_directory=gnuplot_output_dir + print("\nRendering traces are available in %s directory" % name_of_directory) + global keep_temp_files + keep_temp_files=False + except: + print("Could not run gnuplot on mymath or mygraph !\n") + sys.exit(1); + +def print_help(): + print('fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>') + print() + print('-h --help : Print this help') + print('-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files') + print('-b or --bandwidth : A predefined pattern for selecting *_bw.log files') + print('-i or --iops : A predefined pattern for selecting *_iops.log files') + print('-g or --gnuplot : Render gnuplot traces before exiting') + print('-o or --outputfile <file> : The basename for gnuplot traces') + print(' - Basename is set with the pattern if defined') + print('-d or --outputdir <dir> : The directory where gnuplot shall render files') + print('-t or --title <title> : The title of the gnuplot traces') + print(' - Title is set with the block size detected in fio traces') + print('-G or --Global <type> : Search for <type> in .global files match by a pattern') + print(' - Available types are : min, max, avg, stddev') + print(' - The .global extension is added automatically to the pattern') + print('-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)') + print('-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)') + print('-v or --verbose : Increasing verbosity') + print('-k or --keep : Keep all temporary files from gnuplot\'s output dir') + +def main(argv): + mode='unknown' + pattern='' + pattern_set_by_user=False + title='No title' + gnuplot_output_filename='result' + gnuplot_output_dir='./' + gpm_dir="/usr/share/fio/" + disk_perf=[] + run_gnuplot=False + parse_global=False + global_search='' + min_time=0 + max_time=-1 + global verbose + verbose=False + global temporary_files + temporary_files=[] + global keep_temp_files + keep_temp_files=True + force_keep_temp_files=False + + if not os.path.isfile(gpm_dir+'math.gpm'): + gpm_dir="/usr/local/share/fio/" + if not os.path.isfile(gpm_dir+'math.gpm'): + print("Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n") + sys.exit(3) + + try: + opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) + except getopt.GetoptError: + print("Error: One of the options passed to the cmdline was not supported") + print("Please fix your command line or read the help (-h option)") + sys.exit(2) + + for opt, arg in opts: + if opt in ("-b", "--bandwidth"): + pattern='*_bw.log' + elif opt in ("-i", "--iops"): + pattern='*_iops.log' + elif opt in ("-v", "--verbose"): + verbose=True + elif opt in ("-k", "--keep"): + #User really wants to keep the temporary files + force_keep_temp_files=True + elif opt in ("-p", "--pattern"): + pattern_set_by_user=True + pattern=arg + pattern=pattern.replace('\\','') + elif opt in ("-o", "--outputfile"): + gnuplot_output_filename=arg + elif opt in ("-d", "--outputdir"): + gnuplot_output_dir=arg + if not gnuplot_output_dir.endswith('/'): + gnuplot_output_dir=gnuplot_output_dir+'/' + if not os.path.exists(gnuplot_output_dir): + os.makedirs(gnuplot_output_dir) + elif opt in ("-t", "--title"): + title=arg + elif opt in ("-m", "--min_time"): + min_time=arg + elif opt in ("-M", "--max_time"): + max_time=arg + elif opt in ("-g", "--gnuplot"): + run_gnuplot=True + elif opt in ("-G", "--Global"): + parse_global=True + global_search=arg + elif opt in ("-h", "--help"): + print_help() + sys.exit(1) + + # Adding .global extension to the file + if parse_global==True: + if not gnuplot_output_filename.endswith('.global'): + pattern = pattern+'.global' + + fio_data_file=find_file('.',pattern) + if len(fio_data_file) == 0: + print("No log file found with pattern %s!" % pattern) + # Try numjob log file format if per_numjob_logs=1 + if (pattern == '*_bw.log'): + fio_data_file=find_file('.','*_bw.*.log') + if (pattern == '*_iops.log'): + fio_data_file=find_file('.','*_iops.*.log') + if len(fio_data_file) == 0: + sys.exit(1) + else: + print("Using log file per job format instead") + else: + print("%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)) + + fio_data_file=sorted(fio_data_file, key=str.lower) + for file in fio_data_file: + print(' |-> %s' % file) + if "_bw." in file : + mode="Bandwidth (KB/sec)" + if "_iops." in file : + mode="IO per Seconds (IO/sec)" + if (title == 'No title') and (mode != 'unknown'): + if "Bandwidth" in mode: + title='Bandwidth benchmark with %d fio results' % len(fio_data_file) + if "IO" in mode: + title='IO benchmark with %d fio results' % len(fio_data_file) + + print() + #We need to adjust the output filename regarding the pattern required by the user + if (pattern_set_by_user == True): + gnuplot_output_filename=pattern + # As we do have some regexp in the pattern, let's make this simpliest + # We do remove the simpliest parts of the expression to get a clear file name + gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') + gnuplot_output_filename=gnuplot_output_filename.replace('*','-') + gnuplot_output_filename=gnuplot_output_filename.replace('--','-') + gnuplot_output_filename=gnuplot_output_filename.replace('.log','') + # Insure that we don't have any starting or trailing dash to the filename + gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename + gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename + if (gnuplot_output_filename == ''): + gnuplot_output_filename='default' + + if parse_global==True: + parse_global_files(fio_data_file, global_search) + else: + blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) + title="%s @ Blocksize = %dK" % (title,blk_size/1024) + compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) + compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) + + if (run_gnuplot==True): + render_gnuplot(fio_data_file, gnuplot_output_dir) + + # Shall we clean the temporary files ? + if keep_temp_files==False and force_keep_temp_files==False: + # Cleaning temporary files + if verbose: print("Cleaning temporary files") + for f in enumerate(temporary_files): + if verbose: print(" -> %s"%f[1]) + try: + os.remove(f[1]) + except: + True + +#Main +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/fiotools-aio/lib/graph2D.gpm b/fiotools-aio/lib/graph2D.gpm new file mode 100644 index 0000000..769b754 --- /dev/null +++ b/fiotools-aio/lib/graph2D.gpm @@ -0,0 +1,55 @@ +# This Gnuplot file has been generated by eNovance + +needed_args = 8 +if (exists("ARGC") && ARGC >= needed_args) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" >= needed_args) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ + ARG6 = "$5"; \ + ARG7 = "$6"; \ + ARG8 = "$7"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +avg_num = ARG8 + 0 +avg_str = sprintf("%g", avg_num) + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' +#set terminal x11 + +#Preparing Axes +#set logscale x +set ytics axis out auto +#set data style lines +set key top left reverse +set xlabel "Time (Seconds)" +set ylabel ARG5 +set xrange [0:] +set yrange [0:] + +#Set Color style +#set palette rgbformulae 22,9,23 +#set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" + +plot ARG2 using 2:3 with linespoints title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' + +set output ARG6 . '.png' +plot ARG2 using 2:3 smooth csplines title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' + +set output ARG7 . '.png' +plot ARG2 using 2:3 smooth bezier title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str .')' + +#pause -1 +#The End diff --git a/fiotools-aio/lib/graph3D.gpm b/fiotools-aio/lib/graph3D.gpm new file mode 100644 index 0000000..ac2cdf6 --- /dev/null +++ b/fiotools-aio/lib/graph3D.gpm @@ -0,0 +1,95 @@ +# This Gnuplot file has been generated by eNovance + +needed_args = 5 +if (exists("ARGC") && ARGC >= needed_args) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" >= needed_args) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' +#set terminal x11 +#3D Config +set isosamples 30 +set hidden3d +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder + +#Preparing Axes +#set logscale x +set ytics axis out 0,1 +#set data style lines +set grid back +set key top left reverse +set ylabel "Disk" +set xlabel "Time (Seconds)" +set zlabel ARG5 +set cbrange [0:] +set zrange [0:] + +#Set Color style +#set palette rgbformulae 22,9,23 +set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 + +#Multiploting +set multiplot + +#Top Left View +set size 0.5,0.5 +set view 64,216 +set origin 0,0.5 +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Top Right View +set size 0.5,0.5 +set origin 0.5,0.5 +set view 90,0 +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Bottom Right View +set size 0.5,0.5 +set origin 0.5,0 +set view 63,161 +set pm3d at s solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Bottom Left View +set size 0.5,0.5 +set origin 0,0 +set pm3d map +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#Unsetting multiplotting +unset multiplot +#pause -1 + +#Preparing 3D Interactive view +set mouse +set terminal png size 1024,768 +set output ARG4 . '-3D.png' + +#set term x11 +set view 64,216 +set origin 0,0 +set size 1,1 +set pm3d at bs solid hidden3d 100 scansbackward +set pm3d depthorder +splot ARG2 using 2:1:3 with linespoints title ARG3 + +#pause -1 +#The End diff --git a/fiotools-aio/lib/math.gpm b/fiotools-aio/lib/math.gpm new file mode 100644 index 0000000..0a2aff5 --- /dev/null +++ b/fiotools-aio/lib/math.gpm @@ -0,0 +1,42 @@ +# This Gnuplot file has been generated by eNovance +if (exists("ARGC") && ARGC > 5) \ + found_args = 1; \ +else if (strlen("$$#") < 3 && "$#" > 5) \ + found_args = 1; \ + ARG1 = "$0"; \ + ARG2 = "$1"; \ + ARG3 = "$2"; \ + ARG4 = "$3"; \ + ARG5 = "$4"; \ + ARG6 = "$5"; \ +else \ + found_args = 0; \ + print "Aborting: could not find all arguments"; \ + exit + +avg_num = ARG6 + 0 +avg_str = sprintf("%g", avg_num) + +set title ARG1 + +set terminal png size 1280,1024 +set output ARG4 . '.png' + +set palette rgbformulae 7,5,15 +set style line 100 lt 7 lw 0.5 +set style fill transparent solid 0.9 noborder +set auto x +set ylabel ARG5 +set xlabel "Disk" +set yrange [0:] +set style data histogram +set style histogram cluster gap 1 +set style fill solid border -1 +set boxwidth 2 +#set xtic rotate by -10 scale 10 font ",8" +set bmargin 3 +set xtics axis out +set xtic rotate by 45 scale 0 font ",8" autojustify +set xtics offset 0,-1 border -5,1,5 +set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" +plot ARG2 using 2:xtic(1) ti col, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')' diff --git a/fiotools-aio/plot.sh b/fiotools-aio/plot.sh index 8edd355..45a1080 100644 --- a/fiotools-aio/plot.sh +++ b/fiotools-aio/plot.sh @@ -1,4 +1,100 @@ -#!/bin/bash +#!/bin/sh +set -eu + +echo "Plotting data..." +PLOTNAME=${PLOTNAME:-FIO} +OUTDIR=${OUTDIR:-plots} +mkdir -p "$OUTDIR" + +# Expand globs in POSIX and ignore non-matches +find_logs() { + for pat in "$@"; do + # unquoted to allow globbing + for f in $pat; do + case "$f" in + *'*'*|*'?'*|*'['*']'*) + : ;; # still a pattern -> no matches + *) + [ -f "$f" ] && printf '%s\n' "$f" + ;; + esac + done + done +} + +BW_LOGS="$(find_logs '*_bw.log' '*_bw.*.log')" +IOPS_LOGS="$(find_logs '*_iops.log' '*_iops.*.log')" +LAT_LOGS="$(find_logs '*_lat*.log' '*_clat*.log' '*_slat*.log')" + +WORK=".norm" +rm -rf "$WORK"; mkdir "$WORK" + +# Normalize time to integer milliseconds from t0 +normalize() { + in="$1"; out="$WORK/$(basename "$1")" + # first numeric timestamp + first="$(awk 'NF{print $1; exit}' "$in" 2>/dev/null || echo 0)" + awk -v t0="$first" 'BEGIN{OFS="\t"; mode=0} + NR==1{ + if (t0>=1000000000000) mode=1; # epoch ms + else if (t0>=1000000000) mode=2; # epoch s + else mode=3; # relative seconds + } + { + t=$1 + if (mode==1) t = t - t0; # already ms + else t = (t - t0) * 1000; # seconds -> ms + $1 = int(t + 0.5) # force integer ms + print + }' "$in" > "$out" +} + +# Normalize all logs we found +for f in $BW_LOGS $IOPS_LOGS $LAT_LOGS; do + [ -n "${f:-}" ] && normalize "$f" || : +done + +# best-effort bs hint for titles +BS_HINT="" +if [ -n "${JOBFILES:-}" ] && [ -f "$JOBFILES" ]; then + BS_HINT="$(awk -F= '$0 ~ /(^|[,;[:space:]])bs[[:space:]]*=/ {print $2; exit}' "$JOBFILES" 2>/dev/null || true)" +fi +[ -z "$BS_HINT" ] && [ -f fio.output ] && BS_HINT="$( + awk -F'[= ,]' '$0 ~ / bs=/ {for(i=1;i<=NF;i++) if($i=="bs"){print $(i+1); exit}}' fio.output 2>/dev/null || true +)" + +cd "$WORK" + +# detect optional flags +HAS_NO3D="" +if fio2gnuplot -h 2>&1 | grep -Eq '(^|[[:space:]])--no-3d([[:space:]]|,|$)'; then + HAS_NO3D="--no-3d" +fi +HAS_LAT="" +if fio2gnuplot -h 2>&1 | grep -Eq '(^|[[:space:]])-l([[:space:]]|,|$)'; then + HAS_LAT="-l" +fi + +GP_OPTS="-g" +[ -n "$HAS_NO3D" ] && GP_OPTS="$GP_OPTS $HAS_NO3D" + +ttl_bs="" +[ -n "$BS_HINT" ] && ttl_bs=" (bs=$BS_HINT)" + +if [ -n "$BW_LOGS" ]; then + echo "$(printf '%s\n' $BW_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_bw*'" + fio2gnuplot $GP_OPTS -b -t "${PLOTNAME} - Bandwidth${ttl_bs}" -d "../$OUTDIR" || true +fi + +if [ -n "$IOPS_LOGS" ]; then + echo "$(printf '%s\n' $IOPS_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_iops*'" + fio2gnuplot $GP_OPTS -i -t "${PLOTNAME} - IOPS${ttl_bs}" -d "../$OUTDIR" || true +fi + +if [ -n "$LAT_LOGS" ] && [ -n "$HAS_LAT" ]; then + echo "$(printf '%s\n' $LAT_LOGS | wc -l | awk '{print $1}') files Selected with pattern '*_lat|clat|slat*'" + fio2gnuplot $GP_OPTS $HAS_LAT -t "${PLOTNAME} - Latency${ttl_bs}" -d "../$OUTDIR" || true +fi + +cd .. -fio2gnuplot -t $PLOTNAME-bw -b -g -p '*_bw*' -fio2gnuplot -t $PLOTNAME-iops -i -g -p '*_iops*' diff --git a/fiotools-aio/run.sh b/fiotools-aio/run.sh index 91b8098..c511d99 100644 --- a/fiotools-aio/run.sh +++ b/fiotools-aio/run.sh @@ -1,3 +1,15 @@ -#!/bin/bash +#!/bin/sh -fio --output=fio.output $JOBFILES +echo "Running FIO job $JOBFILES" +LOG_MS="${LOG_MS:-1000}" # 1s buckets +STEM="${STEM:-result}" # our merged log stem + +# Run the user-supplied job, but force our own logs as well +fio \ + --write_bw_log="$STEM" \ + --write_iops_log="$STEM" \ + --write_lat_log="$STEM" \ + --log_avg_msec="$LOG_MS" \ + --log_unix_epoch=1 \ + --per_job_logs=0 \ + ${JOBFILES} 2>&1 | tee fio.output \ No newline at end of file diff --git a/fiotools-aio/runall.sh b/fiotools-aio/runall.sh index 07a9d7f..3cf353e 100644 --- a/fiotools-aio/runall.sh +++ b/fiotools-aio/runall.sh @@ -1,7 +1,11 @@ -#!/bin/bash +#!/bin/sh +OUTDIR="${OUTDIR:-plots}" + +echo "Running all-in-one..." sh /opt/check.sh sh /opt/run.sh sh /opt/plot.sh -python -m SimpleHTTPServer 8000 +echo "Starting server..." +python3 -m http.server 8000 diff --git a/kubernetes/fiotools-aio-portworx-read.yaml b/kubernetes/fiotools-aio-portworx-read.yaml index d5ecfea..d326502 100644 --- a/kubernetes/fiotools-aio-portworx-read.yaml +++ b/kubernetes/fiotools-aio-portworx-read.yaml @@ -1,11 +1,10 @@ - kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 +apiVersion: storage.k8s.io/v1 metadata: - name: fio-tester-class-read -provisioner: kubernetes.io/portworx-volume + name: fio-tester-class-write +provisioner: pxd.portworx.com parameters: - repl: "1" + repl: "1" --- @@ -14,9 +13,8 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: name: fio-data-read - annotations: - volume.beta.kubernetes.io/storage-class: fio-tester-class-read spec: + storageClassName: fio-tester-class-write accessModes: - ReadWriteOnce resources: @@ -25,7 +23,7 @@ spec: --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: fio-tester-read @@ -38,6 +36,9 @@ spec: maxUnavailable: 1 type: RollingUpdate replicas: 1 + selector: + matchLabels: + app: fio-testing-write template: metadata: labels: @@ -46,6 +47,7 @@ spec: containers: - name: fio-container image: wallnerryan/fiotools-aio + imagePullPolicy: Always ports: - containerPort: 8000 volumeMounts: diff --git a/kubernetes/fiotools-aio-portworx-write.yaml b/kubernetes/fiotools-aio-portworx-write.yaml index 542a480..d6c2d48 100644 --- a/kubernetes/fiotools-aio-portworx-write.yaml +++ b/kubernetes/fiotools-aio-portworx-write.yaml @@ -1,11 +1,10 @@ - kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 +apiVersion: storage.k8s.io/v1 metadata: - name: fio-tester-class-write -provisioner: kubernetes.io/portworx-volume + name: fio-tester-class-write +provisioner: pxd.portworx.com parameters: - repl: "1" + repl: "1" --- @@ -14,9 +13,8 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: name: fio-data-write - annotations: - volume.beta.kubernetes.io/storage-class: fio-tester-class-write spec: + storageClassName: fio-tester-class-write accessModes: - ReadWriteOnce resources: @@ -25,7 +23,7 @@ spec: --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: fio-tester-write @@ -38,6 +36,9 @@ spec: maxUnavailable: 1 type: RollingUpdate replicas: 1 + selector: + matchLabels: + app: fio-testing-write template: metadata: labels: @@ -45,7 +46,8 @@ spec: spec: containers: - name: fio-container - image: wallnerryan/fiotools-aio + image: wallnerryan/fiotools-aio:fio336alpine + imagePullPolicy: Always ports: - containerPort: 8000 volumeMounts: From 8c7f578929d63b2b03dacaec5fdc94d7554eb041 Mon Sep 17 00:00:00 2001 From: Ryan Wallner <rwallner@purestorage.com> Date: Tue, 30 Sep 2025 15:18:05 -0400 Subject: [PATCH 2/2] fix CVEs, new build process, newer FIO --- README.md | 216 ++++++++++++++++++++++++++++++++++++++- base-fiotools/Dockerfile | 60 +++++++++-- buildimages.sh | 131 +++++++++++++++--------- 3 files changed, 347 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 8e9bd64..471d3d6 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,218 @@ wallnerryan/fio-plotserve - https://www.spinics.net/lists/fio/msg05517.html -**build** +## Build -`env bash ./buildimages.sh` \ No newline at end of file +This repo ships a single orchestrator script, `buildimages.sh`, that builds all the fio-tools images in the correct order, either **locally** (single-arch) or as a **multi-arch** build with a **push to Docker Hub**. + +It solves common pitfalls like: +- child images failing with `FROM ${BASE_IMAGE}` because the base isn’t visible to BuildKit +- builder/context confusion on Docker Desktop +- reliably wiring build args (gnuplot/python versions) through the whole stack + +--- + +## What it builds + +The script builds these images (in order): + +1. `base-fiotools` → foundation image (gnuplot built from source, Python tuned, etc.) +2. `fio-genplots` → plotting helper +3. `fio-tool` → runner +4. `fio-plotserve` → plot web server +5. `fiotools-aio` → all-in-one image + +All dependent Dockerfiles use: + +```dockerfile +# syntax=docker/dockerfile:1.6 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +``` + +The script passes `--build-arg BASE_IMAGE=<tag>` so each layer builds against the **just-built base** (local mode) or the **just-pushed base** (push mode). + +--- + +## Prerequisites + +- Docker Desktop or Docker Engine 24+ (with `docker buildx` bundled) +- (For multi-arch) QEMU emulation: the script sets it up automatically with `tonistiigi/binfmt` +- (For push) Docker Hub login: `docker login` + +--- + +## Modes + +### 1) Local build (default) +- `FIOTOOLS_DOCKERHUB_PUSH=false` (or unset) +- Uses **classic `docker build`** so `FROM ${BASE_IMAGE}` resolves to the **local** tag you just built. +- **Single-arch only**: `PLATFORMS` **must match the host** (e.g. `linux/amd64` on Intel, `linux/arm64` on Apple Silicon). + +### 2) Multi-arch + push +- `FIOTOOLS_DOCKERHUB_PUSH=true` +- Uses **buildx** with a **container driver**, builds **multi-arch** and **pushes** to Docker Hub. +- Base is pushed first; dependents then **pull** that base by tag. + +--- + +## Environment variables + +| Var | Default | Purpose | +|---|---|---| +| `FIOTOOLS_DOCKERHUB_USERNAME` | `fiotools` | Docker Hub namespace/repo prefix | +| `FIOTOOLS_DOCKERHUB_TAG` | `tag` | Tag applied to all images | +| `FIOTOOLS_DOCKERHUB_PUSH` | `false` | `true` = multi-arch buildx build + push | +| `PLATFORMS` | `linux/amd64,linux/arm64` | Target platforms (push mode). In local mode, **must equal host**. | +| `DOCKER_CONTEXT` | `default` | Docker context used by the script | +| `GNUPLOT_VERSION` | `5.4.10` | Built from source in `base-fiotools` | +| `PY_VER` | `3.12.5` | CPython version compiled in `base-fiotools` | + +--- + +## Usage + +### Local single-arch (build into your daemon) +```bash +# Pick the platform that matches your host CPU +export PLATFORMS=linux/amd64 # on Intel/Linux/Windows +# or +export PLATFORMS=linux/arm64 # on Apple Silicon + +export FIOTOOLS_DOCKERHUB_USERNAME=<insert> +export FIOTOOLS_DOCKERHUB_TAG=<tag> +export FIOTOOLS_DOCKERHUB_PUSH=false + +./buildimages.sh +``` + +You’ll end up with locally available images like: +``` +wallnerryan/base-fiotools:fio336alpinev2 +wallnerryan/fio-genplots:fio336alpinev2 +wallnerryan/fio-tool:fio336alpinev2 +wallnerryan/fio-plotserve:fio336alpinev2 +wallnerryan/fiotools-aio:fio336alpinev2 +``` + +### Multi-arch push (amd64 + arm64) +```bash +export FIOTOOLS_DOCKERHUB_USERNAME=<user> +export FIOTOOLS_DOCKERHUB_TAG=<tag> +export FIOTOOLS_DOCKERHUB_PUSH=true +export PLATFORMS=linux/amd64,linux/arm64 + +docker login +./buildimages.sh +``` + +This will: +1) build & **push** `wallnerryan/base-fiotools:fio336alpinev2` for both arches +2) build & **push** the remaining images using that base + +--- + +## Customizing versions + +Override at runtime: + +```bash +export GNUPLOT_VERSION=5.4.11 +export PY_VER=3.12.7 +./buildimages.sh +``` + +Both values are passed as `--build-arg`s into `base-fiotools`. + +--- + +## Common pitfalls & fixes + +- **“not found … base-fiotools:TAG” during dependent build** + You ran a buildx containerized build that can’t see your local daemon image. + ✅ Use local mode (default) → script runs **classic `docker build`** for all images, or + ✅ Use push mode so dependents **pull** the base from the registry. + +- **“additional instances of driver ‘docker’ cannot be created”** + Old builders/contexts cause conflicts. The script avoids these, but if you hit leftovers: + ```bash + docker buildx ls + docker buildx rm <stale-builder-name> + docker context use default + ``` + +- **“use \`docker context use default\`”** + You’re on a non-default context. Either: + ```bash + docker context use default + ``` + or set `DOCKER_CONTEXT`: + ```bash + export DOCKER_CONTEXT=default + ./buildimages.sh + ``` + +- **Local mode with mismatched PLATFORMS** + Local `--load` can only load the host’s architecture. Set: + ```bash + export PLATFORMS=$(uname -m | grep -qi arm && echo linux/arm64 || echo linux/amd64) + ``` + +--- + +## What changed vs. naive buildx flows? + +- Local builds use **classic `docker build`** so your `FROM ${BASE_IMAGE}` can reference the **locally built tag**. +- Multi-arch builds use a **single containerized builder** that can **push**, ensuring dependents can pull the just-pushed base. + +--- + +## Outputs & naming + +All images are tagged as: +``` +<NS>/<component>:<TAG> +``` +Where `<NS>` = `FIOTOOLS_DOCKERHUB_USERNAME`, `<TAG>` = `FIOTOOLS_DOCKERHUB_TAG`. + +--- + +## Clean up + +To remove the buildx builder created by push mode (optional): +```bash +docker buildx rm fiotools || true +``` + +To reset contexts: +```bash +docker context use default +``` + +#### Force a clean rebuild (no cache) locally: + +`NO_CACHE=1 ./buildimages.sh` + + +Also force parent pulls (ignore base-layer cache): + +`NO_CACHE=1 PULL_BASE=1 ./buildimages.sh` + + +Multi-arch push with no cache: + +`FIOTOOLS_DOCKERHUB_PUSH=true NO_CACHE=true PULL_BASE=true ./buildimages.sh` + +Optional: completely nuke buildx caches + +If you want to also wipe any persisted builder caches: + +`docker --context "$DOCKER_CONTEXT" buildx prune -af` + +--- + +## Security notes + +- `base-fiotools` builds **gnuplot from source** and compiles **CPython** so you can avoid CVEs seen in Alpine’s prebuilt stacks. +- You can bump `GNUPLOT_VERSION` and `PY_VER` to pick up upstream fixes quickly. +- Downstream images inherit from the base—so security improvements are centralized. diff --git a/base-fiotools/Dockerfile b/base-fiotools/Dockerfile index 634e1ed..d9db2fb 100644 --- a/base-fiotools/Dockerfile +++ b/base-fiotools/Dockerfile @@ -4,8 +4,6 @@ ARG GNUPLOT_VERSION=5.4.10 FROM alpine:${ALPINE_VERSION} AS gnuplot-build ARG GNUPLOT_VERSION -# Build gnuplot to avoid CVEs in upstreadm 3.20 alpine -# two criticals are in glib and harfbuzz pulled in by Alpine’s gnuplot stack (via cairo/pango). RUN set -eux; \ apk add --no-cache \ build-base wget tar \ @@ -27,22 +25,67 @@ RUN set -eux; \ make -j"$(nproc)"; \ make install-strip -# ---------- Stage 2: runtime ---------- +# ---------- Stage 2: build CPython WITHOUT sqlite ---------- +ARG ALPINE_VERSION=3.20 +ARG PY_VER=3.12.5 +FROM alpine:${ALPINE_VERSION} AS python-build +ARG PY_VER + +RUN set -eux; \ + apk add --no-cache \ + build-base wget tar xz \ + openssl-dev bzip2-dev zlib-dev libffi-dev \ + readline-dev ncurses-dev expat-dev; \ + mkdir -p /tmp/build && cd /tmp/build; \ + wget -O Python.tar.xz "https://www.python.org/ftp/python/${PY_VER}/Python-${PY_VER}.tar.xz"; \ + tar -xf Python.tar.xz; \ + cd Python-${PY_VER}; \ + ./configure \ + --prefix=/usr/local \ + --enable-optimizations \ + --with-lto \ + --with-ensurepip=install; \ + make -j"$(nproc)"; \ + make install; \ + /usr/local/bin/pip3 --no-cache-dir install --upgrade pip; \ + /usr/local/bin/pip3 --no-cache-dir install six + +# ---------- Stage 3: runtime ---------- ARG ALPINE_VERSION=3.20 FROM alpine:${ALPINE_VERSION} LABEL maintainer="Ryan Wallner <ryan.wallner@portworx.com>" -# runtime deps (no cairo/pango/harfbuzz) +# Minimal runtime: keep python3 (from our build), fio, tini, HTTPS tools, +# and libs needed by gnuplot (GD/png/jpeg/freetype/zlib/readline). No jq, no sqlite. +# Prefer 3.20 repos, but add a v3.21 alias for expat only +RUN set -eux; \ + printf '%s\n' \ + "https://dl-cdn.alpinelinux.org/alpine/v3.20/main" \ + "https://dl-cdn.alpinelinux.org/alpine/v3.20/community" \ + "@v321 https://dl-cdn.alpinelinux.org/alpine/v3.21/main" \ + > /etc/apk/repositories + +# (optional) cache-buster so the layer re-runs when you change this arg +ARG APK_EXPAT_VERSION=2.7.2-r0 + RUN set -eux; \ apk add --no-cache \ - fio jq wget ca-certificates tini python3 py3-six \ - gd libpng freetype libjpeg-turbo zlib readline ncurses-libs ttf-dejavu + fio tini ca-certificates wget \ + libgd libpng freetype libjpeg-turbo zlib readline ttf-dejavu \ + "expat@v321>=${APK_EXPAT_VERSION}" -# bring in the self-built gnuplot + its shared data +# sanity check (kept in layer logs) +RUN apk info -v expat && \ + ldd /usr/bin/gnuplot | grep -i expat || true + +# Bring in our self-built gnuplot COPY --from=gnuplot-build /usr/bin/gnuplot /usr/bin/gnuplot COPY --from=gnuplot-build /usr/share/gnuplot /usr/share/gnuplot -# Make fio’s gnuplot templates work headlessly: +# Bring in our self-built Python (no sqlite-libs pulled in) +COPY --from=python-build /usr/local /usr/local + +# Make fio’s gnuplot templates work headlessly / consistently RUN set -eux; \ sed -i 's/^set term .*/set term png size 1280,720/' /usr/share/fio/*.gpm; \ sed -i 's|font "arial"|font "/usr/share/fonts/TTF/DejaVuSans.ttf"|' /usr/share/fio/*.gpm; \ @@ -59,4 +102,3 @@ WORKDIR /tmp/fio-data ENTRYPOINT ["/sbin/tini","--"] CMD ["/bin/sh"] - diff --git a/buildimages.sh b/buildimages.sh index 9a7e955..54019da 100755 --- a/buildimages.sh +++ b/buildimages.sh @@ -5,71 +5,104 @@ NS="${FIOTOOLS_DOCKERHUB_USERNAME:-fiotools}" TAG="${FIOTOOLS_DOCKERHUB_TAG:-tag}" PUSH="${FIOTOOLS_DOCKERHUB_PUSH:-false}" PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}" +DOCKER_CONTEXT="${DOCKER_CONTEXT:-default}" -echo "Using repo: $NS tag: $TAG platforms: $PLATFORMS push: $PUSH" +# NEW: knobs +NO_CACHE="${NO_CACHE:-false}" # set to true/1/yes to force rebuilds +PULL_BASE="${PULL_BASE:-false}" # set to true to always --pull parents +PROGRESS="${PROGRESS:-auto}" # plain|tty|auto -# ---- helpers --------------------------------------------------------------- +echo "Using repo: $NS tag: $TAG platforms: $PLATFORMS push: $PUSH context: $DOCKER_CONTEXT no_cache: $NO_CACHE pull_base: $PULL_BASE" -is_true() { - case "${1,,}" in - 1|true|yes|y) return 0 ;; - *) return 1 ;; - esac -} +tolower(){ printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]'; } +is_true(){ case "$(tolower "$1")" in 1|true|yes|y) return 0;; *) return 1;; esac; } -host_load_platform() { - # choose the platform we can --load into the host daemon +host_platform() { case "$(uname -m)" in - x86_64) echo "linux/amd64" ;; + x86_64|amd64) echo "linux/amd64" ;; aarch64|arm64) echo "linux/arm64" ;; - *) echo "linux/amd64" ;; + *) echo "linux/amd64" ;; esac } -ensure_builder() { - local name="$1" driver="$2" - if ! docker buildx inspect "$name" >/dev/null 2>&1; then - docker buildx create --name "$name" --driver "$driver" --use >/dev/null +dc(){ docker --context="$DOCKER_CONTEXT" "$@"; } + +# ensure we’re on the intended context +docker context use "$DOCKER_CONTEXT" >/dev/null + +# common flags applied to ALL builds +COMMON_FLAGS=(--progress "$PROGRESS") +if is_true "$NO_CACHE"; then + COMMON_FLAGS+=(--no-cache) +fi +if is_true "$PULL_BASE"; then + COMMON_FLAGS+=(--pull) +fi + + +BASE_REF="${NS}/base-fiotools:${TAG}" +GNUPLOT_VERSION="${GNUPLOT_VERSION:-5.4.10}" +PYVER="${PY_VER:-3.12.5}" + +if is_true "$PUSH"; then + # ---------- multi-arch build & push (buildx container driver) ---------- + # builder that can push multi-arch + if ! dc buildx inspect fiotools >/dev/null 2>&1; then + dc buildx create --name fiotools --driver docker-container --use >/dev/null else - docker buildx use "$name" >/dev/null + dc buildx use fiotools >/dev/null fi -} + # qemu for cross-builds (idempotent) + dc run --privileged --rm tonistiigi/binfmt --install all >/dev/null 2>&1 || true -# ---- builder selection ----------------------------------------------------- + # Add attestations (required to satisfy Docker Scout / supply chain) + ATTEST_FLAGS=(--provenance=mode=max --sbom=true) -if is_true "$PUSH"; then - # multi-arch + push -> container driver - ensure_builder fiotools docker-container - # enable qemu emulation for cross-builds (idempotent) - docker run --privileged --rm tonistiigi/binfmt --install all >/dev/null - BUILD_EXTRA=(--push --platform "$PLATFORMS") -else - # local single-arch build that can see host images -> docker driver - ensure_builder fiotools-local docker - LOAD_PLAT="$(host_load_platform)" - BUILD_EXTRA=(--load --platform "$LOAD_PLAT") - echo "Local load platform: $LOAD_PLAT" -fi + bx() { + dc buildx build \ + "${ATTEST_FLAGS[@]}" \ + "${COMMON_FLAGS[@]}" \ + "$@" \ + --label org.opencontainers.image.source="https://github.com/wallnerryan/fio-tools" \ + --label org.opencontainers.image.revision="$(git rev-parse --short HEAD)" \ + --label org.opencontainers.image.created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --label org.opencontainers.image.version="$TAG" + } -build() { - local img="$1" dir="$2" - shift 2 || true - echo "==> Building $img from $dir" - docker buildx build "$dir" -t "$img" "${BUILD_EXTRA[@]}" "$@" -} + # 1) base -> push + bx base-fiotools/ \ + -t "$BASE_REF" \ + --build-arg GNUPLOT_VERSION="$GNUPLOT_VERSION" \ + --build-arg PY_VER="$PYVER" \ + --platform "$PLATFORMS" --push -# ---- build order ----------------------------------------------------------- + # 2) dependents -> push (they’ll pull $BASE_REF from the registry) + bx fio-genplots/ -t "${NS}/fio-genplots:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" --platform "$PLATFORMS" --push + bx fio-tool/ -t "${NS}/fio-tool:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" --platform "$PLATFORMS" --push + bx fio-plotserve/ -t "${NS}/fio-plotserve:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" --platform "$PLATFORMS" --push + bx fiotools-aio/ -t "${NS}/fiotools-aio:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" --platform "$PLATFORMS" --push -# 1) base (must be first) -BASE_REF="${NS}/base-fiotools:${TAG}" -GNUPLOT_VERSION=5.4.10 -build "$BASE_REF" base-fiotools/ --build-arg GNUPLOT_VERSION=$GNUPLOT_VERSION +else + # ---------- local single-arch builds (classic docker build) ---------- + REQ_PLAT="$(echo "$PLATFORMS" | tr -d ' ')" + HOST_PLAT="$(host_platform)" + if [[ "$REQ_PLAT" != "$HOST_PLAT" ]]; then + echo "ERROR: Local load must match host platform. Set PLATFORMS=$HOST_PLAT or set FIOTOOLS_DOCKERHUB_PUSH=true." >&2 + exit 3 + fi -# 2) dependents (pass the base via ARG) -build "${NS}/fio-genplots:${TAG}" fio-genplots/ --build-arg BASE_IMAGE="$BASE_REF" -build "${NS}/fio-tool:${TAG}" fio-tool/ --build-arg BASE_IMAGE="$BASE_REF" -build "${NS}/fio-plotserve:${TAG}" fio-plotserve/ --build-arg BASE_IMAGE="$BASE_REF" -build "${NS}/fiotools-aio:${TAG}" fiotools-aio/ --build-arg BASE_IMAGE="$BASE_REF" + # 1) base -> local daemon + dc build base-fiotools/ \ + -t "$BASE_REF" \ + --build-arg GNUPLOT_VERSION="$GNUPLOT_VERSION" \ + --build-arg PY_VER="$PYVER" \ + "${COMMON_FLAGS[@]}" -echo "Done." + # 2) dependents -> use local base tag; no registry pull needed + dc build fio-genplots/ -t "${NS}/fio-genplots:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" + dc build fio-tool/ -t "${NS}/fio-tool:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" + dc build fio-plotserve/ -t "${NS}/fio-plotserve:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" + dc build fiotools-aio/ -t "${NS}/fiotools-aio:${TAG}" --build-arg BASE_IMAGE="$BASE_REF" +fi +echo "Done."