From 5eac3b00238b450ac0679121a76f1e566ca8f468 Mon Sep 17 00:00:00 2001

From: Bill O'Donnell <billodo@redhat.com>

Date: Fri, 4 May 2018 14:43:40 -0500

Subject: [PATCH] make fio scripts python3-ready

Many distributions are moving to python3 by default.  Here's

an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas

were hand fixed: remove superfluous parentheses introduced by

2to3 converter in print function calls, shebang modifications

to use environment variable for python version, and byte-string

decode correction in steadystate_tests.py following 2to3

conversion.

The modified scripts pass rudimentary testing when run under

python2.7 as well as python3.

Signed-off-by: Bill O'Donnell <billodo@redhat.com>

Signed-off-by: Jens Axboe <axboe@kernel.dk>

 doc/conf.py                     |  3 +++

 tools/fiologparser.py           |  3 +++

 unit_tests/steadystate_tests.py | 20 ++++++++++++--------

 3 files changed, 18 insertions(+), 8 deletions(-)

From 50d2a37543908e79c41b1d5a4c2cb2d65c5af21b Mon Sep 17 00:00:00 2001

From: Bill O'Donnell <billodo@redhat.com>

Date: Tue, 22 May 2018 12:49:09 -0500

Subject: [PATCH] make fio scripts python3-ready (part 2)

NOTE: this patch (part 2) is in addition to commit 5eac3b00238b

(this patch was missed on that submit/commit (part 1)).

Many distributions are moving to python3 by default.  This is

another patch to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools.

The modified scripts pass rudimentary testing when run under

python2.7 as well as python3.

Signed-off-by: Bill O'Donnell <billodo@redhat.com>

Signed-off-by: Jens Axboe <axboe@kernel.dk>

---

 tools/fio_jsonplus_clat2csv | 13 ++++--

 tools/plot/fio2gnuplot      | 97 ++++++++++++++++++++++++---------------------

 2 files changed, 60 insertions(+), 50 deletions(-)

Index: fio-3.6/doc/conf.py

===================================================================

--- fio-3.6.orig/doc/conf.py

+++ fio-3.6/doc/conf.py

@@ -22,6 +22,9 @@

 # -- General configuration ------------------------------------------------

+from __future__ import absolute_import

+from __future__ import print_function

+

 # If your documentation needs a minimal Sphinx version, state it here.

 #

 # needs_sphinx = '1.0'

Index: fio-3.6/tools/fiologparser.py

===================================================================

--- fio-3.6.orig/tools/fiologparser.py

+++ fio-3.6/tools/fiologparser.py

@@ -1,4 +1,5 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

+# Note: this script is python2 and python 3 compatible.

 #

 # fiologparser.py

 #

@@ -13,6 +14,8 @@

 #

 # to see per-interval average completion latency.

+from __future__ import absolute_import

+from __future__ import print_function

 import argparse

 import math

Index: fio-3.6/unit_tests/steadystate_tests.py

===================================================================

--- fio-3.6.orig/unit_tests/steadystate_tests.py

+++ fio-3.6/unit_tests/steadystate_tests.py

@@ -1,4 +1,5 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

+# Note: this script is python2 and python 3 compatible.

 #

 # steadystate_tests.py

 #

@@ -18,6 +19,8 @@

 # if ss attained: min runtime = ss_dur + ss_ramp

 # if not attained: runtime = timeout

+from __future__ import absolute_import

+from __future__ import print_function

 import os

 import sys

 import json

@@ -26,11 +29,12 @@ import pprint

 import argparse

 import subprocess

 from scipy import stats

+from six.moves import range

 def parse_args():

     parser = argparse.ArgumentParser()

     parser.add_argument('fio',

-                        help='path to fio executable');

+                        help='path to fio executable')

     parser.add_argument('--read',

                         help='target for read testing')

     parser.add_argument('--write',

@@ -45,7 +49,7 @@ def check(data, iops, slope, pct, limit,

     data = data[measurement]

     mean = sum(data) / len(data)

     if slope:

-        x = range(len(data))

+        x = list(range(len(data)))

         m, intercept, r_value, p_value, std_err = stats.linregress(x,data)

         m = abs(m)

         if pct:

@@ -89,11 +93,11 @@ if __name__ == '__main__':

                   'output': "set steady state BW threshold to 12" },

               ]

     for test in parsing:

-        output = subprocess.check_output([args.fio] + test['args']);

-        if test['output'] in output:

-            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])

+        output = subprocess.check_output([args.fio] + test['args'])

+        if test['output'] in output.decode():

+            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))

         else:

-            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

+            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))

 #

 # test some read workloads

@@ -117,7 +121,7 @@ if __name__ == '__main__':

             args.read = '/dev/zero'

             extra = [ "--size=134217728" ]  # 128 MiB

         else:

-            print "ERROR: file for read testing must be specified on non-posix systems"

+            print("ERROR: file for read testing must be specified on non-posix systems")

             sys.exit(1)

     else:

         extra = []

@@ -216,7 +220,7 @@ if __name__ == '__main__':

                 else:

                     result = 'FAILED '

                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)

-            print line

+            print(line)

             if 'steadystate' in jsonjob:

                 pp.pprint(jsonjob['steadystate'])

         jobnum += 1

Index: fio-3.6/tools/fio_jsonplus_clat2csv

===================================================================

--- fio-3.6.orig/tools/fio_jsonplus_clat2csv

+++ fio-3.6/tools/fio_jsonplus_clat2csv

@@ -1,4 +1,5 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

+# Note: this script is python2 and python3 compatible.

 #

 # fio_jsonplus_clat2csv

 #

@@ -60,9 +61,13 @@

 # 10304ns is the 100th percentile for read latency

 #

+from __future__ import absolute_import

+from __future__ import print_function

 import os

 import json

 import argparse

+import six

+from six.moves import range

 def parse_args():

@@ -87,7 +92,7 @@ def percentile(idx, run_total):

 def more_lines(indices, bins):

-    for key, value in indices.iteritems():

+    for key, value in six.iteritems(indices):

         if value < len(bins[key]):

             return True

@@ -116,8 +121,8 @@ def main():

                                    "Are you sure you are using json+ output?")

             bins[ddir] = [[int(key), value] for key, value in

-                          jsondata['jobs'][jobnum][ddir][bins_loc]

-                          ['bins'].iteritems()]

+                          six.iteritems(jsondata['jobs'][jobnum][ddir][bins_loc]

+                          ['bins'])]

             bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0])

             run_total[ddir] = [0 for x in range(0, len(bins[ddir]))]

@@ -165,7 +170,7 @@ def main():

                         output.write(", , , ")

                 output.write("\n")

-            print "{0} generated".format(outfile)

+            print("{0} generated".format(outfile))

 if __name__ == '__main__':

Index: fio-3.6/tools/plot/fio2gnuplot

===================================================================

--- fio-3.6.orig/tools/plot/fio2gnuplot

+++ fio-3.6/tools/plot/fio2gnuplot

@@ -1,4 +1,5 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

+# Note: this script is python2 and python3 compatible.

 #

 #  Copyright (C) 2013 eNovance SAS <licensing@enovance.com>

 #  Author: Erwan Velu  <erwan@enovance.com>

@@ -19,6 +20,8 @@

 #  along with this program; if not, write to the Free Software

 #  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

+from __future__ import absolute_import

+from __future__ import print_function

 import os

 import fnmatch

 import sys

@@ -26,6 +29,8 @@ import getopt

 import re

 import math

 import shutil

+from six.moves import map

+from six.moves import range

 def find_file(path, pattern):

 	fio_data_file=[]

@@ -39,7 +44,7 @@ def find_file(path, pattern):

 	return fio_data_file

 def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):

-	if verbose: print "Generating rendering scripts"

+	if verbose: print("Generating rendering scripts")

 	filename=gnuplot_output_dir+'mygraph'

 	temporary_files.append(filename)

 	f=open(filename,'w')

@@ -124,7 +129,7 @@ def generate_gnuplot_math_script(title,g

 	f.close()

 def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir):

-	if verbose: print "Processing data file 2/2"

+	if verbose: print("Processing data file 2/2")

 	temp_files=[]

 	pos=0

@@ -152,7 +157,7 @@ def compute_temp_file(fio_data_file,disk

 	end_time=max_time

 	if end_time == -1:

 		end_time="infinite"

-	if verbose: print "Processing data file 1/2 with %s

+	if verbose: print("Processing data file 1/2 with %s

 	files=[]

 	temp_outfile=[]

 	blk_size=0

@@ -198,8 +203,8 @@ def compute_temp_file(fio_data_file,disk

 				try:

 					blk_size=int(block_size)

 				except:

-					print "Error while reading the following line :"

-					print line

+					print("Error while reading the following line :")

+					print(line)

 					sys.exit(1);

 			# We ignore the first 500msec as it doesn't seems to be part of the real benchmark

@@ -225,7 +230,7 @@ def compute_temp_file(fio_data_file,disk

 	return blk_size

 def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):

-	if verbose: print "Computing Maths"

+	if verbose: print("Computing Maths")

 	global_min=[]

 	global_max=[]

 	average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w')

@@ -243,14 +248,14 @@ def compute_math(fio_data_file, title,gn

 	max_file.write('DiskName %s\n'% mode)

 	average_file.write('DiskName %s\n'% mode)

 	stddev_file.write('DiskName %s\n'% mode )

-	for disk in xrange(len(fio_data_file)):

+	for disk in range(len(fio_data_file)):

 #		print disk_perf[disk]

 	    	min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))

 	    	max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))

 	    	average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))

 	    	stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))

 		avg  = average(disk_perf[disk])

-		variance = map(lambda x: (x - avg)**2, disk_perf[disk])

+		variance = [(x - avg)**2 for x in disk_perf[disk]]

 		standard_deviation = math.sqrt(average(variance))

 #		print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation)

 		average_file.write('%d %d\n' % (disk, avg))

@@ -264,7 +269,7 @@ def compute_math(fio_data_file, title,gn

 	global_disk_perf = sum(disk_perf, [])

 	avg  = average(global_disk_perf)

-	variance = map(lambda x: (x - avg)**2, global_disk_perf)

+	variance = [(x - avg)**2 for x in global_disk_perf]

 	standard_deviation = math.sqrt(average(variance))

 	global_file.write('min=%.2f\n' % min(global_disk_perf))

@@ -331,52 +336,52 @@ def parse_global_files(fio_data_file, gl

 					max_file=file

 	# Let's print the avg output

 	if global_search == "avg":

-		print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)

+		print("Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file))

 	else:

-		print "Global search %s is not yet implemented\n" % global_search

+		print("Global search %s is not yet implemented\n" % global_search)

 def render_gnuplot(fio_data_file, gnuplot_output_dir):

-	print "Running gnuplot Rendering"

+	print("Running gnuplot Rendering")

 	try:

 		# Let's render all the compared files if some

 		if len(fio_data_file) > 1:

-			if verbose: print " |-> Rendering comparing traces"

+			if verbose: print(" |-> Rendering comparing traces")

 			os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir)

-		if verbose: print " |-> Rendering math traces"

+		if verbose: print(" |-> Rendering math traces")

 		os.system("cd %s; gnuplot mymath" % gnuplot_output_dir)

-		if verbose: print " |-> Rendering 2D & 3D traces"

+		if verbose: print(" |-> Rendering 2D & 3D traces")

 		os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir)

 		name_of_directory="the current"

 		if gnuplot_output_dir != "./":

 			name_of_directory=gnuplot_output_dir

-		print "\nRendering traces are available in %s directory" % name_of_directory

+		print("\nRendering traces are available in %s directory" % name_of_directory)

 		global keep_temp_files

 		keep_temp_files=False

 	except:

-		print "Could not run gnuplot on mymath or mygraph !\n"

+		print("Could not run gnuplot on mymath or mygraph !\n")

 		sys.exit(1);

 def print_help():

-    print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>'

-    print

-    print '-h --help                           : Print this help'

-    print '-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files'

-    print '-b           or --bandwidth         : A predefined pattern for selecting *_bw.log files'

-    print '-i           or --iops              : A predefined pattern for selecting *_iops.log files'

-    print '-g           or --gnuplot           : Render gnuplot traces before exiting'

-    print '-o           or --outputfile <file> : The basename for gnuplot traces'

-    print '                                       - Basename is set with the pattern if defined'

-    print '-d           or --outputdir <dir>   : The directory where gnuplot shall render files'

-    print '-t           or --title <title>     : The title of the gnuplot traces'

-    print '                                       - Title is set with the block size detected in fio traces'

-    print '-G           or --Global <type>     : Search for <type> in .global files match by a pattern'

-    print '                                       - Available types are : min, max, avg, stddev'

-    print '                                       - The .global extension is added automatically to the pattern'

-    print '-m           or --min_time <time>   : Only consider data starting from <time> seconds (default is 0)'

-    print '-M           or --max_time <time>   : Only consider data ending before <time> seconds (default is -1 aka nolimit)'

-    print '-v           or --verbose           : Increasing verbosity'

-    print '-k           or --keep              : Keep all temporary files from gnuplot\'s output dir'

+    print('fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>')

+    print()

+    print('-h --help                           : Print this help')

+    print('-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files')

+    print('-b           or --bandwidth         : A predefined pattern for selecting *_bw.log files')

+    print('-i           or --iops              : A predefined pattern for selecting *_iops.log files')

+    print('-g           or --gnuplot           : Render gnuplot traces before exiting')

+    print('-o           or --outputfile <file> : The basename for gnuplot traces')

+    print('                                       - Basename is set with the pattern if defined')

+    print('-d           or --outputdir <dir>   : The directory where gnuplot shall render files')

+    print('-t           or --title <title>     : The title of the gnuplot traces')

+    print('                                       - Title is set with the block size detected in fio traces')

+    print('-G           or --Global <type>     : Search for <type> in .global files match by a pattern')

+    print('                                       - Available types are : min, max, avg, stddev')

+    print('                                       - The .global extension is added automatically to the pattern')

+    print('-m           or --min_time <time>   : Only consider data starting from <time> seconds (default is 0)')

+    print('-M           or --max_time <time>   : Only consider data ending before <time> seconds (default is -1 aka nolimit)')

+    print('-v           or --verbose           : Increasing verbosity')

+    print('-k           or --keep              : Keep all temporary files from gnuplot\'s output dir')

 def main(argv):

     mode='unknown'

@@ -403,14 +408,14 @@ def main(argv):

     if not os.path.isfile(gpm_dir+'math.gpm'):

 	    gpm_dir="/usr/local/share/fio/"

     	    if not os.path.isfile(gpm_dir+'math.gpm'):

-		    print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n"

+		    print("Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n")

 		    sys.exit(3)

     try:

 	    opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep'])

     except getopt.GetoptError:

-	 print "Error: One of the options passed to the cmdline was not supported"

-	 print "Please fix your command line or read the help (-h option)"

+	 print("Error: One of the options passed to the cmdline was not supported")

+	 print("Please fix your command line or read the help (-h option)")

          sys.exit(2)

     for opt, arg in opts:

@@ -457,7 +462,7 @@ def main(argv):

     fio_data_file=find_file('.',pattern)

     if len(fio_data_file) == 0:

-	    print "No log file found with pattern %s!" % pattern

+	    print("No log file found with pattern %s!" % pattern)

 	    # Try numjob log file format if per_numjob_logs=1

             if (pattern == '*_bw.log'):

                 fio_data_file=find_file('.','*_bw.*.log')

@@ -466,13 +471,13 @@ def main(argv):

             if len(fio_data_file) == 0:

                 sys.exit(1)

             else:

-                print "Using log file per job format instead"

+                print("Using log file per job format instead")

     else:

-	    print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)

+	    print("%d files Selected with pattern '%s'" % (len(fio_data_file), pattern))

     fio_data_file=sorted(fio_data_file, key=str.lower)

     for file in fio_data_file:

-	print ' |-> %s' % file

+	print(' |-> %s' % file)

 	if "_bw.log" in file :

 		mode="Bandwidth (KB/sec)"

 	if "_iops.log" in file :

@@ -483,7 +488,7 @@ def main(argv):

 	    if "IO" in mode:

 		    title='IO benchmark with %d fio results' % len(fio_data_file)

-    print

+    print()

     #We need to adjust the output filename regarding the pattern required by the user

     if (pattern_set_by_user == True):

 	    gnuplot_output_filename=pattern

@@ -514,9 +519,9 @@ def main(argv):

 	# Shall we clean the temporary files ?

 	if keep_temp_files==False and force_keep_temp_files==False:

 	    	# Cleaning temporary files

-		if verbose: print "Cleaning temporary files"

+		if verbose: print("Cleaning temporary files")

 		for f in enumerate(temporary_files):

-		    	if verbose: print " -> %s"%f[1]

+			if verbose: print(" -> %s"%f[1])

 			try:

 			    os.remove(f[1])

 			except:

Index: fio-3.6/tools/hist/fiologparser_hist.py

===================================================================

--- fio-3.6.orig/tools/hist/fiologparser_hist.py

+++ fio-3.6/tools/hist/fiologparser_hist.py

@@ -1,4 +1,4 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

 """ 

     Utility for converting *_clat_hist* files generated by fio into latency statistics.

Index: fio-3.6/tools/hist/half-bins.py

===================================================================

--- fio-3.6.orig/tools/hist/half-bins.py

+++ fio-3.6/tools/hist/half-bins.py

@@ -1,4 +1,4 @@

-#!/usr/bin/python2.7

+#!/usr/bin/python3

 """ Cut the number bins in half in fio histogram output. Example usage:

         $ half-bins.py -c 2 output_clat_hist.1.log > smaller_clat_hist.1.log