From 701e7e2bb6017e5995c2cf09b2574ddf2606ce40 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Nov 03 2020 12:03:19 +0000 Subject: import fio-3.19-3.el8 --- diff --git a/.fio.metadata b/.fio.metadata index 92ca5ef..e839755 100644 --- a/.fio.metadata +++ b/.fio.metadata @@ -1 +1 @@ -e4e6d653bd851d1d4652cfe3de3008cf583025e7 SOURCES/fio-3.7.tar.bz2 +cad3b3d78c5c2c9a116ee53642eec17a77c5ead2 SOURCES/fio-3.19.tar.bz2 diff --git a/.gitignore b/.gitignore index 862008b..715dc75 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -SOURCES/fio-3.7.tar.bz2 +SOURCES/fio-3.19.tar.bz2 diff --git a/SOURCES/0001-Add-option-latency_run-to-continue-enable-latency_ta.patch b/SOURCES/0001-Add-option-latency_run-to-continue-enable-latency_ta.patch new file mode 100644 index 0000000..6e2877b --- /dev/null +++ b/SOURCES/0001-Add-option-latency_run-to-continue-enable-latency_ta.patch @@ -0,0 +1,187 @@ +From e1bcd541f63f9029f6c50116831303ad06292edc Mon Sep 17 00:00:00 2001 +From: Song Liu +Date: Sun, 17 May 2020 22:39:49 -0700 +Subject: [PATCH] Add option latency_run to continue enable latency_target + +Currently, latency_target run will exist once fio find the highest queue +depth that meets latency_target. Add option latency_run. If set, fio will +continue running and try to meet latency_target by adusting queue depth. + +Signed-off-by: Song Liu +--- + HOWTO | 7 +++++++ + cconv.c | 2 ++ + fio.1 | 5 +++++ + fio.h | 1 + + io_u.c | 18 +++++++++++++++++- + options.c | 10 ++++++++++ + server.h | 2 +- + thread_options.h | 2 ++ + 8 files changed, 45 insertions(+), 2 deletions(-) + +diff --git a/HOWTO b/HOWTO +index 430c7b62..f0b4ffe4 100644 +--- a/HOWTO ++++ b/HOWTO +@@ -2551,6 +2551,13 @@ I/O latency + defaults to 100.0, meaning that all I/Os must be equal or below to the value + set by :option:`latency_target`. + ++.. option:: latency_run=bool ++ ++ Used with :option:`latency_target`. If false (default), fio will find ++ the highest queue depth that meets :option:`latency_target` and exit. If ++ true, fio will continue running and try to meet :option:`latency_target` ++ by adjusting queue depth. ++ + .. option:: max_latency=time + + If set, fio will exit the job with an ETIMEDOUT error if it exceeds this +diff --git a/cconv.c b/cconv.c +index 48218dc4..449bcf7b 100644 +--- a/cconv.c ++++ b/cconv.c +@@ -288,6 +288,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, + o->latency_window = le64_to_cpu(top->latency_window); + o->max_latency = le64_to_cpu(top->max_latency); + o->latency_percentile.u.f = fio_uint64_to_double(le64_to_cpu(top->latency_percentile.u.i)); ++ o->latency_run = le32_to_cpu(top->latency_run); + o->compress_percentage = le32_to_cpu(top->compress_percentage); + o->compress_chunk = le32_to_cpu(top->compress_chunk); + o->dedupe_percentage = le32_to_cpu(top->dedupe_percentage); +@@ -487,6 +488,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, + top->latency_window = __cpu_to_le64(o->latency_window); + top->max_latency = __cpu_to_le64(o->max_latency); + top->latency_percentile.u.i = __cpu_to_le64(fio_double_to_uint64(o->latency_percentile.u.f)); ++ top->latency_run = __cpu_to_le32(o->latency_run); + top->compress_percentage = cpu_to_le32(o->compress_percentage); + top->compress_chunk = cpu_to_le32(o->compress_chunk); + top->dedupe_percentage = cpu_to_le32(o->dedupe_percentage); +diff --git a/fio.1 b/fio.1 +index a2379f98..3a7a359b 100644 +--- a/fio.1 ++++ b/fio.1 +@@ -2275,6 +2275,11 @@ The percentage of I/Os that must fall within the criteria specified by + defaults to 100.0, meaning that all I/Os must be equal or below to the value + set by \fBlatency_target\fR. + .TP ++.BI latency_run \fR=\fPbool ++Used with \fBlatency_target\fR. If false (default), fio will find the highest ++queue depth that meets \fBlatency_target\fR and exit. If true, fio will continue ++running and try to meet \fBlatency_target\fR by adjusting queue depth. ++.TP + .BI max_latency \fR=\fPtime + If set, fio will exit the job with an ETIMEDOUT error if it exceeds this + maximum latency. When the unit is omitted, the value is interpreted in +diff --git a/fio.h b/fio.h +index bbf057c1..7610026d 100644 +--- a/fio.h ++++ b/fio.h +@@ -377,6 +377,7 @@ struct thread_data { + unsigned int latency_qd_high; + unsigned int latency_qd_low; + unsigned int latency_failed; ++ unsigned int latency_stable_count; + uint64_t latency_ios; + int latency_end_run; + +diff --git a/io_u.c b/io_u.c +index aa8808b8..ae1438fd 100644 +--- a/io_u.c ++++ b/io_u.c +@@ -1391,6 +1391,7 @@ static bool __lat_target_failed(struct thread_data *td) + td->latency_qd_low--; + + td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2; ++ td->latency_stable_count = 0; + + dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high); + +@@ -1440,6 +1441,21 @@ static void lat_target_success(struct thread_data *td) + + td->latency_qd_low = td->latency_qd; + ++ if (td->latency_qd + 1 == td->latency_qd_high) { ++ /* ++ * latency_qd will not incease on lat_target_success(), so ++ * called stable. If we stick with this queue depth, the ++ * final latency is likely lower than latency_target. Fix ++ * this by increasing latency_qd_high slowly. Use a naive ++ * heuristic here. If we get lat_target_success() 3 times ++ * in a row, increase latency_qd_high by 1. ++ */ ++ if (++td->latency_stable_count >= 3) { ++ td->latency_qd_high++; ++ td->latency_stable_count = 0; ++ } ++ } ++ + /* + * If we haven't failed yet, we double up to a failing value instead + * of bisecting from highest possible queue depth. If we have set +@@ -1459,7 +1475,7 @@ static void lat_target_success(struct thread_data *td) + * Same as last one, we are done. Let it run a latency cycle, so + * we get only the results from the targeted depth. + */ +- if (td->latency_qd == qd) { ++ if (!o->latency_run && td->latency_qd == qd) { + if (td->latency_end_run) { + dprint(FD_RATE, "We are done\n"); + td->done = 1; +diff --git a/options.c b/options.c +index b18cea33..da401aed 100644 +--- a/options.c ++++ b/options.c +@@ -3672,6 +3672,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_LATPROF, + }, ++ { ++ .name = "latency_run", ++ .lname = "Latency Run", ++ .type = FIO_OPT_BOOL, ++ .off1 = offsetof(struct thread_options, latency_run), ++ .help = "Keep adjusting queue depth to match latency_target", ++ .def = "0", ++ .category = FIO_OPT_C_IO, ++ .group = FIO_OPT_G_LATPROF, ++ }, + { + .name = "invalidate", + .lname = "Cache invalidate", +diff --git a/server.h b/server.h +index 279b6917..de01a5c8 100644 +--- a/server.h ++++ b/server.h +@@ -48,7 +48,7 @@ struct fio_net_cmd_reply { + }; + + enum { +- FIO_SERVER_VER = 82, ++ FIO_SERVER_VER = 83, + + FIO_SERVER_MAX_FRAGMENT_PDU = 1024, + FIO_SERVER_MAX_CMD_MB = 2048, +diff --git a/thread_options.h b/thread_options.h +index c78ed43d..09ccd5b2 100644 +--- a/thread_options.h ++++ b/thread_options.h +@@ -324,6 +324,7 @@ struct thread_options { + unsigned long long latency_target; + unsigned long long latency_window; + fio_fp64_t latency_percentile; ++ uint32_t latency_run; + + unsigned int sig_figs; + +@@ -612,6 +613,7 @@ struct thread_options_pack { + uint64_t latency_window; + uint64_t max_latency; + fio_fp64_t latency_percentile; ++ uint32_t latency_run; + + uint32_t sig_figs; + +-- +2.17.0 + diff --git a/SOURCES/0001-Unbreak-the-pmemblk-engine.patch b/SOURCES/0001-Unbreak-the-pmemblk-engine.patch new file mode 100644 index 0000000..832d2b1 --- /dev/null +++ b/SOURCES/0001-Unbreak-the-pmemblk-engine.patch @@ -0,0 +1,37 @@ +From 8644ef7c4c49aa6d6492b3b250a06b841496d7fd Mon Sep 17 00:00:00 2001 +From: Bart Van Assche +Date: Sat, 27 Jun 2020 07:26:24 -0700 +Subject: [PATCH] Unbreak the pmemblk engine + +Reported-by: Yi Zhang +Tested-by: Yi Zhang +Fixes: e9c7be0e32e6 ("pmemblk: Fix a memory leak") +Signed-off-by: Bart Van Assche +--- + engines/pmemblk.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/engines/pmemblk.c b/engines/pmemblk.c +index 730f4d7..e2eaa15 100644 +--- a/engines/pmemblk.c ++++ b/engines/pmemblk.c +@@ -220,14 +220,14 @@ static fio_pmemblk_file_t pmb_open(const char *pathspec, int flags) + pmb->pmb_nblocks = pmemblk_nblock(pmb->pmb_pool); + + fio_pmemblk_cache_insert(pmb); ++ } else { ++ free(path); + } + + pmb->pmb_refcnt += 1; + + pthread_mutex_unlock(&CacheLock); + +- free(path); +- + return pmb; + + error: +-- +2.9.5 + diff --git a/SOURCES/0001-init-fix-unit-of-latency_window.patch b/SOURCES/0001-init-fix-unit-of-latency_window.patch new file mode 100644 index 0000000..524d1e4 --- /dev/null +++ b/SOURCES/0001-init-fix-unit-of-latency_window.patch @@ -0,0 +1,28 @@ +From 2e3fb343ec883674a4927f2da983759bf90a0671 Mon Sep 17 00:00:00 2001 +From: Song Liu +Date: Sun, 17 May 2020 22:46:21 -0700 +Subject: [PATCH] init: fix unit of latency_window + +latency_window has unit of microseconds, and is compared against +usec_window. Therefore, there is no need to fix it up to nanoseconds. + +Signed-off-by: Song Liu +--- + init.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/init.c b/init.c +index b5315334..0431f700 100644 +--- a/init.c ++++ b/init.c +@@ -956,7 +956,6 @@ static int fixup_options(struct thread_data *td) + */ + o->max_latency *= 1000ULL; + o->latency_target *= 1000ULL; +- o->latency_window *= 1000ULL; + + return ret; + } +-- +2.17.0 + diff --git a/SOURCES/0001-thread_options-Use-unsigned-int-type-for-exit_what-a.patch b/SOURCES/0001-thread_options-Use-unsigned-int-type-for-exit_what-a.patch new file mode 100644 index 0000000..641b5b3 --- /dev/null +++ b/SOURCES/0001-thread_options-Use-unsigned-int-type-for-exit_what-a.patch @@ -0,0 +1,178 @@ +From fd56c235caa42870e6dc33d661514375ea95ffc5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Andr=C3=A9=20Wild?= +Date: Fri, 14 Aug 2020 15:52:09 +0200 +Subject: [PATCH] thread_options: Use unsigned int type for exit_what and + stonewall +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fixes: 64402a8 ("Expand choices for exitall") +Fixes: https://github.com/axboe/fio/issues/1065 +Signed-off-by: André Wild +--- + cconv.c | 8 ++++---- + examples/exitwhat.fio | 8 ++++---- + fio.1 | 29 +++++++++++++++++++++-------- + server.h | 2 +- + thread_options.h | 9 ++++----- + 5 files changed, 34 insertions(+), 22 deletions(-) + +diff --git a/cconv.c b/cconv.c +index 2469389b..4b0c3490 100644 +--- a/cconv.c ++++ b/cconv.c +@@ -237,8 +237,8 @@ void convert_thread_options_to_cpu(struct thread_options *o, + o->loops = le32_to_cpu(top->loops); + o->mem_type = le32_to_cpu(top->mem_type); + o->mem_align = le32_to_cpu(top->mem_align); +- o->exit_what = le16_to_cpu(top->exit_what); +- o->stonewall = le16_to_cpu(top->stonewall); ++ o->exit_what = le32_to_cpu(top->exit_what); ++ o->stonewall = le32_to_cpu(top->stonewall); + o->new_group = le32_to_cpu(top->new_group); + o->numjobs = le32_to_cpu(top->numjobs); + o->cpus_allowed_policy = le32_to_cpu(top->cpus_allowed_policy); +@@ -437,8 +437,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top, + top->loops = cpu_to_le32(o->loops); + top->mem_type = cpu_to_le32(o->mem_type); + top->mem_align = cpu_to_le32(o->mem_align); +- top->exit_what = cpu_to_le16(o->exit_what); +- top->stonewall = cpu_to_le16(o->stonewall); ++ top->exit_what = cpu_to_le32(o->exit_what); ++ top->stonewall = cpu_to_le32(o->stonewall); + top->new_group = cpu_to_le32(o->new_group); + top->numjobs = cpu_to_le32(o->numjobs); + top->cpus_allowed_policy = cpu_to_le32(o->cpus_allowed_policy); +diff --git a/examples/exitwhat.fio b/examples/exitwhat.fio +index a1099f0f..c91d7375 100644 +--- a/examples/exitwhat.fio ++++ b/examples/exitwhat.fio +@@ -1,7 +1,7 @@ + # We want to run fast1 as long as slow1 is running, but also have a cumulative + # report of fast1 (group_reporting=1/new_group=1). exitall=1 would not cause + # fast1 to stop after slow1 is done. Setting exit_what=stonewall will cause +-# alls jobs up until the next stonewall=1 setting to be stopped, when job slow1 ++# alls jobs up until the next stonewall setting to be stopped, when job slow1 + # finishes. + # In this example skipping forward to slow2/fast2. slow2 has exit_what=all set, + # which means all jobs will be cancelled when slow2 finishes. In particular, +@@ -15,7 +15,7 @@ group_reporting=1 + exitall=1 + + [slow1] +-rw=r ++rw=read + numjobs=1 + ioengine=sync + new_group=1 +@@ -32,8 +32,8 @@ iodepth=32 + rate=300,300,300 + + [slow2] +-stonewall=1 +-rw=w ++stonewall ++rw=write + numjobs=1 + ioengine=sync + new_group=1 +diff --git a/fio.1 b/fio.1 +index cdd105d7..1c90e4a5 100644 +--- a/fio.1 ++++ b/fio.1 +@@ -2569,7 +2569,8 @@ been exceeded before retrying operations. + Wait for preceding jobs in the job file to exit, before starting this + one. Can be used to insert serialization points in the job file. A stone + wall also implies starting a new reporting group, see +-\fBgroup_reporting\fR. ++\fBgroup_reporting\fR. Optionally you can use `stonewall=0` to disable or ++`stonewall=1` to enable it. + .TP + .BI exitall + By default, fio will continue running all other jobs when one job finishes. +@@ -2577,15 +2578,27 @@ Sometimes this is not the desired action. Setting \fBexitall\fR will instead + make fio terminate all jobs in the same group, as soon as one job of that + group finishes. + .TP +-.BI exit_what ++.BI exit_what \fR=\fPstr + By default, fio will continue running all other jobs when one job finishes. +-Sometimes this is not the desired action. Setting \fBexit_all\fR will instead ++Sometimes this is not the desired action. Setting \fBexitall\fR will instead + make fio terminate all jobs in the same group. The option \fBexit_what\fR +-allows to control which jobs get terminated when \fBexitall\fR is enabled. The +-default is \fBgroup\fR and does not change the behaviour of \fBexitall\fR. The +-setting \fBall\fR terminates all jobs. The setting \fBstonewall\fR terminates +-all currently running jobs across all groups and continues execution with the +-next stonewalled group. ++allows you to control which jobs get terminated when \fBexitall\fR is enabled. ++The default value is \fBgroup\fR. ++The allowed values are: ++.RS ++.RS ++.TP ++.B all ++terminates all jobs. ++.TP ++.B group ++is the default and does not change the behaviour of \fBexitall\fR. ++.TP ++.B stonewall ++terminates all currently running jobs across all groups and continues ++execution with the next stonewalled group. ++.RE ++.RE + .TP + .BI exec_prerun \fR=\fPstr + Before running this job, issue the command specified through +diff --git a/server.h b/server.h +index de01a5c8..efa70e7c 100644 +--- a/server.h ++++ b/server.h +@@ -48,7 +48,7 @@ struct fio_net_cmd_reply { + }; + + enum { +- FIO_SERVER_VER = 83, ++ FIO_SERVER_VER = 84, + + FIO_SERVER_MAX_FRAGMENT_PDU = 1024, + FIO_SERVER_MAX_CMD_MB = 2048, +diff --git a/thread_options.h b/thread_options.h +index 3fe48ecc..14f1cbe9 100644 +--- a/thread_options.h ++++ b/thread_options.h +@@ -202,8 +202,8 @@ struct thread_options { + + unsigned long long max_latency; + +- unsigned short exit_what; +- unsigned short stonewall; ++ unsigned int exit_what; ++ unsigned int stonewall; + unsigned int new_group; + unsigned int numjobs; + os_cpu_mask_t cpumask; +@@ -494,8 +494,8 @@ struct thread_options_pack { + uint32_t mem_type; + uint32_t mem_align; + +- uint16_t exit_what; +- uint16_t stonewall; ++ uint32_t exit_what; ++ uint32_t stonewall; + uint32_t new_group; + uint32_t numjobs; + /* +@@ -546,7 +546,6 @@ struct thread_options_pack { + uint32_t lat_percentiles; + uint32_t slat_percentiles; + uint32_t percentile_precision; +- uint32_t pad3; + fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN]; + + uint8_t read_iolog_file[FIO_TOP_STR_MAX]; +-- +2.17.0 + diff --git a/SOURCES/fio-eliminate-extra-log-samples.patch b/SOURCES/fio-eliminate-extra-log-samples.patch new file mode 100644 index 0000000..b9f3b99 --- /dev/null +++ b/SOURCES/fio-eliminate-extra-log-samples.patch @@ -0,0 +1,52 @@ +From 640150c1b2c3cdbdd8baa5f1f3e7214a5c9a6533 Mon Sep 17 00:00:00 2001 +From: Vincent Fu +Date: Tue, 31 Mar 2020 07:26:16 -0400 +Subject: [PATCH] stat: eliminate extra log samples + +b2a432bfbb6d inadvertently added extra log samples. + +$ ./fio-canonical/fio --name=test --time_based --runtime=10s --write_lat_log=fio-07-b2a432 --log_avg_msec=1000 --size=1G --rw=rw +test: (g=0): rw=rw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=1 +fio-3.17-93-gb2a4 +Starting 1 process +... +$ cat fio-07-b2a432_clat.1.log +1000, 5851, 0, 0, 0 +1000, 2551, 1, 0, 0 +1000, 5028, 1, 0, 0 +2000, 4175, 0, 0, 0 +2000, 3214, 1, 0, 0 +2000, 60619, 0, 0, 0 +... + +There should only be two lines at each timestamp (one for reads, one for +writes), but the first two timestamps have three lines each. + +The cause is an inadvertent change in stat.c:add_log_sample() of +__add_stat_to_log to _add_stat_to_log. Reverting to the two-underscore +version resolves this issue. + +Fixes: https://github.com/axboe/fio/issues/947 +Fixes: b2a432bfbb6d ("Per-command priority: Priority logging and libaio/io_uring cmdprio_percentage") +Signed-off-by: Vincent Fu +--- + stat.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/stat.c b/stat.c +index d8c01d14..efa811d2 100644 +--- a/stat.c ++++ b/stat.c +@@ -2749,7 +2749,7 @@ static unsigned long add_log_sample(struct thread_data *td, + return diff; + } + +- _add_stat_to_log(iolog, elapsed, td->o.log_max != 0, priority_bit); ++ __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0, priority_bit); + + iolog->avg_last[ddir] = elapsed - (this_window - iolog->avg_msec); + return iolog->avg_msec; +-- +2.17.1 + + diff --git a/SPECS/fio.spec b/SPECS/fio.spec index 8da9e0e..18685ed 100644 --- a/SPECS/fio.spec +++ b/SPECS/fio.spec @@ -1,5 +1,5 @@ Name: fio -Version: 3.7 +Version: 3.19 Release: 3%{?dist} Summary: Multithreaded IO generation tool @@ -8,10 +8,16 @@ License: GPLv2 URL: http://git.kernel.dk/?p=fio.git;a=summary Source: http://brick.kernel.dk/snaps/%{name}-%{version}.tar.bz2 +Patch0: fio-eliminate-extra-log-samples.patch +Patch1: 0001-Unbreak-the-pmemblk-engine.patch +Patch2: 0001-init-fix-unit-of-latency_window.patch +Patch3: 0001-Add-option-latency_run-to-continue-enable-latency_ta.patch +Patch4: 0001-thread_options-Use-unsigned-int-type-for-exit_what-a.patch + BuildRequires: gcc BuildRequires: libaio-devel -BuildRequires: python3-devel BuildRequires: zlib-devel +BuildRequires: python3-devel %ifarch x86_64 BuildRequires: libpmem-devel BuildRequires: libpmemblk-devel @@ -32,13 +38,19 @@ one wants to simulate. %prep %setup -q +%patch0 -p1 +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 pathfix.py -i %{__python3} -pn \ + doc/conf.py \ tools/fio_jsonplus_clat2csv \ tools/fiologparser.py \ tools/hist/*.py \ tools/plot/fio2gnuplot \ - unit_tests/steadystate_tests.py + t/*.py %build ./configure --disable-optimizations @@ -56,6 +68,18 @@ make install prefix=%{_prefix} mandir=%{_mandir} DESTDIR=$RPM_BUILD_ROOT INSTALL %{_datadir}/%{name}/* %changelog +* Thu Aug 20 2020 Eric Sandeen 3.19-3 +- Fix regression in stonewall (#1869305) + +* Tue Jul 14 2020 Eric Sandeen 3.19-2 +- Fix regression in pmemblk engine (#1846843) + +* Mon Apr 20 2020 Eric Sandeen 3.19-1 +- Rebase to new upstream + bugfix + +* Fri Jun 07 2019 Eric Sandeen 3.7-5 +- Rebuild w/ tests in place (#1681954) + * Wed Aug 01 2018 Charalampos Stratakis - 3.7-3 - Fix python shebangs in a more portable way