From 4f1397154a3eb61df0cbfdf368558f812afab9e2 Mon Sep 17 00:00:00 2001 From: aggieNick02 Date: Mon, 28 Feb 2022 11:49:58 -0600 Subject: [PATCH] Option to record alternate_epoch but keep zero-based timestamps Allow the option to set and record alternate_epoch but leave the timestamps in all log files zero-based. This lets a user record the time at which a job began with the desired clock_id while keeping nice zero-based timestamps in the log files. (The zero-based timestamps are easier to look at visually and also make the log files smaller, and timestamps against the desired clock_id can be reconstructed by the user if desired.) The option log_alternate_eopch_clock_id is renamed to alternate_epoch_clock_id. The new options relating to alternate_epoch have not been in a released version of fio, so I expect this renaming should be ok. Signed-off-by: Nick Neumann nick@pcpartpicker.com --- HOWTO.rst | 23 +++++++++++++++-------- backend.c | 6 +++++- cconv.c | 6 ++++-- fio.1 | 24 +++++++++++++++--------- libfio.c | 6 +++++- options.c | 21 +++++++++++++++------ rate-submit.c | 6 +++++- stat.c | 6 +++++- stat.h | 1 + thread_options.h | 7 +++++-- 10 files changed, 75 insertions(+), 31 deletions(-) diff --git a/HOWTO.rst b/HOWTO.rst index 0978879ce6..3ae7130d10 100644 --- a/HOWTO.rst +++ b/HOWTO.rst @@ -3699,20 +3699,27 @@ Measurements and reporting If set, fio will log Unix timestamps to the log files produced by enabling write_type_log for each log type, instead of the default zero-based - timestamps. + timestamps. alternate_epoch will also be set as a Unix timestamp. .. option:: log_alternate_epoch=bool - If set, fio will log timestamps based on the epoch used by the clock specified - in the log_alternate_epoch_clock_id option, to the log files produced by - enabling write_type_log for each log type, instead of the default zero-based - timestamps. + If set, fio will set alternate_epoch based on the clock specified + in :option:`alternate_epoch_clock_id`. Further, fio will add alternate_epoch + to the timestamps in the log files produced by enabling write_type_log for + each log type. Fio will also report the alternate_epoch in its json output. -.. option:: log_alternate_epoch_clock_id=int +.. option:: record_alternate_epoch=bool + + If set, fio will set alternate_epoch based on the clock specified + in :option:`alternate_epoch_clock_id`, but timestamps in log files produced + by enabling write_type_log for each log type will not be affected. Fio will + report the alternate_epoch in its json output. + +.. option:: alternate_epoch_clock_id=int Specifies the clock_id to be used by clock_gettime to obtain the alternate epoch - if either log_unix_epoch or log_alternate_epoch are true. Otherwise has no - effect. Default value is 0, or CLOCK_REALTIME. + if log_unix_epoch, log_alternate_epoch, or record_alternate_epoch are true. + Otherwise has no effect. Default value is 0, or CLOCK_REALTIME. .. option:: block_error_percentiles=bool diff --git a/backend.c b/backend.c index a21dfef637..cc7494527b 100644 --- a/backend.c +++ b/backend.c @@ -1808,7 +1808,11 @@ static void *thread_main(void *data) if (rate_submit_init(td, sk_out)) goto err; - set_epoch_time(td, o->log_unix_epoch | o->log_alternate_epoch, o->log_alternate_epoch_clock_id); + set_epoch_time(td, + o->log_unix_epoch | + o->log_alternate_epoch | + o->record_alternate_epoch, + o->alternate_epoch_clock_id); fio_getrusage(&td->ru_start); memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch)); memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch)); diff --git a/cconv.c b/cconv.c index 62d02e366e..fa1ab8e0b5 100644 --- a/cconv.c +++ b/cconv.c @@ -198,7 +198,8 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->log_gz_store = le32_to_cpu(top->log_gz_store); o->log_unix_epoch = le32_to_cpu(top->log_unix_epoch); o->log_alternate_epoch = le32_to_cpu(top->log_alternate_epoch); - o->log_alternate_epoch_clock_id = le32_to_cpu(top->log_alternate_epoch_clock_id); + o->record_alternate_epoch = le32_to_cpu(top->record_alternate_epoch); + o->alternate_epoch_clock_id = le32_to_cpu(top->alternate_epoch_clock_id); o->norandommap = le32_to_cpu(top->norandommap); o->softrandommap = le32_to_cpu(top->softrandommap); o->bs_unaligned = le32_to_cpu(top->bs_unaligned); @@ -428,7 +429,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->log_gz_store = cpu_to_le32(o->log_gz_store); top->log_unix_epoch = cpu_to_le32(o->log_unix_epoch); top->log_alternate_epoch = cpu_to_le32(o->log_alternate_epoch); - top->log_alternate_epoch_clock_id = cpu_to_le32(o->log_alternate_epoch_clock_id); + top->record_alternate_epoch = cpu_to_le32(o->record_alternate_epoch); + top->alternate_epoch_clock_id = cpu_to_le32(o->alternate_epoch_clock_id); top->norandommap = cpu_to_le32(o->norandommap); top->softrandommap = cpu_to_le32(o->softrandommap); top->bs_unaligned = cpu_to_le32(o->bs_unaligned); diff --git a/fio.1 b/fio.1 index 984106558e..1588737e72 100644 --- a/fio.1 +++ b/fio.1 @@ -3395,18 +3395,24 @@ parameter. The files will be stored with a `.fz' suffix. .BI log_unix_epoch \fR=\fPbool If set, fio will log Unix timestamps to the log files produced by enabling write_type_log for each log type, instead of the default zero-based -timestamps. +timestamps. alternate_epoch will also be set as a Unix timestamp. .TP .BI log_alternate_epoch \fR=\fPbool -If set, fio will log timestamps based on the epoch used by the clock specified -in the \fBlog_alternate_epoch_clock_id\fR option, to the log files produced by -enabling write_type_log for each log type, instead of the default zero-based -timestamps. -.TP -.BI log_alternate_epoch_clock_id \fR=\fPint +If set, fio will set alternate_epoch based on the clock specified in +\fBlog_alternate_epoch_clock_id\fR. Further, fio will add alternate_epoch to the +timestamps in the log files produced by enabling write_type_log for each log type. +Fio will also report the alternate_epoch in its json output. +.TP +.BI record_alternate_epoch \fR=\fPbool +If set, fio will set alternate_epoch based on the clock specified in +\fBlog_alternate_epoch_clock_id\fR, but timestamps in log files produced by +enabling write_type_log for each log type will not be affected. Fio will report +the alternate_epoch in its json output. +.TP +.BI alternate_epoch_clock_id \fR=\fPint Specifies the clock_id to be used by clock_gettime to obtain the alternate epoch -if either \fBBlog_unix_epoch\fR or \fBlog_alternate_epoch\fR are true. Otherwise has no -effect. Default value is 0, or CLOCK_REALTIME. +if \fBBlog_unix_epoch\fR, \fBlog_alternate_epoch\fR, or \fBrecord_alternate_epoch\fR +are true. Otherwise has no effect. Default value is 0, or CLOCK_REALTIME. .TP .BI block_error_percentiles \fR=\fPbool If set, record errors in trim block-sized units from writes and trims and diff --git a/libfio.c b/libfio.c index 1a8917768b..bef4211921 100644 --- a/libfio.c +++ b/libfio.c @@ -142,7 +142,11 @@ void reset_all_stats(struct thread_data *td) td->ts.runtime[i] = 0; } - set_epoch_time(td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id); + set_epoch_time(td, + td->o.log_unix_epoch | + td->o.log_alternate_epoch | + td->o.record_alternate_epoch, + td->o.alternate_epoch_clock_id); memcpy(&td->start, &td->epoch, sizeof(td->epoch)); memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch)); memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch)); diff --git a/options.c b/options.c index e06d9b66ad..33ed98d6c7 100644 --- a/options.c +++ b/options.c @@ -4510,7 +4510,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .lname = "Log epoch unix", .type = FIO_OPT_BOOL, .off1 = offsetof(struct thread_options, log_unix_epoch), - .help = "Use Unix time in log files", + .help = "Use Unix time in log files. Equivalent to log_alternate_epoch with default value for alternate_epoch_clock_id", .category = FIO_OPT_C_LOG, .group = FIO_OPT_G_INVALID, }, @@ -4519,16 +4519,25 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .lname = "Log epoch alternate", .type = FIO_OPT_BOOL, .off1 = offsetof(struct thread_options, log_alternate_epoch), - .help = "Use alternate epoch time in log files. Uses the same epoch as that is used by clock_gettime with specified log_alternate_epoch_clock_id.", + .help = "Set a job's alternate_epoch using by calling clock_gettime(alternate_epoch_clock_id, &ts) when the job begins, and set alternate_epoch to ts converted to milliseconds. Further, add alternate_epoch to the timestamps in the log files. This means log file timestamps are also using the epoch specified with alternate_epoch_clock_id. Report the alternate_epoch in json output", .category = FIO_OPT_C_LOG, .group = FIO_OPT_G_INVALID, }, { - .name = "log_alternate_epoch_clock_id", - .lname = "Log alternate epoch clock_id", + .name = "record_alternate_epoch", + .lname = "Record alternate epoch", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct thread_options, record_alternate_epoch), + .help = "Set a job's alternate_epoch using by calling clock_gettime(alternate_epoch_clock_id, &ts) when the job begins, and set alternate_epoch to ts converted to milliseconds. Unlike log_alternate_epoch/log_unix_epoch, does not add alternate_epoch to the timestamps in the log files, so the timestamps in logs remain zero-based. Report the alternate_epoch in json output. A true value for log_alternate_epoch/log_unix_epoch functionally overrides this option.", + .category = FIO_OPT_C_LOG, + .group = FIO_OPT_G_INVALID, + }, + { + .name = "alternate_epoch_clock_id", + .lname = "alternate epoch clock_id", .type = FIO_OPT_INT, - .off1 = offsetof(struct thread_options, log_alternate_epoch_clock_id), - .help = "If log_alternate_epoch or log_unix_epoch is true, this option specifies the clock_id from clock_gettime whose epoch should be used. If neither of those is true, this option has no effect. Default value is 0, or CLOCK_REALTIME", + .off1 = offsetof(struct thread_options, alternate_epoch_clock_id), + .help = "If log_alternate_epoch, log_unix_epoch, or record_alternate_epoch is true, this option specifies the clock_id to be used when calling clock_gettime to determine a job's alternate_epoch. If none of those is true, this option has no effect. Default value is 0, or CLOCK_REALTIME", .category = FIO_OPT_C_LOG, .group = FIO_OPT_G_INVALID, }, diff --git a/rate-submit.c b/rate-submit.c index 268356d17a..11e90dc5bd 100644 --- a/rate-submit.c +++ b/rate-submit.c @@ -173,7 +173,11 @@ static int io_workqueue_init_worker_fn(struct submit_worker *sw) if (td->io_ops->post_init && td->io_ops->post_init(td)) goto err_io_init; - set_epoch_time(td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id); + set_epoch_time(td, + td->o.log_unix_epoch | + td->o.log_alternate_epoch | + td->o.record_alternate_epoch, + td->o.alternate_epoch_clock_id); fio_getrusage(&td->ru_start); clear_io_state(td, 1); diff --git a/stat.c b/stat.c index 7947edb42f..ed0fdfa584 100644 --- a/stat.c +++ b/stat.c @@ -1691,6 +1691,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts, root = json_create_object(); json_object_add_value_string(root, "jobname", ts->name); json_object_add_value_int(root, "groupid", ts->groupid); + json_object_add_value_int(root, "alternate_epoch", ts->alternate_epoch); json_object_add_value_int(root, "error", ts->error); /* ETA Info */ @@ -2505,6 +2506,7 @@ void __show_run_stats(void) */ ts->thread_number = td->thread_number; ts->groupid = td->groupid; + ts->alternate_epoch = td->alternate_epoch; /* * first pid in group, not very useful... @@ -3016,11 +3018,13 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data, cur_log = get_cur_log(iolog); if (cur_log) { struct io_sample *s; + bool add_alternate_epoch; + add_alternate_epoch = iolog->td->o.log_alternate_epoch || iolog->td->o.log_unix_epoch; s = get_sample(iolog, cur_log, cur_log->nr_samples); s->data = data; - s->time = t + (iolog->td ? iolog->td->alternate_epoch : 0); + s->time = t + ((iolog->td && add_alternate_epoch) ? iolog->td->alternate_epoch : 0); io_sample_set_ddir(iolog, s, ddir); s->bs = bs; s->priority = priority; diff --git a/stat.h b/stat.h index eb7845afec..fa4262320c 100644 --- a/stat.h +++ b/stat.h @@ -170,6 +170,7 @@ struct thread_stat { uint32_t error; uint32_t thread_number; uint32_t groupid; + unsigned long long alternate_epoch; uint32_t pid; char description[FIO_JOBDESC_SIZE]; uint32_t members; diff --git a/thread_options.h b/thread_options.h index 4162c42faf..c36699d84d 100644 --- a/thread_options.h +++ b/thread_options.h @@ -173,7 +173,8 @@ struct thread_options { unsigned int log_gz_store; unsigned int log_unix_epoch; unsigned int log_alternate_epoch; - unsigned int log_alternate_epoch_clock_id; + unsigned int record_alternate_epoch; + unsigned int alternate_epoch_clock_id; unsigned int norandommap; unsigned int softrandommap; unsigned int bs_unaligned; @@ -491,7 +492,8 @@ struct thread_options_pack { uint32_t log_gz_store; uint32_t log_unix_epoch; uint32_t log_alternate_epoch; - uint32_t log_alternate_epoch_clock_id; + uint32_t record_alternate_epoch; + uint32_t alternate_epoch_clock_id; uint32_t norandommap; uint32_t softrandommap; uint32_t bs_unaligned; @@ -505,6 +507,7 @@ struct thread_options_pack { struct zone_split zone_split[DDIR_RWDIR_CNT][ZONESPLIT_MAX]; uint32_t zone_split_nr[DDIR_RWDIR_CNT]; + uint32_t pad5; fio_fp64_t zipf_theta; fio_fp64_t pareto_h;