From 668033d312d36e49cc8fab176d21c9791654a169 Mon Sep 17 00:00:00 2001 From: aws-viewer-for-cbmc-release-ci Date: Thu, 14 Apr 2022 12:23:09 -0400 Subject: [PATCH 1/4] Create version 1.25 release candidate --- lib/litani.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/litani.py b/lib/litani.py index 1aa2167a..a164da73 100644 --- a/lib/litani.py +++ b/lib/litani.py @@ -33,9 +33,9 @@ TIME_FORMAT_W = "%Y-%m-%dT%H:%M:%SZ" TIME_FORMAT_MS = "%Y-%m-%dT%H:%M:%S.%fZ" VERSION_MAJOR = 1 -VERSION_MINOR = 24 +VERSION_MINOR = 25 VERSION_PATCH = 0 -RC = False +RC = True RC_STR = "-rc" if RC else "" VERSION = "%d.%d.%d%s" % (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH, RC_STR) From fd044d183e9a8b98e2ffd1164df1f764daaafe0a Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Tue, 3 May 2022 08:35:23 +0000 Subject: [PATCH 2/4] Accept lists of input/output files embedded in JSON files If an argument to --inputs or --outputs starts with "@" the remaining argument is now treated as the name of a JSON file to be read. This should help work around the OS limit on the maximum number of command-line arguments. --- doc/src/man/litani-add-job.scdoc | 6 +++++- lib/graph.py | 8 ++++---- lib/litani.py | 18 ++++++++++++++++++ litani | 4 ++-- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/doc/src/man/litani-add-job.scdoc b/doc/src/man/litani-add-job.scdoc index c4191cd2..84ff03ab 100644 --- a/doc/src/man/litani-add-job.scdoc +++ b/doc/src/man/litani-add-job.scdoc @@ -71,6 +71,9 @@ configure times. *--inputs* _F_ [_F_ ...] A list of inputs that this job depends on. Litani interprets each _F_ as a file: + - If _F_ starts with @, then treat the remainder of the file name as JSON + file containing a list of files, which in turn are to be handled as + specified in the following items. - If every _F_ exists and has an older timestamp than all of this job's outputs, then Litani will not run this job. - If some of the _F_ are newer than any of this job's outputs, then those @@ -81,7 +84,8 @@ configure times. out-of-date files, then the build will fail. *--outputs* _F_ [_F_ ...] - A list of outputs that this job emits. Litani interprets each _F_ as a file, + A list of outputs that this job emits. Litani interprets each _F_ as a file + (or a JSON file if prefixed with @, as described for *--inputs* above), and expects that the command will write a file with that name upon completion. If a job _J_ has _F_ as an output, but does not actually write a file called _F_, then _J_ will run unconditionally because _F_ will always be considered diff --git a/lib/graph.py b/lib/graph.py index 35ddb95b..8a2a3a33 100644 --- a/lib/graph.py +++ b/lib/graph.py @@ -177,12 +177,12 @@ def build(self): args["command"]) self.nodes.add(cmd_node) - for inputt in args.get("inputs") or []: + for inputt in lib.litani.expand_args(args.get("inputs")): in_node = lib.graph.DependencyNode(inputt) self.nodes.add(in_node) self.edges.add(lib.graph.Edge(src=in_node, dst=cmd_node)) - for output in args.get("outputs") or []: + for output in lib.litani.expand_args(args.get("outputs")): out_node = lib.graph.DependencyNode(output) self.nodes.add(out_node) self.edges.add(lib.graph.Edge(src=cmd_node, dst=out_node)) @@ -265,12 +265,12 @@ def __str__(self): args["pipeline_name"], args["description"], args["command"]) nodes.add(cmd_node) if args["outputs"]: - for output in args["outputs"]: + for output in lib.litani.expand_args(args["outputs"]): out_node = DependencyNode(output) nodes.add(out_node) edges.add(Edge(src=cmd_node, dst=out_node)) if args["inputs"]: - for inputt in args["inputs"]: + for inputt in lib.litani.expand_args(args["inputs"]): in_node = DependencyNode(inputt) nodes.add(in_node) edges.add(Edge(src=in_node, dst=cmd_node)) diff --git a/lib/litani.py b/lib/litani.py index a164da73..51a6cf6c 100644 --- a/lib/litani.py +++ b/lib/litani.py @@ -269,3 +269,21 @@ def unlink_expired(): # No need to release lock after deletion else: lock_dir.release() + + +def expand_args(files): + """Produce a list of files by expanding any "@"-prefixed file names to their + JSON-list contents. + """ + if not files: + return [] + + result = [] + for f in files: + if f.startswith("@"): + with open(f[1:]) as json_file: + result.extend(json.load(json_file)) + else: + result.append(f) + + return result diff --git a/litani b/litani index c8002d84..a2d4b424 100755 --- a/litani +++ b/litani @@ -444,8 +444,8 @@ def fill_out_ninja(cache, rules, builds, pools): pools[name] = depth for entry in cache["jobs"]: - outs = entry["outputs"] or [] - ins = entry["inputs"] or [] + outs = lib.litani.expand_args(entry["outputs"]) + ins = lib.litani.expand_args(entry["inputs"]) if "description" in entry: description = entry["description"] From 8e6cbd984007262ac87e765fdfbd16e30771733c Mon Sep 17 00:00:00 2001 From: Ronak Fofaliya Date: Wed, 4 May 2022 11:18:56 -0400 Subject: [PATCH 3/4] Keep litani.7 at the first index in report Fixes #154 --- doc/configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/configure b/doc/configure index bb4949f2..d96578fa 100755 --- a/doc/configure +++ b/doc/configure @@ -206,6 +206,10 @@ def main(): add_litani_7(builds, html_mans, args.gen_html) + for h in html_mans: + if "litani.7" in str(h.resolve()): + html_mans.insert(0, html_mans.pop(html_mans.index(h))) + if args.gen_html: builds.append({ "inputs": html_mans + [ From 456cc598930f61d741e85f188d550d92df5bb38c Mon Sep 17 00:00:00 2001 From: Ronak Fofaliya Date: Wed, 4 May 2022 14:38:54 -0400 Subject: [PATCH 4/4] Fix litani run.json and litani outcome-table.json man pages --- doc/configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configure b/doc/configure index d96578fa..f617c75f 100755 --- a/doc/configure +++ b/doc/configure @@ -134,7 +134,7 @@ def man_to_html(man_name, man_out, builds): def convert_man_dir_to_man( src_dir, dst_dir, rule, html_mans, builds, gen_html, extra_inputs=None): for man in (src_dir).iterdir(): - man_name = str(man.name).split(".", 1)[0] + man_name = man.stem if man.suffix == ".scdoc": with open(man) as fp: line = fp.readline().rstrip() @@ -207,7 +207,7 @@ def main(): add_litani_7(builds, html_mans, args.gen_html) for h in html_mans: - if "litani.7" in str(h.resolve()): + if "litani.7" in h.name: html_mans.insert(0, html_mans.pop(html_mans.index(h))) if args.gen_html: