diff --git a/setup.cfg b/setup.cfg index ce8979c8e..72c193d25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ summary = Ceph test framework python_requires = >=3.10 packages = find: install_requires = + GitPython PyYAML ansible-core==2.16.6 apache-libcloud diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 79fd92eda..406769609 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -95,7 +95,7 @@ def current_branch(path: str) -> str: return result -def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error=True): +def enforce_repo_state(dest_clone, dest_path, repo_url, branch, commit=None, remove_on_error=True): """ Use git to either clone or update a given repo, forcing it to switch to the specified branch. @@ -114,25 +114,100 @@ def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error # sentinel to track whether the repo has checked out the intended # version, in addition to being cloned repo_reset = os.path.join(dest_path, '.fetched_and_reset') + log.info("enforce_repo_state %s %s %s %s %s", dest_clone, dest_path, repo_url, branch, commit) try: - if not os.path.isdir(dest_path): - clone_repo(repo_url, dest_path, branch, shallow=commit is None) + if not os.path.isdir(dest_clone): + bare_repo(dest_clone) elif not commit and not is_fresh(sentinel): - set_remote(dest_path, repo_url) - fetch_branch(dest_path, branch) + #set_remote(dest_path, repo_url) + #fetch_branch(dest_path, branch) touch_file(sentinel) - if commit and os.path.exists(repo_reset): - return + #if commit and os.path.exists(repo_reset): + #return - reset_repo(repo_url, dest_path, branch, commit) - touch_file(repo_reset) + myfetch(dest_clone, repo_url, branch, commit) + myworkspace(dest_clone, dest_path) + #reset_repo(repo_url, dest_path, branch, commit) + #touch_file(repo_reset) # remove_pyc_files(dest_path) except (BranchNotFoundError, CommitNotFoundError): if remove_on_error: shutil.rmtree(dest_path, ignore_errors=True) raise +def bare_repo(git_dir): + log.info("bare_repo %s", git_dir) + args = ['git', 'init', '--bare', git_dir] + proc = subprocess.Popen(args) + #args, + #stdout=subprocess.PIPE, + #stderr=subprocess.STDOUT) + if proc.wait() != 0: + raise RuntimeError("oops") + +def myworkspace(git_dir, workspace_dir): + log.info("myworkspace %s %s", git_dir, workspace_dir) + + if os.path.exists(workspace_dir): + args = [ + 'git', + 'log', + '-1', + ] + proc = subprocess.Popen(args,cwd=workspace_dir) + if proc.wait() != 0: + raise RuntimeError("oops") + return + + args = [ + 'git', + 'worktree', + 'add', + #'--detach', + '-B', os.path.basename(workspace_dir), + '--no-track', + '--force', + workspace_dir, + 'FETCH_HEAD' + ] + proc = subprocess.Popen(args,cwd=git_dir) + #args, + #stdout=subprocess.PIPE, + #stderr=subprocess.STDOUT) + if proc.wait() != 0: + raise RuntimeError("oops") + + +def myfetch(git_dir, url, branch, commit=None): + log.info("myfetch %s %s %s %s", git_dir, url, branch, commit) + validate_branch(branch) + if commit is not None: + args = ['git', 'log', '-1', commit] + proc = subprocess.Popen(args, cwd=git_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if proc.wait() == 0: + return + args = ['git', 'fetch', url] + if commit is not None: + args.append(commit) + else: + args.append(branch) + proc = subprocess.Popen(args,cwd=git_dir) + #proc = subprocess.Popen( + #args, + #cwd=git_dir, + #) + #stdout=subprocess.PIPE, + #stderr=subprocess.STDOUT) + if proc.wait() != 0: + not_found_str = "fatal: couldn't find remote ref %s" % branch + out = proc.stdout.read().decode() + log.error(out) + if not_found_str in out.lower(): + raise BranchNotFoundError(branch) + else: + raise GitError("git fetch failed!") + def clone_repo(repo_url, dest_path, branch, shallow=True): """ @@ -354,6 +429,7 @@ def fetch_repo(url, branch, commit=None, bootstrap=None, lock=True): os.mkdir(src_base_path) ref_dir = ref_to_dirname(commit or branch) dirname = '%s_%s' % (url_to_dirname(url), ref_dir) + dest_clone = os.path.join(src_base_path, url_to_dirname(url)) dest_path = os.path.join(src_base_path, dirname) # only let one worker create/update the checkout at a time lock_path = dest_path.rstrip('/') + '.lock' @@ -362,7 +438,8 @@ def fetch_repo(url, branch, commit=None, bootstrap=None, lock=True): try: while proceed(): try: - enforce_repo_state(url, dest_path, branch, commit) + #enforce_repo_state(url, dest_path, branch, commit) + enforce_repo_state(dest_clone, dest_path, url, branch, commit) if bootstrap: sentinel = os.path.join(dest_path, '.bootstrapped') if commit and os.path.exists(sentinel) or is_fresh(sentinel): diff --git a/teuthology/suite/build_graph.py b/teuthology/suite/build_graph.py new file mode 100644 index 000000000..6f9521df6 --- /dev/null +++ b/teuthology/suite/build_graph.py @@ -0,0 +1,200 @@ +import logging +import os +import random +import yaml + +from teuthology.suite import graph + +log = logging.getLogger(__name__) + + +def build_graph(path, subset=None, no_nested_subset=False, seed=None, suite_repo_path=None, config=None): + + + """ + Return a list of items descibed by path such that if the list of + items is chunked into mincyclicity pieces, each piece is still a + good subset of the suite. + + A good subset of a product ensures that each facet member appears + at least once. A good subset of a sum ensures that the subset of + each sub collection reflected in the subset is a good subset. + + A mincyclicity of 0 does not attempt to enforce the good subset + property. + + The input is just a path. The output is an array of (description, + [file list]) tuples. + + For a normal file we generate a new item for the result list. + + For a directory, we (recursively) generate a new item for each + file/dir. + + For a directory with a magic '+' file, we generate a single item + that concatenates all files/subdirs (A Sum). + + For a directory with a magic '%' file, we generate a result set + for each item in the directory, and then do a product to generate + a result list with all combinations (A Product). If the file + contains an integer, it is used as the divisor for a random + subset. + + For a directory with a magic '$' file, or for a directory whose name + ends in '$', we generate a list of all items that we will randomly + choose from. + + The final description (after recursion) for each item will look + like a relative path. If there was a % product, that path + component will appear as a file with braces listing the selection + of chosen subitems. + + :param path: The path to search for yaml fragments + :param subset: (index, outof) + :param no_nested_subset: disable nested subsets + :param seed: The seed for repeatable random test + """ + + if subset: + log.info( + 'Subset=%s/%s' % + (str(subset[0]), str(subset[1])) + ) + if no_nested_subset: + log.info("no_nested_subset") + random.seed(seed) + (which, divisions) = (0,1) if subset is None else subset + G = graph.Graph() + log.info("building graph") + _build_graph(G, path, suite_repo_path=suite_repo_path, config=config) + #log.debug("graph:\n%s", G.print()) This is expensive with the print as an arg. + configs = [] + log.info("walking graph") + for desc, paths in G.walk(which, divisions, no_nested_subset): + log.debug("generated %s", desc) + configs.append((desc, paths)) + log.info("generated %d configs", len(configs)) + return configs + +# To start: let's plug git into Lua so we can inspect versions of Ceph! +# - Use Lua to control how large the subset should be.. based on a target number of jobs.. +# - Use Lua to tag parts of a suite suite that should be included in a broader smoke run. +# - Use Lua to create the graph. + +#Graph +#Lua rewrite +#Change edge randomization based on visitation. Prune before adding to nodes list during walk. +#Set tags at root of graph. Then Lua code in dir prunes at graph creation time. +#Set subset based on target # of jobs +# TODO: maybe reimplement graph.lua so that we can have the graph expand / prune with lua code provided by qa/ suite +# reef.lua: +# git = lupa.git +# function generate() +# ... +# end +# function prune() +# end +def _build_graph(G, path, **kwargs): + flatten = kwargs.pop('flatten', False) + suite_repo_path = kwargs.get('suite_repo_path', None) + config = kwargs.get('config', None) + + if os.path.basename(path)[0] == '.': + return None + if not os.path.exists(path): + raise IOError('%s does not exist (abs %s)' % (path, os.path.abspath(path))) + if os.path.isfile(path): + if path.endswith('.yaml'): + node = graph.Node(path, G) + with open(path) as f: + txt = f.read() + node.set_content(yaml.safe_load(txt)) + return node + if path.endswith('.lua'): + if suite_repo_path is not None: + import git + Gsuite = git.Repo(suite_repo_path) + else: + Gsuite = None + log.info("%s", Gsuite) + node = graph.LuaGraph(path, G, Gsuite) + node.load() + return node + return None + if os.path.isdir(path): + if path.endswith('.disable'): + return None + files = sorted(os.listdir(path)) + if len(files) == 0: + return None + subg = graph.SubGraph(path, G) + specials = ('+', '$', '%') + if '+' in files: + # concatenate items + for s in specials: + if s in files: + files.remove(s) + + current = subg.source + for fn in sorted(files): + node = _build_graph(G, os.path.join(path, fn), flatten=True, **kwargs) + if node: + current.add_edge(node) + current = node + subg.link_node_to_sink(current) + elif path.endswith('$') or '$' in files: + # pick a random item -- make sure we don't pick any magic files + for s in specials: + if s in files: + files.remove(s) + + for fn in sorted(files): + node = _build_graph(G, os.path.join(path, fn), flatten=False, **kwargs) + if node: + subg.source.add_edge(node) # to source + subg.link_node_to_sink(node) # to sink + subg.set_subset(len(files), force=True) + elif '%' in files: + # convolve items + for s in specials: + if s in files: + files.remove(s) + + with open(os.path.join(path, '%')) as f: + divisions = f.read() + if len(divisions) == 0: + divisions = 1 + else: + divisions = int(divisions) + assert divisions > 0 + subg.set_subset(divisions) + + current = subg.source + for fn in sorted(files): + node = _build_graph(G, os.path.join(path, fn), flatten=False, **kwargs) + if node: + current.add_edge(node) + current = node + subg.link_node_to_sink(current) + subg.set_subset(divisions) + else: + # list items + for s in specials: + if s in files: + files.remove(s) + + current = subg.source + for fn in sorted(files): + node = _build_graph(G, os.path.join(path, fn), flatten=flatten, **kwargs) + if node: + current.add_edge(node) # to source + if flatten: + current = node + else: + subg.link_node_to_sink(node) # to sink + if flatten: + subg.link_node_to_sink(current) # to sink + + return subg + + raise RuntimeError(f"Invalid path {path} seen in _build_graph") diff --git a/teuthology/suite/fragment-generate.lua b/teuthology/suite/fragment-generate.lua new file mode 100644 index 000000000..4a68aa68c --- /dev/null +++ b/teuthology/suite/fragment-generate.lua @@ -0,0 +1,53 @@ +-- allow only some Lua (and lunatic) builtins for use by scripts +local SCRIPT_ENV = { + assert = assert, + error = error, + ipairs = ipairs, + next = next, + pairs = pairs, + tonumber = tonumber, + tostring = tostring, + py_attrgetter = python.as_attrgetter, + py_dict = python.builtins.dict, + py_len = python.builtins.len, + py_list = python.builtins.list, + py_tuple = python.builtins.tuple, + py_enumerate = python.enumerate, + py_iterex = python.iterex, + py_itemgetter = python.as_itemgetter, + math = math, +} +local SCRIPT_MT = { + __index = SCRIPT_ENV, +} + +function new_script(script, log, deep_merge, yaml_load) + -- create a restricted sandbox for the script: + local env = setmetatable({ + --deep_merge = deep_merge, + log = log, + --yaml_load = yaml_load, + }, SCRIPT_MT) + + -- avoid putting check_filters in _ENV + -- try to keep line numbers correct: + local header = [[local function main(...) ]] + local footer = [[ end return main]] + local function chunks() + --coroutine.yield(header) + if #script > 0 then + coroutine.yield(script) + end + --coroutine.yield(footer) + end + + print('new_script', script) + + -- put the script in a coroutine so we can yield success/failure from + -- anywhere in the script, including in nested function calls. + local f, err = load(coroutine.wrap(chunks), 'teuthology', 't', env) + if f == nil then + error("failure to load script: "..err) + end + return env, f +end diff --git a/teuthology/suite/graph.py b/teuthology/suite/graph.py new file mode 100644 index 000000000..8f24a260e --- /dev/null +++ b/teuthology/suite/graph.py @@ -0,0 +1,252 @@ +# TODO Tests: +# - that all subsets produce the full suite, no overlap +# - $ behavior +# - % behavior +# - nested dirs + +import bisect +import logging +import os +import random +import re + +log = logging.getLogger(__name__) + +class Graph(object): + def __init__(self): + self.nodes = [] + self.root = None + self.epoch = 0 + + def add_node(self, node): + if 0 == len(self.nodes): + self.root = node + self.nodes.append(node) + self.epoch += 1 + + def add_edge(self, n1, n2): + self.epoch += 1 + + @staticmethod + def collapse_desc(desc): + desc = re.sub(r" +", " ", desc) + desc = re.sub(r"/ {", "/{", desc) + desc = re.sub(r"{ ", "{", desc) + desc = re.sub(r" }", "}", desc) + return desc + + # N.B. avoid recursion because Python function calls are criminally slow. + def walk(self, which, divisions, no_nested_subset): + log.info(f"walking graph {self.root} with {self.path_count()} paths and subset = {which}/{divisions}") + l = random.sample(self.root.outgoing_sorted, k=len(self.root.outgoing_sorted)) + nodes = [(x, 1) for x in l] + path = [(self.root, self.root.subset)] + count = 0 + while nodes: + (node, backtrack) = nodes.pop() + del path[backtrack:] + + parent_divisions = path[-1][1] + nested_divisions = parent_divisions * divisions + current_subset = count % nested_divisions + next_path = (((nested_divisions - current_subset) + which) % nested_divisions) + if node.count <= next_path: + # prune + count = count + node.count + continue + + child_divisions = node.subset if not no_nested_subset or node.force_subset else 1 + path.append((node, parent_divisions * child_divisions)) + if len(node.outgoing) == 0: + assert next_path == 0 + assert current_subset == which + count = count + 1 + desc = [] + frags = [] + for (n, _) in path: + desc.append(n.desc()) + if n.content: + frags.append((n.path, n.content)) + yield Graph.collapse_desc(" ".join(desc)), frags + else: + backtrack_to = len(path) + for n in random.sample(node.outgoing_sorted, k=len(node.outgoing_sorted)): + nodes.append((n, backtrack_to)) + + def path_count(self): + return self.root.path_count(self.epoch) + + def print(self, *args, **kwargs): + return self.root.print(*args, **kwargs) + +class Node(object): + def __init__(self, path, graph): + self.path = path + self.basename = os.path.basename(self.path) + self.name, self.extension = os.path.splitext(self.basename) + self.content = None + self.graph = graph + self.outgoing = set() + self.outgoing_sorted = [] + self.count = 1 + self.subset = 1 + self.force_subset = False + self.draw = True + self.epoch = 0 + self.graph.add_node(self) + self.birth = self.graph.epoch + + def desc(self): + return self.name + + def add_edge(self, node): + if node not in self.outgoing: + self.outgoing.add(node) + # N.B.: a Python set is unordered and we will need to randomize during + # path walks. To make that reproducible with the same seed, the + # shuffled set must be (arbitrarily) ordered first. + bisect.insort(self.outgoing_sorted, node) + self.graph.add_edge(self, node) + + def set_content(self, content): + self.content = content + + def path_count(self, epoch): + if self.epoch < epoch: + count = 0 + for node in self.outgoing: + count = count + node.path_count(epoch) + self.count = max(1, count) + self.epoch = epoch + return self.count + + def __hash__(self): + return hash(id(self)) + + def __eq__(self, other): + if isinstance(other, Node): + return False + return self.path == other.path + + def __lt__(self, other): + if not isinstance(other, Node): + raise TypeError("not comparable") + return self.birth < other.birth + + def __str__(self): + return f"[node paths={self.count} edges={len(self.outgoing)} `{self.path}']" + +class NullNode(Node): + def __init__(self, name, graph): + super().__init__(name, graph) + self.draw = False + + def desc(self): + raise NotImplemented("no desc") + +class SourceNode(NullNode): + def __init__(self, name, graph): + super().__init__(f"source:{name}", graph) + + def desc(self): + return "{" + +class SinkNode(NullNode): + def __init__(self, name, graph): + super().__init__(f"sink:{name}", graph) + + def desc(self): + return "}" + +class SubGraph(Node): + def __init__(self, path, graph): + super().__init__(path, graph) + self.source = SourceNode(path, graph) + self.outgoing.add(self.source) + self.outgoing_sorted = sorted(self.outgoing) + self.sink = SinkNode(path, graph) + self.nodes = set() + self.combinations = 0 + self.count = 0 + + def desc(self): + return f"{self.name}/" + + def set_subset(self, subset, force=False): + # force subset if necessary for e.g. "$" implementation + self.subset = subset + self.force_subset = force + + def add_edge(self, node): + return self.sink.add_edge(node) + + def link_source_to_node(self, node): + self.source.add_edge(node) + + def link_node_to_sink(self, node): + node.add_edge(self.sink) + + @staticmethod + def _nx_add_edge(nxG, node, other, force=False): + if not force and not other.draw: + log.debug(f"_nx_add_edge: skip {other}") + for out in other.outgoing: + SubGraph._nx_add_edge(nxG, node, out, force=force) + else: + log.debug(f"_nx_add_edge: {node} {other}") + nxG.add_edge(node, other) + SubGraph._nx_add_edges(nxG, other, force=force) + + @staticmethod + def _nx_add_edges(nxG, node, force=False): + for out in node.outgoing: + #log.info(f"_nx_add_edges: {node}: {out}") + SubGraph._nx_add_edge(nxG, node, out, force=force) + + def print(self, force=False): + import networkx as nx + #import matplotlib.pyplot as plt + + nxG = nx.DiGraph() + SubGraph._nx_add_edges(nxG, self, force=force) + #log.debug("%s", nxG) + + return "\n".join(nx.generate_network_text(nxG, vertical_chains=True)) + + #pos = nx.spring_layout(nxG) + #nx.draw_networkx_nodes(nxG, pos, node_color='blue', node_size=800) + #nx.draw_networkx_edges(nxG, pos, arrowsize=15) + #nx.draw_networkx_labels(nxG, pos, font_size=12, font_color='black') + + #plt.savefig('graph.svg') + + + +import lupa +import git +import sys +import yaml + +class LuaGraph(SubGraph): + FRAGMENT_GENERATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fragment-generate.lua") + + with open(FRAGMENT_GENERATE) as f: + FRAGMENT_GENERATE_SCRIPT = f.read() + + def __init__(self, path, graph, gSuite): + super().__init__(path, graph) + self.L = lupa.LuaRuntime() + with open(path) as f: + self.script = f.read() + log.info("%s", self.script) + self.gSuite = gSuite + + def load(self): + self.L.execute(self.FRAGMENT_GENERATE_SCRIPT) + new_script = self.L.eval('new_script') + self.env, self.func = new_script(self.script, log) + self.env['graph'] = sys.modules[__name__] + self.env['myself'] = self + self.env['ceph'] = self.gSuite + self.env['yaml'] = yaml + self.func() diff --git a/teuthology/suite/merge.py b/teuthology/suite/merge.py index 0e109af02..95251d935 100644 --- a/teuthology/suite/merge.py +++ b/teuthology/suite/merge.py @@ -132,13 +132,7 @@ def config_merge(configs, suite_name=None, **kwargs): yaml_complete_obj = copy.deepcopy(base_config.to_dict()) deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE)) - for path in paths: - if path not in yaml_cache: - with open(path) as f: - txt = f.read() - yaml_cache[path] = (txt, yaml.safe_load(txt)) - - yaml_fragment_txt, yaml_fragment_obj = yaml_cache[path] + for (path, yaml_fragment_obj) in paths: if yaml_fragment_obj is None: continue yaml_fragment_obj = copy.deepcopy(yaml_fragment_obj) @@ -146,9 +140,9 @@ def config_merge(configs, suite_name=None, **kwargs): if premerge: log.debug("premerge script running:\n%s", premerge) env, script = new_script(premerge, log, deep_merge, yaml.safe_load) - env['base_frag_paths'] = [strip_fragment_path(x) for x in paths] + env['base_frag_paths'] = [strip_fragment_path(path) for (path, yaml) in paths] env['description'] = desc - env['frag_paths'] = paths + env['frag_paths'] = [path for (path, yaml) in paths] env['suite_name'] = suite_name env['yaml'] = yaml_complete_obj env['yaml_fragment'] = yaml_fragment_obj @@ -164,9 +158,9 @@ def config_merge(configs, suite_name=None, **kwargs): postmerge = "\n".join(postmerge) log.debug("postmerge script running:\n%s", postmerge) env, script = new_script(postmerge, log, deep_merge, yaml.safe_load) - env['base_frag_paths'] = [strip_fragment_path(x) for x in paths] + env['base_frag_paths'] = [strip_fragment_path(path) for (path, yaml) in paths] env['description'] = desc - env['frag_paths'] = paths + env['frag_paths'] = [path for (path, yaml) in paths] env['suite_name'] = suite_name env['yaml'] = yaml_complete_obj for k,v in kwargs.items(): diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index ba72a4334..053e30a61 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -21,7 +21,7 @@ from teuthology.suite import util from teuthology.suite.merge import config_merge -from teuthology.suite.build_matrix import build_matrix +from teuthology.suite.build_graph import build_graph from teuthology.suite.placeholder import substitute_placeholders, dict_templ from teuthology.util.time import parse_offset, parse_timestamp, TIMESTAMP_FMT @@ -627,10 +627,12 @@ def schedule_suite(self): if self.args.dry_run: log.debug("Base job config:\n%s" % self.base_config) - configs = build_matrix(suite_path, - subset=self.args.subset, - no_nested_subset=self.args.no_nested_subset, - seed=self.args.seed) + configs = build_graph(suite_path, + subset=self.args.subset, + no_nested_subset=self.args.no_nested_subset, + seed=self.args.seed, + suite_repo_path=self.suite_repo_path, + config=config) generated = len(configs) log.info(f'Suite {suite_name} in {suite_path} generated {generated} jobs (not yet filtered or merged)') configs = list(config_merge(configs,