From 72ec97cf747a2925e6aa738a6c10aa259252656c Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 28 Oct 2023 15:53:47 +0200 Subject: [PATCH] feat: azexpand command (#248) * fix conditionally assign variable * azexpand and azpipelines are now subcommands --- src/Runner.Client/Program.cs | 284 ++++++++++++------ .../Windows/RunnerService.csproj | 2 +- src/Sdk/AzurePipelines/AzureDevops.cs | 20 +- src/Sdk/AzurePipelines/AzurePipelinesUtils.cs | 24 ++ .../ext-core/Program.cs | 2 +- .../ext-core/RequiredParametersProvider.cs | 16 +- 6 files changed, 238 insertions(+), 110 deletions(-) create mode 100644 src/Sdk/AzurePipelines/AzurePipelinesUtils.cs diff --git a/src/Runner.Client/Program.cs b/src/Runner.Client/Program.cs index 7fd4e6a99b8..311e5e4d12e 100644 --- a/src/Runner.Client/Program.cs +++ b/src/Runner.Client/Program.cs @@ -29,6 +29,9 @@ using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using GitHub.Runner.Common; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using Runner.Server.Azure.Devops; +using GitHub.DistributedTask.ObjectTemplating; namespace Runner.Client { @@ -730,6 +733,96 @@ public MyCustomBinder(Func bind) { } return defMismatch; } + + private static bool MatchRepository(string l, string r) + { + var lnameAndRef = l.Split("=", 2).FirstOrDefault()?.Split("@", 2); + var rnameAndRef = r.Split("@", 2); + // Fallback to exact match, this should allow bad repository names to match for local azure tasks + return lnameAndRef?.Length == 2 && rnameAndRef?.Length == 2 ? string.Equals(lnameAndRef[0], rnameAndRef[0], StringComparison.OrdinalIgnoreCase) && lnameAndRef[1] == rnameAndRef[1] : l == r; + } + + private class AzurePipelinesExpander { + + public class MyFileProvider : IFileProvider + { + public MyFileProvider(Parameters handle) { + this.handle = handle; + } + private Parameters handle; + public Task ReadFile(string repositoryAndRef, string path) + { + var reporoot = repositoryAndRef == null ? Path.GetFullPath(handle.Directory ?? ".") : (from r in handle.LocalRepositories where MatchRepository(r, repositoryAndRef) select Path.GetFullPath(r.Substring($"{repositoryAndRef}=".Length))).LastOrDefault(); + if(string.IsNullOrEmpty(reporoot)) { + return null; + } + return File.ReadAllTextAsync(Path.Join(reporoot, path), Encoding.UTF8); + } + } + + public class TraceWriter : GitHub.DistributedTask.ObjectTemplating.ITraceWriter { + public void Error(string format, params object[] args) + { + if(args?.Length == 1 && args[0] is Exception ex) { + Console.Error.WriteLine(string.Format("{0} {1}", format, ex.Message)); + return; + } + try { + Console.Error.WriteLine(args?.Length > 0 ? string.Format(format, args) : format); + } catch { + Console.Error.WriteLine(format); + } + } + + public void Info(string format, params object[] args) + { + try { + Console.WriteLine(args?.Length > 0 ? string.Format(format, args) : format); + } catch { + Console.WriteLine(format); + } + } + + public void Verbose(string format, params object[] args) + { + try { + Console.WriteLine(args?.Length > 0 ? string.Format(format, args) : format); + } catch { + Console.WriteLine(format); + } + } + } + + private class VariablesProvider : IVariablesProvider { + public Dictionary> Variables { get; set; } + public IDictionary GetVariablesForEnvironment(string name = null) { + return Variables?.TryGetValue(name ?? "", out var dict) == true ? dict : null; + } + } + + public static async Task ExpandCurrentPipeline(Parameters handle, string currentFileName) { + var (secs, vars) = await ReadSecretsAndVariables(handle); + var context = new Runner.Server.Azure.Devops.Context { + FileProvider = new MyFileProvider(handle), + TraceWriter = handle.Quiet ? new EmptyTraceWriter() : new TraceWriter(), + Flags = GitHub.DistributedTask.Expressions2.ExpressionFlags.DTExpressionsV1 | GitHub.DistributedTask.Expressions2.ExpressionFlags.ExtendedDirectives, + VariablesProvider = new VariablesProvider { Variables = vars } + }; + Dictionary cparameters = new Dictionary(); + if(handle.Inputs != null) { + foreach(var input in handle.Inputs) { + var opt = input; + var subopt = opt.Split('=', 2); + string varname = subopt[0]; + string varval = subopt.Length == 2 ? subopt[1] : null; + cparameters[varname] = AzurePipelinesUtils.ConvertStringToTemplateToken(varval); + } + } + var template = await AzureDevops.ReadTemplate(context, currentFileName, cparameters); + var pipeline = await new Runner.Server.Azure.Devops.Pipeline().Parse(context.ChildContext(template, currentFileName), template); + return pipeline.ToYaml(); + } + } static int Main(string[] args) { @@ -776,6 +869,9 @@ static int Main(string[] args) "status", "watch", "workflow_run", + // Azure Pipelines Experiments + "azexpand", + "azpipelines", }; var secretOpt = new Option( @@ -1057,7 +1153,8 @@ static int Main(string[] args) // Note that the parameters of the handler method are matched according to the names of the options Func> handler = async (parameters) => { - if(parameters.Parallel == null && !parameters.StartServer && !parameters.List) { + var expandAzurePipeline = string.Equals(parameters.Event, "azexpand", StringComparison.OrdinalIgnoreCase); + if(parameters.Parallel == null && !parameters.StartServer && !parameters.List && !expandAzurePipeline) { parameters.Parallel = 1; } if(parameters.Actor == null) { @@ -1124,7 +1221,7 @@ static int Main(string[] args) } List listener = new List(); try { - if(parameters.Server == null || parameters.StartServer || parameters.StartRunner) { + if(!expandAzurePipeline && (parameters.Server == null || parameters.StartServer || parameters.StartRunner)) { var binpath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); EventHandler _out = (s, e) => { Console.WriteLine(e.Data); @@ -1593,6 +1690,21 @@ await Task.WhenAny(Task.Run(() => { return 1; } } + if(expandAzurePipeline) { + try { + foreach(var workflow in workflows) { + var name = Path.GetRelativePath(parameters.Directory ?? ".", workflow).Replace('\\', '/'); + var res = await AzurePipelinesExpander.ExpandCurrentPipeline(parameters, name); + Console.WriteLine(res); + } + return 0; + } catch (Exception except) { + Console.WriteLine($"Exception: {except.Message}, {except.StackTrace}"); + return 1; + } finally { + cancelWorkflow = null; + } + } try { HttpClientHandler handler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate @@ -1841,86 +1953,11 @@ await Task.WhenAny(Task.Run(() => { } } mp.Add(new StringContent(payloadContent.ToString()), "event", "event.json"); - var envSecrets = new Dictionary>(StringComparer.OrdinalIgnoreCase); - if(parameters.EnvironmentSecretFiles?.Length > 0) { - foreach(var opt in parameters.EnvironmentSecretFiles) { - var subopt = opt.Split('=', 2); - string name = subopt.Length == 2 ? subopt[0] : ""; - string filename = subopt.Length == 2 ? subopt[1] : subopt[0]; - var dict = envSecrets[name] = envSecrets.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - Util.ReadEnvFile(filename, (key, val) => dict[key] = val); - } - } - if(parameters.EnvironmentSecrets?.Length > 0) { - for(int i = 0; i < parameters.EnvironmentSecrets.Length; i++) { - var opt = parameters.EnvironmentSecrets[i]; - var subopt = opt.Split('=', 3); - string name = subopt[0]; - string varname = subopt[1]; - string varval = subopt.Length == 3 ? subopt[2] : null; - if(varval == null) { - await Console.Out.WriteAsync($"{name}={varname}="); - varval = ReadSecret(); - parameters.EnvironmentSecrets[i] = $"{name}={varname}={varval}"; - } - var dict = envSecrets[name] = envSecrets.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - dict[varname] = varval; - } - } + var (envSecrets, envVars) = await ReadSecretsAndVariables(parameters); foreach(var envVarKv in envSecrets) { var ser = new YamlDotNet.Serialization.SerializerBuilder().Build(); mp.Add(new StringContent(ser.Serialize(envVarKv.Value)), "actions-environment-secrets", $"{envVarKv.Key}.secrets"); } - var envVars = new Dictionary>(StringComparer.OrdinalIgnoreCase); - if(parameters.EnvironmentVarFiles?.Length > 0) { - foreach(var opt in parameters.EnvironmentVarFiles) { - var subopt = opt.Split('=', 2); - string name = subopt.Length == 2 ? subopt[0] : ""; - string filename = subopt.Length == 2 ? subopt[1] : subopt[0]; - var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - Util.ReadEnvFile(filename, (key, val) => dict[key] = val); - } - } - if(parameters.VarFiles?.Length > 0) { - foreach(var opt in parameters.VarFiles) { - string name = ""; - string filename = opt; - var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - Util.ReadEnvFile(filename, (key, val) => dict[key] = val); - } - } - if(parameters.EnvironmentVars?.Length > 0) { - for(int i = 0; i < parameters.EnvironmentVars.Length; i++) { - var opt = parameters.EnvironmentVars[i]; - var subopt = opt.Split('=', 3); - string name = subopt[0]; - string varname = subopt[1]; - string varval = subopt.Length == 3 ? subopt[2] : null; - if(varval == null) { - await Console.Out.WriteAsync($"{name}={varname}="); - varval = await Console.In.ReadLineAsync(); - parameters.EnvironmentVars[i] = $"{name}={varname}={varval}"; - } - var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - dict[varname] = varval; - } - } - if(parameters.Vars?.Length > 0) { - for(int i = 0; i < parameters.Vars.Length; i++) { - var opt = parameters.Vars[i]; - var subopt = opt.Split('=', 2); - string name = ""; - string varname = subopt[0]; - string varval = subopt.Length == 2 ? subopt[1] : null; - if(varval == null) { - await Console.Out.WriteAsync($"{varname}="); - varval = await Console.In.ReadLineAsync(); - parameters.Vars[i] = $"{varname}={varval}"; - } - var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); - dict[varname] = varval; - } - } foreach(var envVarKv in envVars) { var ser = new YamlDotNet.Serialization.SerializerBuilder().Build(); mp.Add(new StringContent(ser.Serialize(envVarKv.Value)), "actions-environment-variables", $"{envVarKv.Key}.vars"); @@ -2109,13 +2146,7 @@ await Task.WhenAny(Task.Run(() => { if(line == "event: repodownload") { var endpoint = JsonConvert.DeserializeObject(data); Task.Run(async () => { - Func matchRepository = (l, r) => { - var lnameAndRef = l.Split("=", 2).FirstOrDefault()?.Split("@", 2); - var rnameAndRef = r.Split("@", 2); - // Fallback to exact match, this should allow bad repository names to match for local azure tasks - return lnameAndRef?.Length == 2 && rnameAndRef?.Length == 2 ? string.Equals(lnameAndRef[0], rnameAndRef[0], StringComparison.OrdinalIgnoreCase) && lnameAndRef[1] == rnameAndRef[1] : l == r; - }; - var reporoot = endpoint.Repository == null ? Path.GetFullPath(parameters.Directory ?? ".") : (from r in parameters.LocalRepositories where matchRepository(r, endpoint.Repository) select Path.GetFullPath(r.Substring($"{endpoint.Repository}=".Length))).LastOrDefault(); + var reporoot = endpoint.Repository == null ? Path.GetFullPath(parameters.Directory ?? ".") : (from r in parameters.LocalRepositories where MatchRepository(r, endpoint.Repository) select Path.GetFullPath(r.Substring($"{endpoint.Repository}=".Length))).LastOrDefault(); if(string.Equals(endpoint.Format, "repoexists", StringComparison.OrdinalIgnoreCase) && endpoint.Path == null) { if(reporoot == null) { await client.DeleteAsync(parameters.Server + endpoint.Url, token); @@ -2444,7 +2475,7 @@ await Task.WhenAny(Task.Run(() => { cmd.AddOption(opt); } } - if(ev == "workflow_dispatch") { + if(ev == "workflow_dispatch" || ev == "azexpand") { cmd.AddOption(workflowInputsOpt); } } @@ -2518,6 +2549,87 @@ await Task.WhenAny(Task.Run(() => { return rootCommand.InvokeAsync(args.Length == 1 && args[0] == "--version" ? args : cargs.ToArray()).Result; } + private static async Task<(Dictionary>, Dictionary>)> ReadSecretsAndVariables(Parameters parameters) + { + var envSecrets = new Dictionary>(StringComparer.OrdinalIgnoreCase); + if(parameters.EnvironmentSecretFiles?.Length > 0) { + foreach(var opt in parameters.EnvironmentSecretFiles) { + var subopt = opt.Split('=', 2); + string name = subopt.Length == 2 ? subopt[0] : ""; + string filename = subopt.Length == 2 ? subopt[1] : subopt[0]; + var dict = envSecrets[name] = envSecrets.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + Util.ReadEnvFile(filename, (key, val) => dict[key] = val); + } + } + if(parameters.EnvironmentSecrets?.Length > 0) { + for(int i = 0; i < parameters.EnvironmentSecrets.Length; i++) { + var opt = parameters.EnvironmentSecrets[i]; + var subopt = opt.Split('=', 3); + string name = subopt[0]; + string varname = subopt[1]; + string varval = subopt.Length == 3 ? subopt[2] : null; + if(varval == null) { + await Console.Out.WriteAsync($"{name}={varname}="); + varval = ReadSecret(); + parameters.EnvironmentSecrets[i] = $"{name}={varname}={varval}"; + } + var dict = envSecrets[name] = envSecrets.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + dict[varname] = varval; + } + } + var envVars = new Dictionary>(StringComparer.OrdinalIgnoreCase); + if(parameters.EnvironmentVarFiles?.Length > 0) { + foreach(var opt in parameters.EnvironmentVarFiles) { + var subopt = opt.Split('=', 2); + string name = subopt.Length == 2 ? subopt[0] : ""; + string filename = subopt.Length == 2 ? subopt[1] : subopt[0]; + var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + Util.ReadEnvFile(filename, (key, val) => dict[key] = val); + } + } + if(parameters.VarFiles?.Length > 0) { + foreach(var opt in parameters.VarFiles) { + string name = ""; + string filename = opt; + var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + Util.ReadEnvFile(filename, (key, val) => dict[key] = val); + } + } + if(parameters.EnvironmentVars?.Length > 0) { + for(int i = 0; i < parameters.EnvironmentVars.Length; i++) { + var opt = parameters.EnvironmentVars[i]; + var subopt = opt.Split('=', 3); + string name = subopt[0]; + string varname = subopt[1]; + string varval = subopt.Length == 3 ? subopt[2] : null; + if(varval == null) { + await Console.Out.WriteAsync($"{name}={varname}="); + varval = await Console.In.ReadLineAsync(); + parameters.EnvironmentVars[i] = $"{name}={varname}={varval}"; + } + var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + dict[varname] = varval; + } + } + if(parameters.Vars?.Length > 0) { + for(int i = 0; i < parameters.Vars.Length; i++) { + var opt = parameters.Vars[i]; + var subopt = opt.Split('=', 2); + string name = ""; + string varname = subopt[0]; + string varval = subopt.Length == 2 ? subopt[1] : null; + if(varval == null) { + await Console.Out.WriteAsync($"{varname}="); + varval = await Console.In.ReadLineAsync(); + parameters.Vars[i] = $"{varname}={varval}"; + } + var dict = envVars[name] = envVars.TryGetValue(name, out var v) ? v : new Dictionary(StringComparer.OrdinalIgnoreCase); + dict[varname] = varval; + } + } + return (envSecrets, envVars); + } + private static async Task CollectRepoFiles(string root, string wd, RepoDownload endpoint, MultipartFormDataContent repodownload, List streamsToDispose, long level, Parameters parameters, CancellationTokenSource source) { List> submoduleTasks = new List>(); EventHandler handleoutput = (s, e) => { diff --git a/src/Runner.Service/Windows/RunnerService.csproj b/src/Runner.Service/Windows/RunnerService.csproj index 93a100e7ed9..3b240928cc8 100644 --- a/src/Runner.Service/Windows/RunnerService.csproj +++ b/src/Runner.Service/Windows/RunnerService.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Sdk/AzurePipelines/AzureDevops.cs b/src/Sdk/AzurePipelines/AzureDevops.cs index 05d0194c67b..192aa8e8ddd 100644 --- a/src/Sdk/AzurePipelines/AzureDevops.cs +++ b/src/Sdk/AzurePipelines/AzureDevops.cs @@ -515,13 +515,6 @@ public static async Task ReadTemplate(Runner.Server.Azure.Devops.C variablesData[v.Key] = new StringContextData(v.Value); } } - if(rawStaticVariables != null) { - IDictionary pvars = new Dictionary(StringComparer.OrdinalIgnoreCase); - await ParseVariables(context, pvars, rawStaticVariables, true); - foreach(var v in pvars) { - variablesData[v.Key] = new StringContextData(v.Value.Value); - } - } if(parameters?.Type == TokenType.Mapping) { int providedParameter = 0; @@ -593,6 +586,19 @@ public static async Task ReadTemplate(Runner.Server.Azure.Devops.C templateContext.Errors.Check(); + if(rawStaticVariables != null) { + // See "testworkflows/azpipelines/expressions-docs/Conditionally assign a variable.yml" + templateContext = AzureDevops.CreateTemplateContext(context.TraceWriter ?? new EmptyTraceWriter(), templateContext.GetFileTable().ToArray(), context.Flags, contextData); + rawStaticVariables = TemplateEvaluator.Evaluate(templateContext, "workflow-value", rawStaticVariables, 0, fileId); + templateContext.Errors.Check(); + + IDictionary pvars = new Dictionary(StringComparer.OrdinalIgnoreCase); + await ParseVariables(context, pvars, rawStaticVariables, true); + foreach(var v in pvars) { + variablesData[v.Key] = new StringContextData(v.Value.Value); + } + } + templateContext = AzureDevops.CreateTemplateContext(context.TraceWriter ?? new EmptyTraceWriter(), templateContext.GetFileTable().ToArray(), context.Flags, contextData); var evaluatedResult = TemplateEvaluator.Evaluate(templateContext, schemaName ?? "pipeline-root", pipelineroot, 0, fileId); diff --git a/src/Sdk/AzurePipelines/AzurePipelinesUtils.cs b/src/Sdk/AzurePipelines/AzurePipelinesUtils.cs new file mode 100644 index 00000000000..22509ae2911 --- /dev/null +++ b/src/Sdk/AzurePipelines/AzurePipelinesUtils.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using System.IO; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; + +namespace Runner.Server.Azure.Devops { + + public class AzurePipelinesUtils { + public static TemplateToken ConvertStringToTemplateToken(string res) { + if(res == null) { + return null; + } + var templateContext = AzureDevops.CreateTemplateContext(new EmptyTraceWriter(), new List(), GitHub.DistributedTask.Expressions2.ExpressionFlags.DTExpressionsV1 | GitHub.DistributedTask.Expressions2.ExpressionFlags.ExtendedDirectives); + using (var stringReader = new StringReader(res)) + { + var yamlObjectReader = new YamlObjectReader(null, stringReader, preserveString: true, forceAzurePipelines: true); + var ret = TemplateReader.Read(templateContext, "any", yamlObjectReader, null, out _); + templateContext.Errors.Check(); + return ret; + } + } + } +} \ No newline at end of file diff --git a/src/azure-pipelines-vscode-ext/ext-core/Program.cs b/src/azure-pipelines-vscode-ext/ext-core/Program.cs index e30160bb530..3c85f83e78c 100644 --- a/src/azure-pipelines-vscode-ext/ext-core/Program.cs +++ b/src/azure-pipelines-vscode-ext/ext-core/Program.cs @@ -78,7 +78,7 @@ public static async Task ExpandCurrentPipeline(JSObject handle, string c }; Dictionary cparameters = new Dictionary(); foreach(var kv in JsonConvert.DeserializeObject>(parameters)) { - cparameters[kv.Key] = RequiredParametersProvider.ConvertStringToTemplateToken(kv.Value); + cparameters[kv.Key] = AzurePipelinesUtils.ConvertStringToTemplateToken(kv.Value); } var template = await AzureDevops.ReadTemplate(context, currentFileName, cparameters); var pipeline = await new Runner.Server.Azure.Devops.Pipeline().Parse(context.ChildContext(template, currentFileName), template); diff --git a/src/azure-pipelines-vscode-ext/ext-core/RequiredParametersProvider.cs b/src/azure-pipelines-vscode-ext/ext-core/RequiredParametersProvider.cs index 52f20c1d28b..5bbb007c307 100644 --- a/src/azure-pipelines-vscode-ext/ext-core/RequiredParametersProvider.cs +++ b/src/azure-pipelines-vscode-ext/ext-core/RequiredParametersProvider.cs @@ -14,21 +14,7 @@ public RequiredParametersProvider(JSObject handle) { this.handle = handle; } - public static TemplateToken ConvertStringToTemplateToken(string res) { - if(res == null) { - return null; - } - var templateContext = AzureDevops.CreateTemplateContext(new EmptyTraceWriter(), new List(), GitHub.DistributedTask.Expressions2.ExpressionFlags.DTExpressionsV1 | GitHub.DistributedTask.Expressions2.ExpressionFlags.ExtendedDirectives); - using (var stringReader = new StringReader(res)) - { - var yamlObjectReader = new YamlObjectReader(null, stringReader, preserveString: true, forceAzurePipelines: true); - var ret = TemplateReader.Read(templateContext, "any", yamlObjectReader, null, out _); - templateContext.Errors.Check(); - return ret; - } - } - public async Task GetRequiredParameter(string name) { - return ConvertStringToTemplateToken(await Interop.RequestRequiredParameter(handle, name)); + return AzurePipelinesUtils.ConvertStringToTemplateToken(await Interop.RequestRequiredParameter(handle, name)); } } \ No newline at end of file