From 28558e674f60fef1b165a79c039b1b450759d500 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 14 Feb 2025 15:42:18 +0100 Subject: [PATCH] feat: parse script for preprocessor/no_main_func on deploy (#5292) * feat: parse script for preprocessor/no_main_func on deploy * fix substitute for linux --- backend/Cargo.lock | 1 + backend/parsers/windmill-parser-py/src/lib.rs | 27 ++++++------ backend/parsers/windmill-parser-ts/src/lib.rs | 42 +++++++++---------- .../parsers/windmill-parser-wasm/src/lib.rs | 4 +- .../windmill-parser-wasm/tests/wasm.rs | 8 ++-- backend/substitute_ee_code.sh | 4 +- backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/src/scripts.rs | 17 +++++++- backend/windmill-worker/src/bun_executor.rs | 4 +- backend/windmill-worker/src/deno_executor.rs | 4 +- backend/windmill-worker/src/js_eval.rs | 2 +- .../windmill-worker/src/python_executor.rs | 4 +- cli/metadata.ts | 16 +++++-- 13 files changed, 85 insertions(+), 49 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d0bf892dd8132..9c02a43c36f53 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10988,6 +10988,7 @@ dependencies = [ "windmill-git-sync", "windmill-indexer", "windmill-parser", + "windmill-parser-py", "windmill-parser-ts", "windmill-queue", ] diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 738bc8ed7c798..76f0c415ea1e7 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -60,6 +60,7 @@ fn filter_non_main(code: &str, main_name: &str) -> String { pub fn parse_python_signature( code: &str, override_main: Option, + skip_params: bool, ) -> anyhow::Result { let main_name = override_main.unwrap_or("main".to_string()); @@ -78,11 +79,13 @@ pub fn parse_python_signature( let ast = Suite::parse(&filtered_code, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; - let param = ast.into_iter().find_map(|x| match x { + let params = ast.into_iter().find_map(|x| match x { Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => Some(*args), _ => None, }); - if let Some(params) = param { + + if !skip_params && params.is_some() { + let params = params.unwrap(); //println!("{:?}", params); let def_arg_start = params.args.len() - params.defaults().count(); Ok(MainArgSignature { @@ -149,7 +152,7 @@ pub fn parse_python_signature( star_args: false, star_kwargs: false, args: vec![], - no_main_func: Some(true), + no_main_func: Some(params.is_none()), has_preprocessor: Some(has_preprocessor), }) } @@ -287,7 +290,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt "; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -376,7 +379,7 @@ def main(test1: str, "; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -436,7 +439,7 @@ def main(test1: str, "; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -493,7 +496,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu "#; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -537,7 +540,7 @@ def main(test1: DynSelect_foo): return "#; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -568,7 +571,7 @@ def hello(): return "#; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -596,7 +599,7 @@ def main(): return "#; //println!("{}", serde_json::to_string()?); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -617,10 +620,10 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b "#; println!( "{}", - serde_json::to_string(&parse_python_signature(code, None)?)? + serde_json::to_string(&parse_python_signature(code, None, false)?)? ); assert_eq!( - parse_python_signature(code, None)?, + parse_python_signature(code, None, false)?, MainArgSignature { star_args: false, star_kwargs: false, diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index f69169c0e2658..78a83cd9290b4 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -134,6 +134,7 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result> { pub fn parse_deno_signature( code: &str, skip_dflt: bool, + skip_params: bool, main_override: Option, ) -> anyhow::Result { let cm: Lrc = Default::default(); @@ -179,27 +180,26 @@ pub fn parse_deno_signature( }); let mut c: u16 = 0; - if let Some(params) = params { - let r = MainArgSignature { - star_args: false, - star_kwargs: false, - args: params - .into_iter() - .map(|x| parse_param(x, &cm, skip_dflt, &mut c)) - .collect::>>()?, - no_main_func: Some(false), - has_preprocessor: Some(has_preprocessor), - }; - Ok(r) - } else { - Ok(MainArgSignature { - star_args: false, - star_kwargs: false, - args: vec![], - no_main_func: Some(true), - has_preprocessor: Some(has_preprocessor), - }) - } + let no_main_func = params.is_none(); + let r = MainArgSignature { + star_args: false, + star_kwargs: false, + args: if skip_params { + vec![] + } else { + params + .map(|x| { + x.into_iter() + .map(|x| parse_param(x, &cm, skip_dflt, &mut c)) + .collect::>>() + }) + .transpose()? + .unwrap_or_else(|| vec![]) + }, + no_main_func: Some(no_main_func), + has_preprocessor: Some(has_preprocessor), + }; + Ok(r) } fn parse_param( diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index e3de739f9634c..306d67751cbf3 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -17,10 +17,11 @@ fn wrap_sig(r: anyhow::Result) -> String { #[cfg(feature = "ts-parser")] #[wasm_bindgen] -pub fn parse_deno(code: &str, main_override: Option) -> String { +pub fn parse_deno(code: &str, main_override: Option, skip_params: Option) -> String { wrap_sig(windmill_parser_ts::parse_deno_signature( code, false, + false, main_override, )) } @@ -73,6 +74,7 @@ pub fn parse_python(code: &str, main_override: Option) -> String { wrap_sig(windmill_parser_py::parse_python_signature( code, main_override, + false, )) } diff --git a/backend/parsers/windmill-parser-wasm/tests/wasm.rs b/backend/parsers/windmill-parser-wasm/tests/wasm.rs index 1c201ecd4eedf..9f6d452a21df9 100644 --- a/backend/parsers/windmill-parser-wasm/tests/wasm.rs +++ b/backend/parsers/windmill-parser-wasm/tests/wasm.rs @@ -18,7 +18,7 @@ export function main(test1?: string, test2: string = \"burkina\", } "; assert_eq!( - parse_deno_signature(code, false, None)?, + parse_deno_signature(code, false, false, None)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -159,7 +159,7 @@ export function main(test2 = \"burkina\", } "; assert_eq!( - parse_deno_signature(code, false, None)?, + parse_deno_signature(code, false, false, None)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -236,7 +236,7 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\", } "; assert_eq!( - parse_deno_signature(code, false, None)?, + parse_deno_signature(code, false, false, None)?, MainArgSignature { star_args: false, star_kwargs: false, @@ -282,7 +282,7 @@ export function main(foo: (\"foo\" | \"bar\")[]) { } "; assert_eq!( - parse_deno_signature(code, false, None)?, + parse_deno_signature(code, false, false, None)?, MainArgSignature { star_args: false, star_kwargs: false, diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index c15d19a2597a3..28b890990b0be 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -68,7 +68,7 @@ fi if [ "$REVERT" == "YES" ]; then for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do - ce_file="${ee_file/${EE_CODE_DIR}/.}" + ce_file="${ee_file/${EE_CODE_DIR}/}" ce_file="${root_dirpath}/backend/${ce_file}" if [ "$REVERT_PREVIOUS" == "YES" ]; then git checkout HEAD@{3} ${ce_file} || true @@ -80,7 +80,7 @@ if [ "$REVERT" == "YES" ]; then else # This replaces all files in current repo with alternative EE files in windmill-ee-private for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do - ce_file="${ee_file/${EE_CODE_DIR}/.}" + ce_file="${ee_file/${EE_CODE_DIR}/}" ce_file="${root_dirpath}/backend/${ce_file}" if [[ -f "${ce_file}" ]]; then rm "${ce_file}" diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index cbecf865dec21..675aeec0b15b4 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -36,6 +36,7 @@ windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true windmill-parser.workspace = true windmill-parser-ts.workspace = true +windmill-parser-py.workspace = true windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } tokio.workspace = true diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index cbeb13e10690b..0fd80d0fe4516 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -625,6 +625,19 @@ async fn create_script_internal<'c>( } else { ns.language.clone() }; + + let (no_main_func, has_preprocessor) = match lang { + ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None)?; + (args.no_main_func, args.has_preprocessor) + } + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature(&ns.content, None, true)?; + (args.no_main_func, args.has_preprocessor) + } + _ => (ns.no_main_func, ns.has_preprocessor), + }; + sqlx::query!( "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ @@ -660,9 +673,9 @@ async fn create_script_internal<'c>( ns.timeout, ns.concurrency_key, ns.visible_to_runner_only, - ns.no_main_func, + no_main_func.filter(|x| *x), // should be Some(true) or None codebase, - ns.has_preprocessor, + has_preprocessor.filter(|x| *x), // should be Some(true) or None if ns.on_behalf_of_email.is_some() { Some(&authed.email) } else { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 134511472fd73..ede17da758d02 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -951,6 +951,7 @@ pub async fn handle_bun_job( let args = windmill_parser_ts::parse_deno_signature( inner_content, true, + false, main_override.map(ToString::to_string), )? .args; @@ -960,6 +961,7 @@ pub async fn handle_bun_job( windmill_parser_ts::parse_deno_signature( inner_content, true, + false, Some("preprocessor".to_string()), )? .args, @@ -1573,7 +1575,7 @@ pub async fn start_worker( { // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args; + let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args; let dates = args .iter() .filter_map(|x| { diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 6a7bed5c38073..40350fdeee0ff 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -205,6 +205,7 @@ pub async fn handle_deno_job( let args = windmill_parser_ts::parse_deno_signature( inner_content, true, + false, main_override.map(ToString::to_string), )? .args; @@ -214,6 +215,7 @@ pub async fn handle_deno_job( windmill_parser_ts::parse_deno_signature( inner_content, true, + false, Some("preprocessor".to_string()), )? .args, @@ -533,7 +535,7 @@ pub async fn start_worker( { // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args; + let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args; let dates = args .iter() .filter_map(|x| { diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 9aab14429667d..50f13788e7997 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -775,7 +775,7 @@ pub async fn eval_fetch_timeout( let (sender, mut receiver) = oneshot::channel::(); - let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, None)?.args; + let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, false, None)?.args; let spread = parsed_args .into_iter() .map(|x| { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d86c84dd0d440..531c41887a909 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1296,12 +1296,14 @@ async fn prepare_wrapper( let sig = windmill_parser_py::parse_python_signature( inner_content, main_override.map(ToString::to_string), + false, )?; let pre_sig = if apply_preprocessor { Some(windmill_parser_py::parse_python_signature( inner_content, Some("preprocessor".to_string()), + false, )?) } else { None @@ -1661,7 +1663,7 @@ async fn spawn_uv_install( .replace("{TARGET_DIR}", &venv_p) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; - + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) diff --git a/cli/metadata.ts b/cli/metadata.ts index e78b0bf539b8b..d547cee29f8c8 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -313,10 +313,16 @@ export async function updateScriptSchema( path ); metadataContent.schema = result.schema; - if (result.has_preprocessor == true) + if (result.has_preprocessor) { metadataContent.has_preprocessor = result.has_preprocessor; - if (result.no_main_func === true) + } else { + delete metadataContent.has_preprocessor; + } + if (result.no_main_func) { metadataContent.no_main_func = result.no_main_func; + } else { + delete metadataContent.no_main_func; + } } async function updateScriptLock( @@ -441,7 +447,11 @@ export function inferSchema( content: string, currentSchema: any, path: string -) { +): { + schema: any; + has_preprocessor: boolean | undefined; + no_main_func: boolean | undefined; +} { let inferedSchema: any; if (language === "python3") { inferedSchema = JSON.parse(parse_python(content));