feat: parse script for preprocessor/no_main_func on deploy (#5292)

* feat: parse script for preprocessor/no_main_func on deploy * fix substitute for linux
windmill-labs · Feb 14, 2025 · 28558e6 · 28558e6
1 parent 266f820
commit 28558e6
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 49 deletions.
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs
@@ -60,6 +60,7 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
 pub fn parse_python_signature(
     code: &str,
     override_main: Option<String>,
+    skip_params: bool,
 ) -> anyhow::Result<MainArgSignature> {
     let main_name = override_main.unwrap_or("main".to_string());
 
@@ -78,11 +79,13 @@ pub fn parse_python_signature(
     let ast = Suite::parse(&filtered_code, "main.py")
         .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
 
-    let param = ast.into_iter().find_map(|x| match x {
+    let params = ast.into_iter().find_map(|x| match x {
         Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => Some(*args),
         _ => None,
     });
-    if let Some(params) = param {
+
+    if !skip_params && params.is_some() {
+        let params = params.unwrap();
         //println!("{:?}", params);
         let def_arg_start = params.args.len() - params.defaults().count();
         Ok(MainArgSignature {
@@ -149,7 +152,7 @@ pub fn parse_python_signature(
             star_args: false,
             star_kwargs: false,
             args: vec![],
-            no_main_func: Some(true),
+            no_main_func: Some(params.is_none()),
             has_preprocessor: Some(has_preprocessor),
         })
     }
@@ -287,7 +290,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
 ";
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -376,7 +379,7 @@ def main(test1: str,
 ";
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -436,7 +439,7 @@ def main(test1: str,
 ";
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -493,7 +496,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
 "#;
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -537,7 +540,7 @@ def main(test1: DynSelect_foo): return
 "#;
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -568,7 +571,7 @@ def hello(): return
 "#;
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -596,7 +599,7 @@ def main(): return
 "#;
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,
@@ -617,10 +620,10 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b
 "#;
         println!(
             "{}",
-            serde_json::to_string(&parse_python_signature(code, None)?)?
+            serde_json::to_string(&parse_python_signature(code, None, false)?)?
         );
         assert_eq!(
-            parse_python_signature(code, None)?,
+            parse_python_signature(code, None, false)?,
             MainArgSignature {
                 star_args: false,
                 star_kwargs: false,

diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs
@@ -134,6 +134,7 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
 pub fn parse_deno_signature(
     code: &str,
     skip_dflt: bool,
+    skip_params: bool,
     main_override: Option<String>,
 ) -> anyhow::Result<MainArgSignature> {
     let cm: Lrc<SourceMap> = Default::default();
@@ -179,27 +180,26 @@ pub fn parse_deno_signature(
     });
 
     let mut c: u16 = 0;
-    if let Some(params) = params {
-        let r = MainArgSignature {
-            star_args: false,
-            star_kwargs: false,
-            args: params
-                .into_iter()
-                .map(|x| parse_param(x, &cm, skip_dflt, &mut c))
-                .collect::<anyhow::Result<Vec<Arg>>>()?,
-            no_main_func: Some(false),
-            has_preprocessor: Some(has_preprocessor),
-        };
-        Ok(r)
-    } else {
-        Ok(MainArgSignature {
-            star_args: false,
-            star_kwargs: false,
-            args: vec![],
-            no_main_func: Some(true),
-            has_preprocessor: Some(has_preprocessor),
-        })
-    }
+    let no_main_func = params.is_none();
+    let r = MainArgSignature {
+        star_args: false,
+        star_kwargs: false,
+        args: if skip_params {
+            vec![]
+        } else {
+            params
+                .map(|x| {
+                    x.into_iter()
+                        .map(|x| parse_param(x, &cm, skip_dflt, &mut c))
+                        .collect::<anyhow::Result<Vec<Arg>>>()
+                })
+                .transpose()?
+                .unwrap_or_else(|| vec![])
+        },
+        no_main_func: Some(no_main_func),
+        has_preprocessor: Some(has_preprocessor),
+    };
+    Ok(r)
 }
 
 fn parse_param(

diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs
@@ -17,10 +17,11 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
 
 #[cfg(feature = "ts-parser")]
 #[wasm_bindgen]
-pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
+pub fn parse_deno(code: &str, main_override: Option<String>, skip_params: Option<bool>) -> String {
     wrap_sig(windmill_parser_ts::parse_deno_signature(
         code,
         false,
+        false,
         main_override,
     ))
 }
@@ -73,6 +74,7 @@ pub fn parse_python(code: &str, main_override: Option<String>) -> String {
     wrap_sig(windmill_parser_py::parse_python_signature(
         code,
         main_override,
+        false,
     ))
 }
 

diff --git a/backend/parsers/windmill-parser-wasm/tests/wasm.rs b/backend/parsers/windmill-parser-wasm/tests/wasm.rs
@@ -18,7 +18,7 @@ export function main(test1?: string, test2: string = \"burkina\",
 }
 ";
     assert_eq!(
-        parse_deno_signature(code, false, None)?,
+        parse_deno_signature(code, false, false, None)?,
         MainArgSignature {
             star_args: false,
             star_kwargs: false,
@@ -159,7 +159,7 @@ export function main(test2 = \"burkina\",
 }
 ";
     assert_eq!(
-        parse_deno_signature(code, false, None)?,
+        parse_deno_signature(code, false, false, None)?,
         MainArgSignature {
             star_args: false,
             star_kwargs: false,
@@ -236,7 +236,7 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
 }
 ";
     assert_eq!(
-        parse_deno_signature(code, false, None)?,
+        parse_deno_signature(code, false, false, None)?,
         MainArgSignature {
             star_args: false,
             star_kwargs: false,
@@ -282,7 +282,7 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
 }
 ";
     assert_eq!(
-        parse_deno_signature(code, false, None)?,
+        parse_deno_signature(code, false, false, None)?,
         MainArgSignature {
             star_args: false,
             star_kwargs: false,

diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh
@@ -68,7 +68,7 @@ fi
 
 if [ "$REVERT" == "YES" ]; then
   for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do
-    ce_file="${ee_file/${EE_CODE_DIR}/.}"
+    ce_file="${ee_file/${EE_CODE_DIR}/}"
     ce_file="${root_dirpath}/backend/${ce_file}"
     if [ "$REVERT_PREVIOUS" == "YES" ]; then
       git checkout HEAD@{3} ${ce_file} || true
@@ -80,7 +80,7 @@ if [ "$REVERT" == "YES" ]; then
 else
   # This replaces all files in current repo with alternative EE files in windmill-ee-private
   for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do
-    ce_file="${ee_file/${EE_CODE_DIR}/.}"
+    ce_file="${ee_file/${EE_CODE_DIR}/}"
     ce_file="${root_dirpath}/backend/${ce_file}"
     if [[ -f "${ce_file}" ]]; then
       rm "${ce_file}"

diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml
@@ -36,6 +36,7 @@ windmill-common = { workspace = true, default-features = false }
 windmill-audit.workspace = true
 windmill-parser.workspace = true
 windmill-parser-ts.workspace = true
+windmill-parser-py.workspace = true
 windmill-git-sync.workspace = true
 windmill-indexer = { workspace = true, optional = true }
 tokio.workspace = true

diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs
@@ -625,6 +625,19 @@ async fn create_script_internal<'c>(
     } else {
         ns.language.clone()
     };
+
+    let (no_main_func, has_preprocessor) = match lang {
+        ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => {
+            let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None)?;
+            (args.no_main_func, args.has_preprocessor)
+        }
+        ScriptLang::Python3 => {
+            let args = windmill_parser_py::parse_python_signature(&ns.content, None, true)?;
+            (args.no_main_func, args.has_preprocessor)
+        }
+        _ => (ns.no_main_func, ns.has_preprocessor),
+    };
+
     sqlx::query!(
         "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
          content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
@@ -660,9 +673,9 @@ async fn create_script_internal<'c>(
         ns.timeout,
         ns.concurrency_key,
         ns.visible_to_runner_only,
-        ns.no_main_func,
+        no_main_func.filter(|x| *x), // should be Some(true) or None
         codebase,
-        ns.has_preprocessor,
+        has_preprocessor.filter(|x| *x), // should be Some(true) or None
         if ns.on_behalf_of_email.is_some() {
             Some(&authed.email)
         } else {

diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs
@@ -951,6 +951,7 @@ pub async fn handle_bun_job(
         let args = windmill_parser_ts::parse_deno_signature(
             inner_content,
             true,
+            false,
             main_override.map(ToString::to_string),
         )?
         .args;
@@ -960,6 +961,7 @@ pub async fn handle_bun_job(
                 windmill_parser_ts::parse_deno_signature(
                     inner_content,
                     true,
+                    false,
                     Some("preprocessor".to_string()),
                 )?
                 .args,
@@ -1573,7 +1575,7 @@ pub async fn start_worker(
 
     {
         // let mut start = Instant::now();
-        let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args;
+        let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args;
         let dates = args
             .iter()
             .filter_map(|x| {

diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs
@@ -205,6 +205,7 @@ pub async fn handle_deno_job(
         let args = windmill_parser_ts::parse_deno_signature(
             inner_content,
             true,
+            false,
             main_override.map(ToString::to_string),
         )?
         .args;
@@ -214,6 +215,7 @@ pub async fn handle_deno_job(
                 windmill_parser_ts::parse_deno_signature(
                     inner_content,
                     true,
+                    false,
                     Some("preprocessor".to_string()),
                 )?
                 .args,
@@ -533,7 +535,7 @@ pub async fn start_worker(
 
     {
         // let mut start = Instant::now();
-        let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args;
+        let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args;
         let dates = args
             .iter()
             .filter_map(|x| {

diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs
@@ -775,7 +775,7 @@ pub async fn eval_fetch_timeout(
 
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
 
-    let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, None)?.args;
+    let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, false, None)?.args;
     let spread = parsed_args
         .into_iter()
         .map(|x| {

diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs
@@ -1296,12 +1296,14 @@ async fn prepare_wrapper(
     let sig = windmill_parser_py::parse_python_signature(
         inner_content,
         main_override.map(ToString::to_string),
+        false,
     )?;
 
     let pre_sig = if apply_preprocessor {
         Some(windmill_parser_py::parse_python_signature(
             inner_content,
             Some("preprocessor".to_string()),
+            false,
         )?)
     } else {
         None
@@ -1661,7 +1663,7 @@ async fn spawn_uv_install(
             .replace("{TARGET_DIR}", &venv_p)
             .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
         )?;
-    
+
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
             .current_dir(job_dir)