[GLUTEN-7641][VL] Fix security issue for perf analysis scripts (#8309)

Fix security issue for perf analysis scripts and some minor fixes to the scripts

tools/workload/benchmark_velox/analysis/requirements.txt

@@ -5,24 +5,23 @@ argon2-cffi-bindings==21.2.0
 arrow==1.3.0
 asttokens==2.4.1
 attrs==23.2.0
-Automat==20.2.0
-Babel==2.8.0
+Babel>=2.9.1
 bcrypt==3.2.0
 beautifulsoup4==4.12.3
 black==24.4.2
 bleach==6.1.0
 blinker==1.4
-certifi==2020.6.20
+certifi>=2023.7.22
 cffi==1.16.0
 chardet==4.0.0
 charset-normalizer==3.4.0
 click==8.0.3
 colorama==0.4.4
 comm==0.2.2
-configobj==5.0.6
+configobj>=5.0.9
 constantly==15.1.0
 contourpy==1.2.1
-cryptography==3.4.8
+cryptography>=42.0.2
 cycler==0.12.1
 debugpy==1.8.1
 decorator==5.1.1
@@ -43,15 +42,14 @@ httplib2==0.20.2
 hyperlink==21.0.0
 idna==3.10
 importlib-metadata==4.6.4
-incremental==21.3.0
 ipykernel==6.29.4
 ipython==8.24.0
 ipython-genutils==0.2.0
 ipywidgets==8.1.3
 isoduration==20.11.0
 jedi==0.19.1
 jeepney==0.7.1
-Jinja2==3.0.3
+Jinja2>=3.1.4
 jsonpatch==1.32
 jsonpointer==2.0
 jsonschema==4.22.0
@@ -63,7 +61,7 @@ jupyter_core==5.7.2
 jupyter-events==0.10.0
 jupyter-highlight-selected-word==0.2.0
 jupyter-nbextensions-configurator==0.6.3
-jupyter_server==2.14.0
+jupyter_server>=2.14.1
 jupyter-server-mathjax==0.2.6
 jupyter_server_terminals==0.5.3
 jupyterlab_pygments==0.3.0
@@ -92,7 +90,7 @@ notebook==6.5.6
 notebook_shim==0.2.4
 NotebookScripter==6.0.0
 numpy==1.26.4
-oauthlib==3.2.0
+oauthlib>=3.2.2
 overrides==7.7.0
 packaging==24.0
 pandas==1.5.3
@@ -114,15 +112,15 @@ pyarrow==16.1.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.1
 pycparser==2.22
-Pygments==2.11.2
+Pygments>=2.15.0
 PyHamcrest==2.0.2
 PyHDFS==0.3.1
-PyJWT==2.3.0
+pyjwt>=2.4.0
 pyOpenSSL==21.0.0
 pyparsing==2.4.7
 pyrsistent==0.18.1
 pyserial==3.5
-pyspark==3.3.1
+pyspark>=3.3.2
 python-dateutil==2.9.0.post0
 python-json-logger==2.0.7
 pytz==2022.1
@@ -154,21 +152,21 @@ termcolor==2.5.0
 terminado==0.18.1
 tinycss2==1.3.0
 tomli==2.0.1
-tornado==6.4
+tornado>=6.4.2
 tqdm==4.66.4
 traitlets==5.14.3
-Twisted==22.1.0
+twisted>=24.7.0rc1
 types-python-dateutil==2.9.0.20240316
 typing_extensions==4.12.0
 tzdata==2024.1
 uri-template==1.3.0
-urllib3==1.26.5
+urllib3>=1.26.19
 wadllib==1.3.6
 wcwidth==0.2.13
 webcolors==1.13
 webencodings==0.5.1
 websocket-client==1.8.0
 wheel==0.44.0
 widgetsnbextension==4.0.11
-zipp==1.0.0
+zipp>=3.19.1
 zope.interface==5.4.0

tools/workload/benchmark_velox/analysis/sparklog.ipynb

@@ -268,7 +268,7 @@
     "        with open(outputfolder, 'w') as outfile: \n",
     "            outfile.write(output)\n",
     "        \n",
-    "        traceview_link=f'http://{localhost}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{appidx}.json'\n",
+    "        traceview_link=f'http://{local_ip}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{appidx}.json'\n",
     "        display(HTML(f\"<a href={traceview_link}>{traceview_link}</a>\"))\n",
     "        return traceview_link"
    ]
@@ -5219,7 +5219,7 @@
     "        with open('/home/sparkuser/trace_result/'+self.appid+'.json', 'w') as outfile:  \n",
     "            outfile.write(output)\n",
     "        \n",
-    "        traceview_link=f'http://{localhost}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{self.appid}.json'\n",
+    "        traceview_link=f'http://{local_ip}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{self.appid}.json'\n",
     "        display(HTML(f\"<a href={traceview_link}>{traceview_link}</a>\"))\n",
     "        return traceview_link\n",
     "\n",
@@ -5774,7 +5774,7 @@
     "    if fs.exists(changelog):\n",
     "        with fs.open(changelog) as f:\n",
     "            for l in f.readlines():\n",
-    "                l = l.decode('ascii')\n",
+    "                l = l.decode('utf-8')\n",
     "                if l.startswith(\"commit\"):\n",
     "                    out.append(re.sub(r\"commit +(.+)\",r\"<font color=#BDCA57>commit </font><font color=#23C2BF>\\1</font>\",l))\n",
     "                elif l.startswith(\"Author\"):\n",

tools/workload/benchmark_velox/initialize.ipynb

@@ -2976,18 +2976,26 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {
     "hidden": true
    },
+   "outputs": [],
    "source": [
-    "\n",
-    "```bash\n",
-    "cd ~\n",
-    "virtualenv -p python3 -v paus-env\n",
-    "source paus-env/bin/activate\n",
-    "python3 -m pip install -r ~/gluten/tools/workload/benchmark_velox/analysis/requirements.txt\n",
-    "```"
+    "%cd ~"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hidden": true
+   },
+   "outputs": [],
+   "source": [
+    "!virtualenv -p python3 -v paus-env\n",
+    "!source ~/paus-env/bin/activate && python3 -m pip install -r ~/gluten/tools/workload/benchmark_velox/analysis/requirements.txt"
    ]
   },
   {
@@ -2996,15 +3004,18 @@
     "hidden": true
    },
    "source": [
-    "\n",
-    "We will put all perf analysis notebooks under `$HOME/PAUS`. Create the directory and start the notebook:\n",
-    "\n",
-    "```bash\n",
-    "mkdir -p ~/PAUS\n",
-    "cd ~/PAUS\n",
-    "nohup jupyter notebook --ip=0.0.0.0 --port=8889 &\n",
-    "```\n",
-    "\n"
+    "We will put all perf analysis notebooks under `$HOME/PAUS`. Create the directory and start the notebook:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hidden": true
+   },
+   "outputs": [],
+   "source": [
+    "!mkdir -p ~/PAUS && cd ~/PAUS && source ~/paus-env/bin/activate && nohup jupyter notebook --ip=0.0.0.0 --port=8889 &"
    ]
   },
   {
@@ -3013,11 +3024,18 @@
     "hidden": true
    },
    "source": [
-    "Package the virtual environment so that it can be distributed to other nodes:\n",
-    "```bash\n",
-    "cd ~\n",
-    "tar -czf paus-env.tar.gz paus-env\n",
-    "```"
+    "Package the virtual environment so that it can be distributed to other nodes:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hidden": true
+   },
+   "outputs": [],
+   "source": [
+    "!cd ~ && tar -czf paus-env.tar.gz paus-env"
    ]
   },
   {