Merge pull request #1000 from exasol/develop

Changelist: - #969: Use new CTPG parser in java vm (#970) - #980: Moved flavor template-Exasol-all-java-17 to script-languages repo (#984) - #971: Removed dead code in Java VM (#985) - #982 & #983: Refactorings in ScriptOptions parser (#987) - #988: Adapted Java integration tests for new ScriptOptions parser (#995) - #997: Added documentation for Rollback of Script-Languages-Container (#999)
exasol · Nov 5, 2024 · 4633316 · 4633316
2 parents 0d90dc1 + 70e6d54
commit 4633316
Show file tree

Hide file tree

Showing 46 changed files with 355 additions and 641 deletions.
diff --git a/.current_gitmodules b/.current_gitmodules
diff --git a/.github/workflows/check_bazel_build.yml b/.github/workflows/check_bazel_build.yml
@@ -0,0 +1,29 @@
+name: Check Bazel Build
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      UDF_CLIENT_ENV_FILE: /tmp/.udf_client_env
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Install UDF Client Dependencies
+        run: |
+          sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE"
+      - uses: r-lib/actions/setup-r@v2
+      - name: Build
+        run: |
+          source "$UDF_CLIENT_ENV_FILE"
+          bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --config r --verbose_failures
+        working-directory: ./exaudfclient/
diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml
diff --git a/.github/workflows/check_rollback_7.1.30_script.yml b/.github/workflows/check_rollback_7.1.30_script.yml
@@ -0,0 +1,92 @@
+name: Check Rollback Script 7.1.30
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+
+jobs:
+  Check:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - db-version: "7.1.29"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3'"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3';"
+            support-python2: "no"
+          - db-version: "7.1.19"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient_py3';"
+            support-python2: "yes"
+          - db-version: "7.1.6"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient_py3';"
+            support-python2: "yes"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: "3.10"
+          poetry-version: '1.8.2'
+      - name: Start ITDE
+        run: poetry run itde spawn-test-environment --environment-name exasol_test --database-port-forward 8888 --bucketfs-port-forward 6583 --docker-db-image-version "${{ matrix.db-version }}"
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y perl default-jre
+      - name: Install EXAPlus
+        run: |
+          curl -o exaplus.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/EXAplus-7.0.11.tar.gz && tar xf exaplus.tar.gz
+          echo "/tmp/EXAplus-7.0.11" >> $GITHUB_PATH
+        working-directory: /tmp
+      - name: Check if get_original_script_languages_parameters.sql works
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./get_original_script_languages_parameter.sql`
+          echo "$RESULT" | grep  "$EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER" || (echo SCRIPT_LANGUAGES parameter not as expected && exit 1)
+        env:
+          EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-original-alter-system-parameter }}"
+
+        working-directory: doc/user_guide/resources
+      - name: Install a simple R UDF
+        run: |
+          sleep 120
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_r_udf.sql
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();"
+        working-directory: .github/workflows/scripts
+      - name: Install a simple Python2 UDF
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: |
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_python2_udf.sql
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();"
+        working-directory: .github/workflows/scripts
+      - name: Check if pre_update_check.sql works for R only
+        if:  ${{ matrix.support-python2=='no' }}
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql`
+          echo "$RESULT" | grep "You have 1 UDFs using the pre-shipped R/Python2"
+        working-directory: doc/user_guide/resources
+      - name: Check if pre_update_check.sql works for R and Python2
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql`
+          echo "$RESULT" | grep "You have 2 UDFs using the pre-shipped R/Python2"
+        working-directory: doc/user_guide/resources
+      - name: Run construct_alter_system_command_before_update.sql
+        run: |
+          RESULT=`exaplus -c localhost:8888 -u sys -p exasol -f ./construct_alter_system_command_before_update.sql`
+          echo "$RESULT" | tr -d '\n' | grep "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER" || (echo ALTER SYSTEM command not as expected && exit 1)
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER"
+        working-directory: doc/user_guide/resources
+        env:
+          EXPECTED_NEW_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-new-alter-system-command }}"
+      - name: Test if R UDF still works
+        run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();"
+      - name: Test if Python2 UDF still works
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();"
diff --git a/.github/workflows/scripts/install_python2_udf.sql b/.github/workflows/scripts/install_python2_udf.sql
@@ -0,0 +1,10 @@
+CREATE SCHEMA IF NOT EXISTS TEST;
+OPEN SCHEMA TEST;
+--/
+
+CREATE OR REPLACE PYTHON SCALAR SCRIPT "TEST_PYTHON_VERSION" () RETURNS VARCHAR(2000) AS
+import sys
+def run(ctx):
+    return sys.version
+/
+
diff --git a/.github/workflows/scripts/install_r_udf.sql b/.github/workflows/scripts/install_r_udf.sql
@@ -0,0 +1,10 @@
+CREATE SCHEMA IF NOT EXISTS TEST;
+OPEN SCHEMA TEST;
+
+--/
+CREATE OR REPLACE R SCALAR SCRIPT TEST.R_DEMO() RETURNS VARCHAR(2000) AS
+
+run <- function(ctx) {
+ "Minimal R language UDF"
+}
+/
diff --git a/doc/changes/changes_8.4.0.md b/doc/changes/changes_8.4.0.md
@@ -26,9 +26,19 @@ This release uses version 1.0.0 of the container tool.
   - #967: Added ctpg script options parser
   - #972: Refactor exception handling for Script Options parser
   - #973: Support new lines and white spaces at the beginning in script options values
-
+  - #969: Use new CTPG Parser in Java VM
+  - #980: Moved flavor template-Exasol-all-java-17 to script-languages repo
+  - #971: Removed dead code in Java VM
+  - #982: Added a performance test for ScriptOptions parser for a single line UDF
+  - #983: Refactor CTPG script options Java parser code
+  - #986: Added environment variable for new ScriptOptionsLine parser in UDF client
+  - #990: Forward JAR options without changing order to classpath
+  - #989: Trim script class and import script options
+  - #988: Adapted Java integration tests for new ScriptOptions parser
+
 ## Bugs
   - #977: Fixed Trivy update cache workflow
+  - #993: Added escape sequence for backslash in new Script Options parser
 
 ## Doc
 

diff --git a/doc/user_guide/resources/construct_alter_system_command_before_update.sql b/doc/user_guide/resources/construct_alter_system_command_before_update.sql
@@ -0,0 +1,66 @@
+with minor_version as (
+  select CAST(SUBSTR(PARAM_VALUE,INSTR(PARAM_VALUE,'.',-1,1)+1) as INT) as minor_version
+  from  EXA_METADATA
+  where PARAM_NAME='databaseProductVersion' and PARAM_VALUE like '7.1%'
+),
+container_name as (
+  select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36' as container_name
+  from minor_version
+  where minor_version>=20
+  union all
+  select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R' as container_name
+  from minor_version
+  where minor_version>=7 AND minor_version<20
+  union all
+  select 'ScriptLanguages-release-standard-exasol-7.1.0-1.1.0' as container_name
+  from minor_version
+  where minor_version<7
+
+),
+num_of_pairs as(
+  SELECT
+    p.system_value,
+    length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs
+  FROM exa_parameters p
+  WHERE p.parameter_name in ('SCRIPT_LANGUAGES')
+),
+lang_pairs as(
+  SELECT
+    regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val,
+    nop.system_value
+  FROM num_of_pairs nop
+  connect by level <= nop.num_of_pairs
+),
+parsed_lang_pairs as(
+  SELECT
+    instr(lp.pair_val, '=') as eq_pos,
+    substr(lp.pair_val, 1, local.eq_pos-1) as alias_name,
+    substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content
+  FROM lang_pairs lp
+),
+new_alias_content as (
+  select
+    alias_name,
+    'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient_py3'  as alias_content
+  from container_name
+  join parsed_lang_pairs on true
+  where alias_content='builtin_r'
+  union all
+  select
+    alias_name,
+    'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient'  as alias_content
+  from container_name
+  join parsed_lang_pairs on true
+  where alias_content='builtin_python'
+  union all
+  select
+    alias_name,
+    alias_content
+  from parsed_lang_pairs
+  where alias_content<>'builtin_r' and alias_content<>'builtin_python'
+),
+new_system_value as (
+  select GROUP_CONCAT(alias_name || '=' || alias_content SEPARATOR ' ') as new_system_value
+  from new_alias_content
+)
+select 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || new_system_value || ''';' as command_text from new_system_value;
diff --git a/doc/user_guide/resources/get_original_script_languages_parameter.sql b/doc/user_guide/resources/get_original_script_languages_parameter.sql
@@ -0,0 +1,3 @@
+SELECT 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || system_value || ''';'
+FROM  EXA_PARAMETERS
+WHERE PARAMETER_NAME='SCRIPT_LANGUAGES';
diff --git a/doc/user_guide/resources/pre_update_check.sql b/doc/user_guide/resources/pre_update_check.sql
@@ -0,0 +1,45 @@
+with
+num_of_pairs as(
+    SELECT
+        p.system_value
+        , length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs
+    FROM
+        exa_parameters p
+    WHERE
+        1=1
+        and p.parameter_name in ('SCRIPT_LANGUAGES')
+)
+, lang_pairs as(
+    SELECT
+        regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val
+        , nop.system_value
+    FROM
+        num_of_pairs nop
+    connect by
+        level <= nop.num_of_pairs
+)
+, parsed as(
+    SELECT
+        lp.*
+        , instr(lp.pair_val, '=') as eq_pos
+        , substr(lp.pair_val, 1, local.eq_pos-1) as alias_name
+        , substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content
+    FROM
+        lang_pairs lp
+)
+select
+    case
+        when count(*) = 0 then 'You are not using the pre-shipped R/Python2 in UDFs.'
+        else 'You have '||to_char(count(*))||' UDFs using the pre-shipped R/Python2:
+
+' || GROUP_CONCAT('"'||s.script_schema||'"."'||s.script_name||'"' separator '
+')
+    end as check_results
+FROM
+    parsed p
+        join exa_dba_scripts s
+        on upper(p.alias_name)=s.script_language
+WHERE
+    1=1
+    and (p.alias_content = 'builtin_r' or p.alias_content = 'builtin_python')
+;