ODP-2739: Use ambari-python-wrap instead of python

be/src/codegen/gen_ir_descriptions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # This uses system python to avoid a dependency on impala-python,
 # because this runs during the build.
 #

bin/bootstrap_toolchain.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/check-rat-report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/collect_minidumps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/compare_branches.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

bin/create-test-configuration.sh

@@ -205,7 +205,7 @@ if [ $CREATE_RANGER_POLICY_DB -eq 1 ]; then
   createdb -U hiveuser "${RANGER_POLICY_DB}"
   pushd "${RANGER_HOME}"
   generate_config "${RANGER_TEST_CONF_DIR}/install.properties.template" install.properties
-  python ./db_setup.py
+  ambari-python-wrap ./db_setup.py
   popd
 fi
 

bin/diagnostics/collect_diagnostics.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/diagnostics/experimental/plan-graph.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/diagnostics/experimental/tpcds_run_comparator.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/gen-backend-test-script.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/gen_build_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # This uses system python to avoid a dependency on impala-python,
 # because this runs during the build.
 #

bin/generate_xml_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/impala-config.sh

@@ -237,9 +237,37 @@ export IMPALA_THRIFT_PY_VERSION=0.16.0-p7
 unset IMPALA_THRIFT_PY_URL
 
 # Find system python versions for testing
-export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
-export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}"
 
+if command -v python2 >/dev/null || [ -n "$IMPALA_SYSTEM_PYTHON2_OVERRIDE" ]; then
+  export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
+fi
+
+if command -v ambari-python-wrap >/dev/null || [ -n "$IMPALA_SYSTEM_PYTHON3_OVERRIDE" ]; then
+    # Extract major python version from ambari-python-wrap
+    PYTHON_VER=$($(command -v ambari-python-wrap) --version 2>&1 | tr -s '[:blank:]' . | cut -d '.' -f2)
+
+    if printf "%s" "$PYTHON_VER" | grep "^[23]$" >/dev/null && [ "$PYTHON_VER" -eq 3 ] 2>/dev/null && [ -z "$IMPALA_SYSTEM_PYTHON3_OVERRIDE" ]; then
+        mkdir -p /tmp/acceldata-python/
+        \rm -f /tmp/acceldata-python/python3
+        # CMakeLists.txt expects the binary to match exactly python3, so we create a symlink
+        ln -sv "$(command -v ambari-python-wrap)" /tmp/acceldata-python/python3
+        export IMPALA_SYSTEM_PYTHON3="/tmp/acceldata-python/python3"
+    else 
+        export IMPALA_SYSTEM_PYTHON3="$IMPALA_SYSTEM_PYTHON3_OVERRIDE"
+    fi
+fi
+
+if [ -z "$IMPALA_SYSTEM_PYTHON2" ] && [ -z "$IMPALA_SYSTEM_PYTHON3" ]; then
+    printf "Python 2 was not found in PATH\n" >&2
+    printf "ambari-python-wrap not found in PATH.\n" >&2
+    printf "Neither IMPALA_SYSTEM_PYTHON2_OVERRIDE or IMPALA_SYSTEM_PYTHON3_OVERRIDE have been set\n" >&2
+    printf "At least one of these must be true to continue bulding impala\n" >&2
+    exit 1
+elif [ -z "$IMPALA_SYSTEM_PYTHON3" ]; then
+    printf "Skipping configuration for python 3\n"
+elif [ -z "$IMPALA_SYSTEM_PYTHON2" ]; then
+    printf "Skipping configuration for python 2\n"
+fi
 # Additional Python versions to use when building the impala-shell prebuilt tarball
 # via make_shell_tarball.sh. That tarball includes precompiled packages, so it can be
 # used without additional system dependencies needed for pip install.

bin/impala-python-common.sh

@@ -23,10 +23,9 @@ set -euo pipefail
 setup_report_build_error
 
 . $IMPALA_HOME/bin/set-pythonpath.sh
-
-export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+export LD_LIBRARY_PATH="$(ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 PY_DIR="$(dirname "$0")/../infra/python"
 PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}"
-python "$PY_DIR/bootstrap_virtualenv.py"
+ambari-python-wrap "$PY_DIR/bootstrap_virtualenv.py"

bin/impala-python3-common.sh

@@ -24,9 +24,9 @@ setup_report_build_error
 
 . $IMPALA_HOME/bin/set-pythonpath.sh
 
-export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+export LD_LIBRARY_PATH="$(ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 PY_DIR="$(dirname "$0")/../infra/python"
 PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}-py3"
-python "$PY_DIR/bootstrap_virtualenv.py" --python3
+ambari-python-wrap "$PY_DIR/bootstrap_virtualenv.py" --python3

bin/impala-shell.sh

@@ -30,9 +30,8 @@ PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/bin
 PYTHONPATH=${PYTHONPATH}:${SHELL_HOME}/gen-py
 
 THRIFT_PY_ROOT="${IMPALA_TOOLCHAIN_PACKAGES_HOME}/thrift-${IMPALA_THRIFT_PY_VERSION}"
-
 export LD_LIBRARY_PATH=":$(PYTHONPATH=${PYTHONPATH} \
-  python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+  ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 IMPALA_PY_DIR="$(dirname "$0")/../infra/python"
@@ -48,7 +47,7 @@ for PYTHON_LIB_DIR in ${THRIFT_PY_ROOT}/python/lib{64,}; do
 done
 
 # Note that this uses the external system python executable
-PYTHONPATH=${PYTHONPATH} python "${IMPALA_PY_DIR}/bootstrap_virtualenv.py"
+PYTHONPATH=${PYTHONPATH} ambari-python-wrap "${IMPALA_PY_DIR}/bootstrap_virtualenv.py"
 
 # Enable remote debugging if port was specified via environment variable
 if [[ ${IMPALA_SHELL_DEBUG_PORT:-0} -ne 0 ]]; then

bin/jenkins/critique-gerrit-review.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/jenkins/dockerized-impala-preserve-vars.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/jenkins/populate_m2_directory.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/push_to_asf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/resolve_minidumps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/validate-unified-backend-test-filters.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/function-registry/gen_builtins_catalog.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/function-registry/gen_geospatial_udf_wrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/thrift/generate_error_codes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/thrift/generate_metrics.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/annotate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python -u
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/monitor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/test-with-docker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

fe/src/test/resources/hive-site.xml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

infra/python/deps/pip_download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

lib/python/impala_py_lib/jenkins/generate_junitxml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

lib/python/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/TSSLSocketWithWildcardSAN.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

shell/compatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/ext-py/prettytable-0.7.2/prettytable.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Copyright (c) 2009-2013, Luke Maurits <[email protected]>
 # All rights reserved.

shell/ext-py/prettytable-0.7.2/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 from setuptools import setup
 from prettytable import __version__ as version
 

shell/ext-py/sqlparse-0.3.1/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2009-2018 the sqlparse authors and contributors

shell/ext-py/sqlparse-0.3.1/sqlparse/__main__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2009-2018 the sqlparse authors and contributors

shell/ext-py/sqlparse-0.3.1/sqlparse/cli.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2009-2018 the sqlparse authors and contributors

shell/ext-py/thrift-0.16.0/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/impala-shell

@@ -37,7 +37,7 @@ SHELL_HOME=${IMPALA_SHELL_HOME:-${SCRIPT_DIR}}
 export LC_CTYPE=${LC_CTYPE:-en_US.UTF-8}
 
 # Select python version; prefer 2, use 3 if 2's absent. Allow override with envvar
-PYTHON_EXE="${IMPALA_PYTHON_EXECUTABLE:-python}"
+PYTHON_EXE="${IMPALA_PYTHON_EXECUTABLE:-ambari-python-wrap}"
 if ! command -v "${PYTHON_EXE}" > /dev/null; then
   PYTHON_EXE=python3
 fi

shell/impala_client.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/impala_shell.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/impala_shell_config_defaults.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/option_parser.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/packaging/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

shell/packaging/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

shell/shell_output.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one

testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

testdata/cluster/node_templates/common/etc/hadoop/conf/ozone-site.xml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

tests/comparison/data_generator_mapper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

tests/comparison/data_generator_reducer.py

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file