support GET_PYTHON_MODULES - gather python module usage information

Run ansible with ANSIBLE_DEBUG=true and run each playbook with
PYTHONVERBOSE=1 in order to trace python module execution, so
that we can determine which python packages are required to run
each role.

library/lib.sh

@@ -319,10 +319,15 @@ rolesRunPlaybook() {
     local inventory=$3
     local skip_tags=$4
     local limit=$5
-    local LOGFILE="${test_playbook%.*}"-ANSIBLE-"$ANSIBLE_VER"
+    local LOGFILE="$REPO_NAME-${test_playbook%.*}-ANSIBLE-$ANSIBLE_VER"
     local result=FAIL
-    local cmd log_msg
-    cmd="ansible-playbook -i $inventory $skip_tags $limit $tests_path$test_playbook -vv"
+    local base_cmd cmd envs log_msg
+    envs=""
+    if [ "${GET_PYTHON_MODULES:-}" = true ]; then
+        envs="$envs ANSIBLE_DEBUG=true"
+    fi
+    base_cmd="ansible-playbook -i $inventory $skip_tags $limit $tests_path$test_playbook -vv"
+    cmd="$envs $base_cmd"
     log_msg="Test $test_playbook with ANSIBLE-$ANSIBLE_VER on ${limit/--limit /}"
     # If LSR_TFT_DEBUG is true, print output to terminal
     if [ "$LSR_TFT_DEBUG" == true ] || [ "$LSR_TFT_DEBUG" == True ]; then
@@ -334,6 +339,12 @@ rolesRunPlaybook() {
     mv "$LOGFILE" "$logfile_name"
     LOGFILE=$logfile_name
     rolesUploadLogs "$LOGFILE"
+    if [ "${GET_PYTHON_MODULES:-}" = true ]; then
+        cmd="ansible-playbook -i $inventory $skip_tags $limit process_python_modules_packages.yml -vv"
+        local packages="$LOGFILE.packages"
+        $cmd -e packages_file="$packages" -e logfile="$LOGFILE"
+        rolesUploadLogs "$packages"
+    fi
 }
 
 rolesRunPlaybooksParallel() {
@@ -454,6 +465,19 @@ luns/ create /backstores/fileio/disk${i}"
     rlRun "rm -rf ${role_path}"
 }
 
+# prepare test playbooks for gathering information about python
+# module usage
+rolesSetupGetPythonModules() {
+    local tests_dir test_pb
+    tests_dir="$1"; shift
+    for test_pb in "$@"; do
+        cp "$tests_dir$test_pb" "$tests_dir$test_pb.orig"
+        sed -e '/^  hosts:/a\
+  environment:\
+    PYTHONVERBOSE: "1"' -i "$tests_dir$test_pb"
+    done
+}
+
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #   Verification
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

tests/general/process_python_modules_packages.yml

@@ -0,0 +1,42 @@
+---
+- name: Get python modules from log file and create packages file
+  hosts: all
+  gather_facts: false
+  vars:
+    logfile: ""
+    packages_file: packages.txt
+  tasks:
+    - name: Get list of modules
+      set_fact:
+        modules: "{{ lookup('file', logfile).splitlines() | select('match', __pat) |
+          map('regex_replace', __pat, '\\1') | union(modules | d([])) | unique | list }}"
+      vars:
+        __pat: "^>>># /[^ ]+ matches (/[^ ]+?[.]py).*$"
+      changed_when: false
+      delegate_to: localhost
+
+    - name: Copy modules file to host
+      copy:
+        content: "{{ modules | sort | join(__nl) ~ __nl }}"
+        dest: /tmp/modules.txt
+        mode: "0600"
+      vars:
+        __nl: "\n"
+
+    - name: Get unique list of packages that provide modules in modules.txt
+      shell: |
+        declare -A packages
+        while read -r file; do
+          pkg="$(rpm -qf "$file" --queryformat '%{name}-%{version}-%{release}\n')"
+          packages["$pkg"]="$pkg"
+        done < /tmp/modules.txt
+        for pkg in "${packages[@]}"; do
+          echo "$pkg"
+        done | sort > /tmp/packages.txt
+      changed_when: false
+
+    - name: Copy packages file back to localhost
+      fetch:
+        src: /tmp/packages.txt
+        dest: "{{ packages_file }}"
+        flat: true

tests/general/test.sh

@@ -63,6 +63,10 @@ rlJournalStart
         done
         if [ -n "$ANSIBLE_VER" ]; then
             rolesInstallAnsible
+            if [ "$ANSIBLE_VER" = 2.9 ]; then
+                # does not work with 2.9
+                GET_PYTHON_MODULES=false
+            fi
         else
             rlLogInfo "ANSIBLE_VER not defined - using system ansible if installed"
         fi
@@ -85,10 +89,14 @@ rlJournalStart
         # shellcheck disable=SC2154
         inventory=$(rolesPrepareInventoryVars "$role_path" "$tmt_tree_provision" "$guests_yml")
         rlRun "cat $inventory"
+        tests_path="$collection_path"/ansible_collections/fedora/linux_system_roles/tests/"$REPO_NAME"/
+        if [ "${GET_PYTHON_MODULES:-}" = true ]; then
+            # shellcheck disable=SC2086
+            rolesSetupGetPythonModules "$tests_path" $test_playbooks
+        fi
     rlPhaseEnd
     rlPhaseStartTest
         managed_nodes=$(rolesGetManagedNodes "$guests_yml")
-        tests_path="$collection_path"/ansible_collections/fedora/linux_system_roles/tests/"$REPO_NAME"/
         rolesRunPlaybooksParallel "$tests_path" "$inventory" "$SKIP_TAGS" "$test_playbooks" "$managed_nodes"
     rlPhaseEnd
 rlJournalEnd