fix iso when private_data_dir is more than 1 subdir

* ISO job runs will now correctly mirror the control node
private_data_dir structure even when/if the private_data_dir is multiple
directories deep.
* The filesystem jail for bubblewrap now lives in
/tmp/bwrap_<job_id>_xxx along side private_data_dir
/tmp/bwrap_<job_id>_xxx/awx_<job_id>>_xxx This allows for the cleanup
job to remove all dirs for a job.
* Modified cleanup job to work with new /tmp/bwrap_<job_id>_xxx schema

awx/main/isolated/manager.py

@@ -169,7 +169,7 @@ def dispatch(self, playbook=None, module=None, module_args=None):
 
         extravars = {
             'src': self.private_data_dir,
-            'dest': settings.AWX_PROOT_BASE_PATH,
+            'dest': os.path.split(self.private_data_dir)[0],
             'ident': self.ident,
             'job_id': self.instance.id,
         }

awx/main/tasks.py

@@ -998,14 +998,7 @@ def build_params_process_isolation(self, instance, private_data_dir, cwd):
             show_paths = self.proot_show_paths + local_paths + \
                 settings.AWX_PROOT_SHOW_PATHS
 
-            pi_path = settings.AWX_PROOT_BASE_PATH
-            if not self.instance.is_isolated() and not self.instance.is_containerized:
-                pi_path = tempfile.mkdtemp(
-                    prefix='ansible_runner_pi_',
-                    dir=settings.AWX_PROOT_BASE_PATH
-                )
-                os.chmod(pi_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
-                self.cleanup_paths.append(pi_path)
+            pi_path = os.path.split(private_data_dir)[0]
 
             process_isolation_params = {
                 'process_isolation': True,

awx/main/tests/unit/test_tasks.py

@@ -552,8 +552,8 @@ def test_uses_process_isolation(self, settings):
         task.should_use_proot = lambda instance: True
         task.instance = job
 
-        private_data_dir = '/foo'
-        cwd = '/bar'
+        private_data_dir = os.path.join(settings.AWX_PROOT_BASE_PATH, 'foo')
+        cwd = '/the/bar'
 
         settings.AWX_PROOT_HIDE_PATHS = ['/AWX_PROOT_HIDE_PATHS1', '/AWX_PROOT_HIDE_PATHS2']
         settings.ANSIBLE_VENV_PATH = '/ANSIBLE_VENV_PATH'
@@ -578,7 +578,7 @@ def test_uses_process_isolation(self, settings):
                   '/AWX_PROOT_HIDE_PATHS1',
                   '/AWX_PROOT_HIDE_PATHS2']:
             assert p in process_isolation_params['process_isolation_hide_paths']
-        assert 9 == len(process_isolation_params['process_isolation_hide_paths'])
+        assert 11 == len(process_isolation_params['process_isolation_hide_paths'])
         assert '/ANSIBLE_VENV_PATH' in process_isolation_params['process_isolation_ro_paths']
         assert '/AWX_VENV_PATH' in process_isolation_params['process_isolation_ro_paths']
         assert 2 == len(process_isolation_params['process_isolation_ro_paths'])

awx/plugins/isolated/awx_isolated_cleanup.py

@@ -19,7 +19,6 @@
 
 import glob
 import os
-import re
 import shutil
 import datetime
 import subprocess
@@ -38,32 +37,35 @@ def main():
     # this datetime, then it will be deleted because its job has finished
     job_cutoff = datetime.datetime.now() - datetime.timedelta(hours=1)
 
-    for search_pattern in [
-        '/tmp/awx_[0-9]*_*', '/tmp/ansible_runner_pi_*',
-    ]:
-        for path in glob.iglob(search_pattern):
-            st = os.stat(path)
-            modtime = datetime.datetime.fromtimestamp(st.st_mtime)
+    BASE_DIR = '/tmp'
 
-            if modtime > job_cutoff:
-                continue
-            elif modtime > folder_cutoff:
+    bwrap_pattern = 'bwrap_[0-9]*_*'
+    private_data_dir_pattern = 'awx_[0-9]*_*'
+
+    bwrap_path_pattern = os.path.join(BASE_DIR, bwrap_pattern)
+
+    for bwrap_path in glob.iglob(bwrap_path_pattern):
+        st = os.stat(bwrap_path)
+        modtime = datetime.datetime.fromtimestamp(st.st_mtime)
+
+        if modtime > job_cutoff:
+            continue
+        elif modtime > folder_cutoff:
+            private_data_dir_path_pattern = os.path.join(BASE_DIR, bwrap_path, private_data_dir_pattern)
+            private_data_dir_path = next(glob.iglob(private_data_dir_path_pattern), None)
+            if private_data_dir_path:
                 try:
-                    re_match = re.match(r'\/tmp\/awx_\d+_.+', path)
-                    if re_match is not None:
-                        try:
-                            if subprocess.check_call(['ansible-runner', 'is-alive', path]) == 0:
-                                continue
-                        except subprocess.CalledProcessError:
-                            # the job isn't running anymore, clean up this path
-                            module.debug('Deleting path {} its job has completed.'.format(path))
-                except (ValueError, IndexError):
-                    continue
-            else:
-                module.debug('Deleting path {} because modification date is too old.'.format(path))
-            changed = True
-            paths_removed.add(path)
-            shutil.rmtree(path)
+                    if subprocess.check_call(['ansible-runner', 'is-alive', private_data_dir_path]) == 0:
+                        continue
+                except subprocess.CalledProcessError:
+                    # the job isn't running anymore, clean up this path
+                    module.debug('Deleting path {} its job has completed.'.format(bwrap_path))
+            module.debug('Deleting path {} due to private_data_dir not being found.'.format(bwrap_path))
+        else:
+            module.debug('Deleting path {} because modification date is too old.'.format(bwrap_path))
+        changed = True
+        paths_removed.add(bwrap_path)
+        shutil.rmtree(bwrap_path)
 
     module.exit_json(changed=changed, paths_removed=list(paths_removed))