Improved manifest

Dockerfile

@@ -50,6 +50,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install --upgrade pip
+RUN pip3 install setuptools
 
 # Install & setup the planutils
 RUN pip3 install planutils --trusted-host pypi.org --trusted-host files.pythonhosted.org

MANIFEST.in

@@ -1 +1 @@
-include planutils/planner_configs/*.json
+recursive-include planutils/packages *

planutils/__init__.py

@@ -1,58 +1,120 @@
 
 import argparse, os
 
+from planutils import settings
+from planutils.package_installation import PACKAGES
+
 
 def setup():
 
-    assert not_setup_yet(), "Error: planutils is already setup. Remove ~/.planutils to reset (warning: all cached planners will be lost)."
+    if setup_done():
+        print("\nError: planutils is already setup. Setting up again will wipe all cached packages and settings.")
+        if input("  Proceed? [y/N] ").lower() in ['y', 'yes']:
+            os.system("rm -rf %s" % os.path.join(os.path.expanduser('~'), '.planutils'))
+        else:
+            return
+
+    CUR_DIR = os.path.dirname(os.path.abspath(__file__))
 
     print("\nCreating ~/.planutils...")
     os.mkdir(os.path.join(os.path.expanduser('~'), '.planutils'))
     os.mkdir(os.path.join(os.path.expanduser('~'), '.planutils', 'bin'))
-    os.mkdir(os.path.join(os.path.expanduser('~'), '.planutils', 'bin', 'images'))
+
+    settings.save({
+        'installed': []
+    })
+
+    os.symlink(os.path.join(CUR_DIR, 'packages'),
+               os.path.join(os.path.expanduser('~'), '.planutils', 'packages'))
 
     print("Adding bin folder to path (assuming ~/.bashrc exists)...")
-    os.system('echo \'export PATH="$HOME/.planutils/bin:$PATH"\' >> ~/.bashrc')
-
-    print("Installing planner scripts...")
-    from planutils.planner_installation import PLANNERS
-    for p in PLANNERS:
-        script  = "#!/bin/bash\n"
-        script += "echo\n"
-        script += "echo 'Planner not installed!'\n"
-        script += "read -p \"Download & install? [y/n] \" varchoice\n"
-        script += "if [ $varchoice == \"y\" ]\n"
-        script += "then\n"
-        script += "  planutils --install " + p + "\n"
-        script += "fi\n"
-        script += "echo"
-        with open(os.path.join(os.path.expanduser('~'), '.planutils', 'bin', p), 'w') as f:
-            f.write(script)
-        os.chmod(os.path.join(os.path.expanduser('~'), '.planutils', 'bin', p), 0o0755)
+    with open(os.path.join(os.path.expanduser('~'), '.bashrc'), "a+") as f:
+        f.write("export PLANUTILS_PREFIX=\"~/.planutils\"")
+        f.write("export PATH=\"$PLANUTILS_PREFIX/bin:$PATH\"")
+
+    print("Installing package scripts...")
+    for p in PACKAGES:
+        if PACKAGES[p]['runnable']:
+            script  = "#!/bin/bash\n"
+            script += "if $(planutils check-installed %s)\n" % p
+            script += "then\n"
+            script += "  ~/.planutils/packages/%s/run $@\n" % p
+            script += "else\n"
+            script += "  echo\n"
+            script += "  echo 'Package not installed!'\n"
+            script += "  read -r -p \"  Download & install? [Y/n] \" varchoice\n"
+            script += "  varchoice=${varchoice,,}\n" # tolower
+            script += "  if ! [[ \"$varchoice\" =~ ^(no|n)$ ]]\n"
+            script += "  then\n"
+            script += "    if planutils install %s;\n" % p
+            script += "    then\n"
+            script += "      echo 'Successfully installed %s!'\n" % p
+            script += "      echo\n"
+            script += "      echo \"Original command: %s $@\"\n" % p
+            script += "      read -r -p \"  Re-run command? [Y/n] \" varchoice\n"
+            script += "      varchoice=${varchoice,,}\n" # tolower
+            script += "      if ! [[ \"$varchoice\" =~ ^(no|n)$ ]]\n"
+            script += "      then\n"
+            script += "        ~/.planutils/packages/%s/run $@\n" % p
+            script += "      fi\n"
+            script += "    fi\n"
+            script += "  fi\n"
+            script += "  echo\n"
+            script += "fi\n"
+            with open(os.path.join(os.path.expanduser('~'), '.planutils', 'bin', p), 'w') as f:
+                f.write(script)
+            os.chmod(os.path.join(os.path.expanduser('~'), '.planutils', 'bin', p), 0o0755)
 
 
     print("\nAll set! Be sure to start a new bash session or update your PATH variable to include ~/.planutils/bin\n")
 
-def not_setup_yet():
-    return not os.path.exists(os.path.join(os.path.expanduser('~'), '.planutils'))
+def setup_done():
+    return os.path.exists(os.path.join(os.path.expanduser('~'), '.planutils'))
+
 
 def main():
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument("-i", "--install",
-                        help="install an individual or collection of planners ('list' shows the options)",
-                        metavar="{planner or collection or list}")
-
-    parser.add_argument("-s", "--setup", help="setup planutils for current user", action="store_true")
-
+    parser = argparse.ArgumentParser(prog="planutils")
+    subparsers = parser.add_subparsers(help='sub-command help', dest='command')
+
+    parser_install = subparsers.add_parser('install', help='install package(s) such as a planner')
+    parser_install.add_argument('package', help='package name', nargs='+')
+
+    parser_uninstall = subparsers.add_parser('uninstall', help='uninstall package(s)')
+    parser_uninstall.add_argument('package', help='package name', nargs='+')
+
+    parser_checkinstalled = subparsers.add_parser('check-installed', help='check if a package is installed')
+    parser_checkinstalled.add_argument('package', help='package name')
+
+    parser_list = subparsers.add_parser('list', help='list the available packages')
+    parser_setup = subparsers.add_parser('setup', help='setup planutils for current user')
+    parser_upgrade = subparsers.add_parser('upgrade', help='upgrade all of the installed packages')
+
     args = parser.parse_args()
 
-    if args.setup:
+    if 'setup' == args.command:
         setup()
-    elif not_setup_yet():
-        print("\nPlease run 'planutils --setup' before using utility.\n")
-        exit()
+    elif not setup_done():
+        print("\nPlease run 'planutils setup' before using utility.\n")
+
+    elif 'check-installed' == args.command:
+        from planutils.package_installation import check_installed
+        exit({True:0, False:1}[check_installed(args.package)])
+
+    elif 'install' == args.command:
+        from planutils.package_installation import install
+        exit({True:0, False:1}[install(args.package)])
+
+    elif 'uninstall' == args.command:
+        from planutils.package_installation import uninstall
+        uninstall(args.package)
+
+    elif 'list' == args.command:
+        from planutils.package_installation import package_list
+        package_list()
+
+    elif 'upgrade' == args.command:
+        from planutils.package_installation import upgrade
+        upgrade()
 
-    if args.install:
-        from planutils.planner_installation import install
-        install(args.install)
+    else:
+        parser.print_help()

planutils/package_installation.py

@@ -0,0 +1,173 @@
+
+import json, os, glob, subprocess
+from collections import defaultdict
+
+from planutils import settings
+
+PACKAGES = {}
+
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def check_package(target, manifest):
+    assert os.path.exists(manifest), "Error: Manifest must be defined for %s" % target
+    with open(manifest, 'r') as f:
+        config = json.load(f)
+    for key in ['name', 'description', 'dependencies', 'install-size']:
+        assert key in config, "Error: Manifest for %s must include '%s'" % (base, key)
+
+
+for conf_file in glob.glob(os.path.join(CUR_DIR, 'packages', '*')):
+    base = os.path.basename(conf_file)
+    if base not in ['README.md', 'TEMPLATE']:
+        assert base not in PACKAGES, "Error: Duplicate package config -- %s" % base
+        check_package(base, os.path.join(conf_file, 'manifest.json'))
+        with open(os.path.join(conf_file, 'manifest.json'), 'r') as f:
+            config = json.load(f)
+        PACKAGES[base] = config
+        PACKAGES[base]['runnable'] = os.path.exists(os.path.join(conf_file, 'run'))
+
+
+def check_installed(target):
+    return target in settings.load()['installed']
+
+
+def uninstall(targets):
+
+    for target in targets:
+        if target not in PACKAGES:
+            print("Error: Package not found -- %s" % target)
+            return
+
+    to_check = []
+    for target in targets:
+        if check_installed(target):
+            to_check.append(target)
+        else:
+            print("%s isn't installed." % target)
+
+    if not to_check:
+        return
+
+    s = settings.load()
+    # map a package to all those that depend on it
+    dependency_mapping = defaultdict(set)
+    for p in s['installed']:
+        for dep in PACKAGES[p]['dependencies']:
+            dependency_mapping[dep].add(p)
+
+    # compute all the packages that will be removed
+    to_remove = set()
+    while to_check:
+        check = to_check.pop(0)
+        to_remove.add(check)
+        to_check.extend(list(dependency_mapping[check]))
+
+    print("\nAbout to remove the following packages: %s" % ', '.join(to_remove))
+    if input("  Proceed? [y/N] ").lower() in ['y', 'yes']:
+        for package in to_remove:
+            print ("Uninstalling %s..." % package)
+            subprocess.call('./uninstall', cwd=os.path.join(CUR_DIR, 'packages', package))
+            print ("Finished uninstalling %s" % package)
+            s['installed'].remove(package)
+
+    # Search for any packages that may be no longer required
+    dependency_mapping = defaultdict(set)
+    for p in PACKAGES:
+        for dep in PACKAGES[p]['dependencies']:
+            dependency_mapping[dep].add(p)
+
+    possible_deletions = [d for p in to_remove for d in PACKAGES[p]['dependencies']]
+    while possible_deletions:
+        package = possible_deletions.pop(0)
+        # Consider removing if (1) it's installed; and (2) nothing that requires this is installed
+        if (package in s['installed']) and (not any([p in s['installed'] for p in dependency_mapping[package]])):
+            print("\nPackage may no longer be required: %s" % package)
+            if input("  Remove? [y/N] ").lower() in ['y', 'yes']:
+                print ("Uninstalling %s..." % package)
+                subprocess.call('./uninstall', cwd=os.path.join(CUR_DIR, 'packages', package))
+                print ("Finished uninstalling %s" % package)
+                s['installed'].remove(package)
+                possible_deletions.extend(PACKAGES[package]['dependencies'])
+
+    settings.save(s)
+
+
+def package_list():
+    print("\nInstalled:")
+    installed = set(settings.load()['installed'])
+    for p in installed:
+        print("  %s: %s" % (p, PACKAGES[p]['name']))
+
+    print("\nAvailable:")
+    for p in PACKAGES:
+        if p not in installed:
+            print("  %s: %s" % (p, PACKAGES[p]['name']))
+    print()
+
+def upgrade():
+    s = settings.load()
+    for package in s['installed']:
+        print("Upgrading %s..." % package)
+        subprocess.call('./uninstall', cwd=os.path.join(CUR_DIR, 'packages', package))
+        subprocess.call('./install', cwd=os.path.join(CUR_DIR, 'packages', package))
+
+def install(targets):
+    for target in targets:
+        if target not in PACKAGES:
+            print("Error: Package not found -- %s" % target)
+            return False
+
+    # Compute all those that will need to be installed
+    to_check = []
+    for target in targets:
+        if check_installed(target):
+            print("%s is already installed." % target)
+        else:
+            to_check.append(target)
+
+    done = set()
+    to_install = []
+    while to_check:
+        check = to_check.pop(0)
+        if check not in done:
+            done.add(check)
+            if not check_installed(check):
+                to_install.append(check)
+                to_check.extend(PACKAGES[check]['dependencies'])
+
+    to_install.reverse()
+
+    if to_install:
+        to_install_desc = ["%s (%s)" % (pkg, PACKAGES[pkg]['install-size']) for pkg in to_install]
+        print("\nAbout to install the following packages: %s" % ', '.join(to_install_desc))
+        if input("  Proceed? [Y/n] ").lower() in ['', 'y', 'yes']:
+            installed = []
+            for package in to_install:
+                package_path = os.path.join(CUR_DIR, 'packages', package)
+                print("Installing %s..." % package)
+                try:
+                    installed.append(package)
+                    subprocess.check_call('./install', cwd=package_path)
+                    size = subprocess.check_output('du -sh .',
+                                                   cwd=package_path,
+                                                   shell=True,
+                                                   encoding='utf-8').split('\t')[0]
+                    print("Finished installing %s (size: %s)" % (package, size))
+                except subprocess.CalledProcessError:
+                    print("\nError installing %s. Rolling back changes..." % package)
+                    for p in installed:
+                        subprocess.call('./uninstall', cwd=os.path.join(CUR_DIR, 'packages', p))
+                    return False
+
+            s = settings.load()
+            s['installed'].extend(installed)
+            settings.save(s)
+            return True
+        else:
+            print("Aborting installation.")
+    else:
+        print("Nothing left to install.")
+
+    return False
+

planutils/packages/README.md

@@ -0,0 +1,24 @@
+# planutils packages
+
+Each package must contain the files `manifest.json`, `install`, and `uninstall`. Optionally, `run` can be included if the package installs a command-line executable (e.g., planners). An example of all four files can be found in the `TEMPLATE` directory, and a description of each follows.
+
+## manifest.json
+
+Details on the package. Must include:
+
+1. **name**: Long-form name of the package.
+2. **shortname**: Short-form name that will be used for installation, running, etc.
+3. **description**: General description of the package.
+4. **dependencies**: List of shortnames for other `planutils` packages that are required.
+
+## install
+
+Script to install the package along with any dependencies not part of `planutils`.
+
+## uninstall
+
+Cleanup script to remove the package installation.
+
+## run
+
+(optional) Script to run the installed package. `shortname` specified in the `manifest.json` file will be used for the command-line invocation of this script.

planutils/packages/TEMPLATE/install

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# No need to install planutils dependencies
+# No need to check if already installed
+# The install script will be run from the package's directory
+
+# To use if root is required
+#[ "$UID" -eq 0 ] || (echo "installation requires root access"; exec sudo "$0" "$@")
+
+# Install general linux dependencies
+
+# General setup / configuration
+
+
+# Recipe for singularity images
+## Fetch the image
+#singularity pull --name <image name> <singularity shub url>

planutils/packages/TEMPLATE/manifest.json

@@ -0,0 +1,6 @@
+{
+    "name": "Full name of package",
+    "description": "General description of the package",
+    "install-size": "unknown",
+    "dependencies": ["list", "of", "dependencies"]
+}

planutils/packages/TEMPLATE/run

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+# whatever command-line method needs to be used to run this package