Support remote URLs for loading scripts

requirements.txt

@@ -9,3 +9,4 @@ clinto
 tox
 boto
 django-storages-redux
+titlecase

wooey/backend/utils.py

@@ -28,6 +28,8 @@
 
 from .. import settings as wooey_settings
 
+from titlecase import titlecase
+
 
 def sanitize_name(name):
     return name.replace(' ', '_').replace('-', '_')
@@ -37,6 +39,11 @@ def sanitize_string(value):
     return value.replace('"', '\\"')
 
 
+def titlecase_name(name):
+    name = name.replace('_', ' ').replace('-', ' ')
+    return titlecase(name)
+
+
 def get_storage(local=True):
     if wooey_settings.WOOEY_EPHEMERAL_FILES:
         storage = default_storage.local_storage if local else default_storage
@@ -166,17 +173,22 @@ def get_storage_object(path, local=False):
     return obj
 
 
-def add_wooey_script(script_version=None, script_path=None, group=None):
-    # There is a class called 'Script' which contains the general information about a script. However, that is not where the file details
-    # of the script lie. That is the ScriptVersion model. This allows the end user to tag a script as a favorite/etc. and set
-    # information such as script descriptions/names that do not constantly need to be updated with every version change. Thus,
-    # a ScriptVersion stores the file info and such.
+def add_wooey_script(script_version=None, script_path=None, group=None, script_name=None):
+    """
+    There is a class called 'Script' which contains the general information about a script. However, that is not where the file details
+    of the script lie. That is the ScriptVersion model. This allows the end user to tag a script as a favorite/etc. and set
+    information such as script descriptions/names that do not constantly need to be updated with every version change. Thus,
+    a ScriptVersion stores the file info and such.
+    """
+
     from ..models import Script, ScriptGroup, ScriptParameter, ScriptParameterGroup, ScriptVersion
-    # if we are adding through the admin, at this point the file will be saved already and this method will be receiving
-    # the scriptversion object. Otherwise, we are adding through the managementment command. In this case, the file will be
-    # a location and we need to setup the Script and ScriptVersion in here.
+
+    # If we are adding through the admin, at this point the file will be saved already and this method will be receiving
+    # the scriptversion object. Otherwise, we are adding through the managementment command. In this case, the file
+    # will be a location and we need to setup the Script and ScriptVersion in here.
 
     local_storage = get_storage(local=True)
+
     if script_version is not None:
         # we are updating the script here or creating it through the admin
 
@@ -217,7 +229,8 @@ def add_wooey_script(script_version=None, script_path=None, group=None):
     if isinstance(group, ScriptGroup):
         group = group.group_name
     if group is None:
-        group = 'Wooey Scripts'
+        group = wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP
+
     basename, extension = os.path.splitext(script)
     filename = os.path.split(basename)[1]
 
@@ -235,9 +248,13 @@ def add_wooey_script(script_version=None, script_path=None, group=None):
     except:
         sys.stderr.write('Error parsing version, defaulting to 1. Error message:\n {}'.format(traceback.format_exc()))
         version_string = '1'
+
+    if script_name is None:
+        script_name = d['name']
+
     if script_version is None:
         # we are being loaded from the management command, create/update our script/version
-        script_kwargs = {'script_group': script_group, 'script_name': d['name']}
+        script_kwargs = {'script_group': script_group, 'script_name': script_name}
         version_kwargs = {'script_version': version_string, 'script_path': local_file, 'default_version': True}
         # does this script already exist in the database?
         script_created = Script.objects.filter(**script_kwargs).count() == 0
@@ -263,17 +280,20 @@ def add_wooey_script(script_version=None, script_path=None, group=None):
             else:
                 # get the largest iteration and add 1 to it
                 next_iteration = sorted([i.script_iteration for i in current_versions])[-1]+1
+                current_versions.update(default_version=False)
+
             version_kwargs.update({'script_iteration': next_iteration})
         version_kwargs.update({'script': wooey_script})
         script_version = ScriptVersion(**version_kwargs)
         script_version._script_cl_creation = True
+        script_version.default_version = True
         script_version.save()
     else:
         # we are being created/updated from the admin
         if not script_version.script.script_description:
             script_version.script.script_description = d['description']
         if not script_version.script.script_name:
-            script_version.script.script_name = d['name']
+            script_version.script.script_name = script_name
         past_versions = ScriptVersion.objects.filter(script=script_version.script, script_version=version_string).exclude(pk=script_version.pk)
         script_version.script_iteration = past_versions.count()+1
         past_versions.update(default_version=False)

wooey/management/commands/addscript.py

@@ -1,23 +1,36 @@
 __author__ = 'chris'
 import os
 import sys
+import tempfile
+
 from django.core.management.base import BaseCommand, CommandError
 from django.core.files import File
-from django.conf import settings
 
-from ...backend.utils import add_wooey_script, get_storage, default_storage
+from ...backend.utils import add_wooey_script, get_storage, default_storage, titlecase_name
 from ... import settings as wooey_settings
 
+try:
+    import urllib as request
+    import urlparse
+except ImportError:
+    import urllib.request as request
+    import urllib.parse as urlparse
+
+import zipfile
+import tarfile
+
+ACCEPTED_ARCHIVE_EXTENSIONS = ['.zip', '.gz', '.gzip', '.tgz']
+
 
 class Command(BaseCommand):
     help = 'Adds a script to Wooey'
 
     def add_arguments(self, parser):
         parser.add_argument('script', type=str, help='A script or folder of scripts to add to Wooey.')
         parser.add_argument('--group',
-            dest='group',
-            default='Wooey Scripts',
-            help='The name of the group to create scripts under. Default: Wooey Scripts')
+                            dest='group',
+                            default=wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP,
+                            help='The name of the group to create scripts under. Default: Scripts')
 
     def handle(self, *args, **options):
         script = options.get('script')
@@ -26,15 +39,52 @@ def handle(self, *args, **options):
                 script = args[0]
             else:
                 raise CommandError('You must provide a script path or directory containing scripts.')
+
+        # Check for remote URL; zipfile, etc.
+        # if it is download, extract to temporary folder + substitute scriptpath
+        if urlparse.urlparse(script).scheme != "":
+            # We have remote URL, download to temporary file with same suffix
+            _, ext = os.path.splitext(script)
+            tfile = tempfile.NamedTemporaryFile(suffix=ext)
+            request.urlretrieve(script, tfile.name)
+            script = tfile.name
+
+        if any([script.endswith(ext) for ext in ACCEPTED_ARCHIVE_EXTENSIONS]):
+            # We have an archive; create a temporary folder and extract there
+            tfolder = tempfile.mkdtemp()
+            if script.endswith('.zip'):
+               with zipfile.ZipFile(script, "r") as zf:
+                    zf.extractall(tfolder)
+
+            else:
+                # Must be gzip
+                with tarfile.open(script, 'r:gz') as tf:
+                    tf.extractall(tfolder)
+
+            # Set the script path to the temporary folder and continue as normal
+            script = tfolder
+
         if not os.path.exists(script):
             raise CommandError('{0} does not exist.'.format(script))
-        group = options.get('group', 'Wooey Scripts')
-        scripts = [os.path.join(script, i) for i in os.listdir(script)] if os.path.isdir(script) else [script]
+        group = options.get('group', wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP)
+
+        scripts = []
+        if os.path.isdir(script):
+            for dirpath, _, fnames in os.walk(script):
+                scripts.extend([os.path.join(dirpath, fn) for fn in fnames])
+
+        else:
+            scripts.append(script) # Single script
+
         converted = 0
         for script in scripts:
             if script.endswith('.pyc') or '__init__' in script:
                 continue
             if script.endswith('.py'):
+                # Get the script name here (before storage changes it)
+                script_name = os.path.splitext(os.path.basename(script))[0]  # Get the base script name
+                script_name = titlecase_name(script_name)
+
                 sys.stdout.write('Converting {}\n'.format(script))
                 # copy the script to our storage
                 with open(script, 'r') as f:
@@ -43,7 +93,7 @@ def handle(self, *args, **options):
                         # save it locally as well (the default_storage will default to the remote store)
                         local_storage = get_storage(local=True)
                         local_storage.save(os.path.join(wooey_settings.WOOEY_SCRIPT_DIR, os.path.split(script)[1]), File(f))
-                res = add_wooey_script(script_path=script, group=group)
+                res = add_wooey_script(script_path=script, group=group, script_name=script_name)
                 if res['valid']:
                     converted += 1
         sys.stdout.write('Converted {} scripts\n'.format(converted))

wooey/settings.py

@@ -1,7 +1,7 @@
 __author__ = 'chris'
 import os
 from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _l, ugettext as _
 
 
 def get(key, default):
@@ -18,5 +18,5 @@ def get(key, default):
 WOOEY_SHOW_LOCKED_SCRIPTS = get('WOOEY_SHOW_LOCKED_SCRIPTS', True)
 WOOEY_EPHEMERAL_FILES = get('WOOEY_EPHEMERAL_FILES', False)
 WOOEY_DEFAULT_SCRIPT_GROUP = get('WOOEY_DEFAULT_SCRIPT_GROUP', _('Scripts'))
-WOOEY_SITE_NAME = get('WOOEY_SITE_NAME', _('Wooey!'))
-WOOEY_SITE_TAG = get('WOOEY_SITE_TAG', _('A web UI for Python scripts'))
+WOOEY_SITE_NAME = get('WOOEY_SITE_NAME', 'Wooey!')
+WOOEY_SITE_TAG = get('WOOEY_SITE_TAG', _l('A web UI for Python scripts'))