Vfolder command and migration adjustments

pootle/apps/pootle_store/models.py

@@ -1624,13 +1624,12 @@ def all_pootle_paths(self):
         """
         pootle_paths = super(Store, self).all_pootle_paths()
         if 'virtualfolder' in settings.INSTALLED_APPS:
-            vftis = self.parent_vf_treeitems.values_list(
-                "vfolder__location", "pootle_path")
-            for location, pootle_path in vftis:
+            vftis = self.parent_vf_treeitems
+            for pootle_path in vftis.values_list("pootle_path", flat=True):
                 pootle_paths.extend(
                     [p for p
                      in get_all_pootle_paths(pootle_path)
-                     if p.count('/') > location.count('/')])
+                     if p.count('/') > 3])
         return pootle_paths
 
     # # # /TreeItem

pootle/apps/virtualfolder/apps.py

@@ -17,6 +17,7 @@ class PootleVirtualFolderConfig(AppConfig):
     verbose_name = "Pootle Virtual Folders"
 
     def ready(self):
+        importlib.import_module("virtualfolder.getters")
         importlib.import_module("virtualfolder.receivers")
         importlib.import_module("virtualfolder.providers")
         importlib.import_module("virtualfolder.getters")

pootle/apps/virtualfolder/delegate.py

@@ -10,3 +10,5 @@
 
 
 path_matcher = Getter()
+vfolders_data_tool = Getter()
+vfolder_finder = Getter()

pootle/apps/virtualfolder/getters.py

@@ -8,11 +8,24 @@
 
 from pootle.core.delegate import search_backend
 from pootle.core.plugin import getter
+from pootle_store.models import Store
 
+from .delegate import path_matcher, vfolder_finder
 from .models import VirtualFolder
 from .search import VFolderDBSearchBackend
+from .utils import VirtualFolderFinder, VirtualFolderPathMatcher
 
 
 @getter(search_backend, sender=VirtualFolder)
 def get_vfolder_search_backend(**kwargs_):
     return VFolderDBSearchBackend
+
+
+@getter(path_matcher, sender=VirtualFolder)
+def vf_path_matcher_getter(**kwargs_):
+    return VirtualFolderPathMatcher
+
+
+@getter(vfolder_finder, sender=Store)
+def store_vf_finder_getter(**kwargs_):
+    return VirtualFolderFinder

pootle/apps/virtualfolder/management/commands/add_vfolders.py

@@ -16,6 +16,7 @@
 from django.core.exceptions import ValidationError
 from django.core.management.base import BaseCommand, CommandError
 
+from pootle.core.url_helpers import split_pootle_path
 from virtualfolder.models import VirtualFolder
 
 
@@ -29,6 +30,33 @@ def add_arguments(self, parser):
             help="JSON vfolder configuration file",
         )
 
+    def parse_vfolder_rules(self, location, old_rules):
+        """Extract languages, projects and new rules from location and rules."""
+        languages = set()
+        projects = set()
+        new_rules = set()
+
+        full_rules = [location + old_rule.strip() for old_rule in old_rules]
+
+        for full_rule in full_rules:
+            lang_code, proj_code, dir_path, fname = split_pootle_path(full_rule)
+            if fname:
+                new_rules.add(dir_path + fname)
+            else:
+                new_rules.add(dir_path + "*")
+            languages.add(lang_code)
+            projects.add(proj_code)
+
+        if "{LANG}" in languages:
+            languages = set()
+
+        if "{PROJ}" in projects:
+            projects = set()
+
+        new_rules=",".join(new_rules)
+
+        return languages, projects, new_rules
+
     def handle(self, **options):
         """Add virtual folders from file."""
 
@@ -57,41 +85,73 @@ def handle(self, **options):
         errored_count = 0
 
         for vfolder_item in vfolders:
-            vfolder_item['name'] = vfolder_item['name'].lower()
+            vfolder_item['name'] = vfolder_item['name'].strip().lower()
 
             # Put all the files for each virtual folder as a list and save it
             # as its filter rules.
-            vfolder_item['filter_rules'] = ','.join(
-                vfolder_item['filters']['files'])
+            languages, projects, new_rules = self.parse_vfolder_rules(
+                vfolder_item['location'].strip(),
+                vfolder_item['filters']['files']
+            )
+
+            vfolder_item['filter_rules'] = new_rules
 
             if 'filters' in vfolder_item:
                 del vfolder_item['filters']
 
             # Now create or update the virtual folder.
             try:
                 # Retrieve the virtual folder if it exists.
-                vfolder = VirtualFolder.objects.get(
-                    name=vfolder_item['name'],
-                    location=vfolder_item['location'],
-                )
+                vfolder = VirtualFolder.objects.get(name=vfolder_item['name'])
             except VirtualFolder.DoesNotExist:
                 # If the virtual folder doesn't exist yet then create it.
                 try:
                     self.stdout.write(u'Adding new virtual folder %s...' %
                                       vfolder_item['name'])
+                    vfolder_item['all_projects'] = not projects
+                    vfolder_item['all_languages'] = not languages
                     vfolder = VirtualFolder(**vfolder_item)
                     vfolder.save()
                 except ValidationError as e:
                     errored_count += 1
                     self.stdout.write('FAILED')
                     self.stderr.write(e)
                 else:
+                    if projects:
+                        vfolder.projects.add(
+                            *Project.objects.filter(code__in=projects)
+                        )
+                    if languages:
+                        vfolder.languages.add(
+                            *Language.objects.filter(code__in=languages)
+                        )
                     self.stdout.write('DONE')
                     added_count += 1
             else:
                 # Update the already existing virtual folder.
                 changed = False
 
+                if vfolder.all_projects != not projects:
+                    vfolder.all_projects = not projects
+                    changed = True
+                    logging.debug("'All projects' for virtual folder '%s' "
+                                  "will be changed.", vfolder.name)
+
+                if vfolder.all_languages != not languages:
+                    vfolder.all_languages = not languages
+                    changed = True
+                    logging.debug("'All languages' for virtual folder '%s' "
+                                  "will be changed.", vfolder.name)
+
+                if projects:
+                    vfolder.projects.set(
+                        *Project.objects.filter(code__in=projects)
+                    )
+                if languages:
+                    vfolder.languages.set(
+                        *Language.objects.filter(code__in=languages)
+                    )
+
                 if vfolder.filter_rules != vfolder_item['filter_rules']:
                     vfolder.filter_rules = vfolder_item['filter_rules']
                     changed = True

pootle/apps/virtualfolder/migrations/0011_add_projects_languages.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pootle_language', '0002_case_insensitive_schema'),
+        ('pootle_project', '0010_add_reserved_code_validator'),
+        ('virtualfolder', '0010_remove_virtualfolder_units'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='virtualfolder',
+            name='languages',
+            field=models.ManyToManyField(related_name='vfolders', to='pootle_language.Language', db_index=True),
+        ),
+        migrations.AddField(
+            model_name='virtualfolder',
+            name='projects',
+            field=models.ManyToManyField(related_name='vfolders', to='pootle_project.Project', db_index=True),
+        ),
+    ]

pootle/apps/virtualfolder/migrations/0012_add_all_proj_lang_flags.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('virtualfolder', '0011_add_projects_languages'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='virtualfolder',
+            name='all_languages',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='virtualfolder',
+            name='all_projects',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+from pootle.core.url_helpers import split_pootle_path
+
+
+def parse_vfolder_rules(vf):
+    languages = set()
+    projects = set()
+    new_rules = set()
+
+    full_rules = [vf.location.strip() + rule.strip()
+                  for rule in vf.filter_rules.split(",")]
+
+    for full_rule in full_rules:
+        lang_code, proj_code, dir_path, filename = split_pootle_path(full_rule)
+        if filename:
+            new_rules.add(dir_path + filename)
+        else:
+            new_rules.add(dir_path + "*")
+        languages.add(lang_code)
+        projects.add(proj_code)
+
+    if "{LANG}" in languages:
+        languages = set()
+
+    if "{PROJ}" in projects:
+        projects = set()
+
+    new_rules=",".join(new_rules)
+
+    return languages, projects, new_rules
+
+
+def set_projects_and_languages(app, schema):
+    VirtualFolder = app.get_model("virtualfolder.VirtualFolder")
+    Project = app.get_model("pootle_project.Project")
+    Language = app.get_model("pootle_language.Language")
+    for vf in VirtualFolder.objects.all():
+        languages, projects, new_rules = parse_vfolder_rules(vf)
+        if projects:
+            vf.projects.add(*Project.objects.filter(code__in=projects))
+        if languages:
+            vf.languages.add(*Language.objects.filter(code__in=languages))
+        vf.filter_rules = new_rules
+        vf.all_projects = not projects
+        vf.all_languages = not languages
+        vf.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('virtualfolder', '0012_add_all_proj_lang_flags'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_projects_and_languages)
+    ]

pootle/apps/virtualfolder/migrations/0014_remove_location.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('virtualfolder', '0013_set_projects_languages'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='virtualfolder',
+            unique_together=set([]),
+        ),
+        migrations.RemoveField(
+            model_name='virtualfolder',
+            name='location',
+        ),
+    ]