From 02f99e7e66112bcf0188f1c0fe544a239a1be51e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 20:03:21 +0100 Subject: [PATCH 01/19] Add VirtualFolder utils --- pootle/apps/virtualfolder/utils.py | 220 +++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 pootle/apps/virtualfolder/utils.py diff --git a/pootle/apps/virtualfolder/utils.py b/pootle/apps/virtualfolder/utils.py new file mode 100644 index 00000000000..560089b96a7 --- /dev/null +++ b/pootle/apps/virtualfolder/utils.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +from fnmatch import fnmatch + +from pootle.core.url_helpers import split_pootle_path +from pootle_fs.utils import PathFilter +from pootle_store.models import Store + +from .models import VirtualFolder, VirtualFolderTreeItem + + +def update_vfolder_tree(vf, store): + """For a given VirtualFolder and Store update the VirtualFolderTreeItem + """ + # Create missing VirtualFolderTreeItem tree structure for affected Stores + # after adding or unobsoleting Units. + created = False + try: + vfolder_treeitem = ( + VirtualFolderTreeItem.objects.select_related("directory").get( + directory=store.parent, vfolder=vf)) + except VirtualFolderTreeItem.DoesNotExist: + parts = split_pootle_path(store.parent.pootle_path) + path_parts = ["", parts[0], parts[1], vf.name] + if parts[2]: + path_parts.append(parts[2].strip("/")) + path_parts.append(parts[3]) + pootle_path = "/".join(path_parts) + vfolder_treeitem = ( + VirtualFolderTreeItem.objects.create( + pootle_path=pootle_path, + directory=store.parent, + vfolder=vf)) + created = True + if not created: + # The VirtualFolderTreeItem already existed, so + # calculate again the stats up to the root. + vfolder_treeitem.update_all_cache() + + +class VirtualFolderFinder(object): + """Find vfs for a new store""" + + def __init__(self, store): + self.store = store + + @property + def language(self): + return self.store.translation_project.language + + @property + def project(self): + return self.store.translation_project.project + + @property + def possible_vfolders(self): + return ( + self.project.vfolders.filter(all_languages=True) + | self.language.vfolders.filter(all_projects=True) + | self.project.vfolders.filter(languages=self.language) + | VirtualFolder.objects.filter( + all_languages=True, all_projects=True)) + + def add_to_vfolders(self): + to_add = [] + for vf in self.possible_vfolders: + if vf.path_matcher.should_add_store(self.store): + to_add.append(vf) + if to_add: + self.store.vfolders.add(*to_add) + self.store.set_priority() + for vf in to_add: + update_vfolder_tree(vf, self.store) + + +class VirtualFolderPathMatcher(object): + + tp_path = "/[^/]*/[^/]*/" + + def __init__(self, vf): + self.vf = vf + + @property + def existing_stores(self): + """Currently associated Stores""" + return self.vf.stores.all() + + @property + def languages(self): + """The languages associated with this vfolder + If `all_languages` is set then `None` is returned + """ + if self.vf.all_languages: + return None + return self.vf.languages.values_list("pk", flat=True) + + @property + def projects(self): + """The projects associated with this vfolder + If `all_projects` is set then `None` is returned + """ + if self.vf.all_projects: + return None + return self.vf.projects.values_list("pk", flat=True) + + @property + def matching_stores(self): + """Store qs containing all stores that match + project, language, and rules for this vfolder + """ + return self.filter_from_rules(self.store_qs) + + @property + def rules(self): + """Glob matching rules""" + return ( + "%s" % r.strip() + for r + in self.vf.filter_rules.split(",")) + + @property + def store_manager(self): + """The root object manager for finding/adding stores""" + return Store.objects + + @property + def store_qs(self): + """The stores qs without any rule filtering""" + return self.filter_projects( + self.filter_languages( + self.store_manager)) + + def add_and_remove_stores(self): + """Add Stores that should be associated but arent, delete Store + associations for Stores that are associated but shouldnt be + """ + existing_stores = set(self.existing_stores) + matching_stores = set(self.matching_stores) + to_add = matching_stores - existing_stores + to_remove = existing_stores - matching_stores + if to_add: + self.add_stores(to_add) + if to_remove: + self.remove_stores(to_remove) + return to_add, to_remove + + def add_stores(self, stores): + """Associate a Store""" + self.vf.stores.add(*stores) + + def filter_from_rules(self, qs): + filtered_qs = qs.none() + for rule in self.rules: + filtered_qs = ( + filtered_qs + | qs.filter(pootle_path__regex=self.get_rule_regex(rule))) + return filtered_qs + + def filter_languages(self, qs): + if self.languages is None: + return qs + return qs.filter( + translation_project__language_id__in=self.languages) + + def filter_projects(self, qs): + if self.projects is None: + return qs + return qs.filter( + translation_project__project_id__in=self.projects) + + def get_rule_regex(self, rule): + """For a given *glob* rule, return a pootle_path *regex*""" + return ( + "^%s%s" + % (self.tp_path, + PathFilter().path_regex(rule))) + + def path_matches(self, path): + """Returns bool of whether path is valid for this VF. + """ + for rule in self.rules: + if fnmatch(rule, path): + return True + return False + + def remove_stores(self, stores): + self.vf.stores.remove(*stores) + + def should_add_store(self, store): + if self.store_matches(store) and not self.store_associated(store): + return True + return False + + def store_associated(self, store): + return self.vf.stores.through.objects.filter( + store_id=store.id, + virtualfolder_id=self.vf.id).exists() + + def store_matches(self, store): + return self.path_matches( + "".join(split_pootle_path(store.pootle_path)[2:])) + + def update_stores(self): + """Add and delete Store associations as necessary, and set the + priority for any affected Stores + """ + added, removed = self.add_and_remove_stores() + for store in added: + update_vfolder_tree(self.vf, store) + if store.priority != self.vf.priority: + store.set_priority() + for store in removed: + if store.priority == self.vf.priority: + store.set_priority() From fde20775d314a14dc20fec4784ac736a3d9420de Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 20:02:50 +0100 Subject: [PATCH 02/19] Add path_matcher to virtualfolder model --- pootle/apps/virtualfolder/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index 3e38ea11abf..e379d1b5d7f 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -9,6 +9,7 @@ from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse from django.db import models +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from pootle.core.markup import MarkupField, get_markup_filter_display_name @@ -21,6 +22,8 @@ from pootle_project.models import Project from pootle_store.models import Store +from .delegate import path_matcher + class VirtualFolder(models.Model): @@ -77,6 +80,10 @@ class VirtualFolder(models.Model): class Meta(object): unique_together = ('name', 'location') + @cached_property + def path_matcher(self): + return path_matcher.get(self.__class__)(self) + @property def all_locations(self): """Return a list with all the locations this virtual folder applies. From 9444c29c545ca2c74a9c4be92fc50611f6b2ec90 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 4 Sep 2016 13:29:28 +0100 Subject: [PATCH 03/19] Add virtualfolder getters --- pootle/apps/virtualfolder/apps.py | 1 + pootle/apps/virtualfolder/delegate.py | 2 ++ pootle/apps/virtualfolder/getters.py | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/pootle/apps/virtualfolder/apps.py b/pootle/apps/virtualfolder/apps.py index 5c39de4a237..e7322119723 100644 --- a/pootle/apps/virtualfolder/apps.py +++ b/pootle/apps/virtualfolder/apps.py @@ -17,6 +17,7 @@ class PootleVirtualFolderConfig(AppConfig): verbose_name = "Pootle Virtual Folders" def ready(self): + importlib.import_module("virtualfolder.getters") importlib.import_module("virtualfolder.receivers") importlib.import_module("virtualfolder.providers") importlib.import_module("virtualfolder.getters") diff --git a/pootle/apps/virtualfolder/delegate.py b/pootle/apps/virtualfolder/delegate.py index 4f2185f524b..9f32fca602d 100644 --- a/pootle/apps/virtualfolder/delegate.py +++ b/pootle/apps/virtualfolder/delegate.py @@ -10,3 +10,5 @@ path_matcher = Getter() +vfolders_data_tool = Getter() +vfolder_finder = Getter() diff --git a/pootle/apps/virtualfolder/getters.py b/pootle/apps/virtualfolder/getters.py index 83cdbd1162d..c7e6398388c 100644 --- a/pootle/apps/virtualfolder/getters.py +++ b/pootle/apps/virtualfolder/getters.py @@ -8,11 +8,24 @@ from pootle.core.delegate import search_backend from pootle.core.plugin import getter +from pootle_store.models import Store +from .delegate import path_matcher, vfolder_finder from .models import VirtualFolder from .search import VFolderDBSearchBackend +from .utils import VirtualFolderFinder, VirtualFolderPathMatcher @getter(search_backend, sender=VirtualFolder) def get_vfolder_search_backend(**kwargs_): return VFolderDBSearchBackend + + +@getter(path_matcher, sender=VirtualFolder) +def vf_path_matcher_getter(**kwargs_): + return VirtualFolderPathMatcher + + +@getter(vfolder_finder, sender=Store) +def store_vf_finder_getter(**kwargs_): + return VirtualFolderFinder From afcc46161ac3dbd404abafa46f27e5690d2e77a6 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 5 Sep 2016 19:37:44 +0100 Subject: [PATCH 04/19] Add fields to vfolder model (projects,languages) --- pootle/apps/virtualfolder/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index e379d1b5d7f..0136cc7cb95 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -76,6 +76,14 @@ class VirtualFolder(models.Model): Store, db_index=True, related_name='vfolders') + projects = models.ManyToManyField( + Project, + db_index=True, + related_name='vfolders') + languages = models.ManyToManyField( + Language, + db_index=True, + related_name='vfolders') class Meta(object): unique_together = ('name', 'location') From 316a3fdc98768bc1fe67907a0c8aa79e70536eff Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 17:41:20 +0100 Subject: [PATCH 05/19] Remove location field from virtualfolder --- pootle/apps/virtualfolder/models.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index 0136cc7cb95..441881bf0ba 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -40,14 +40,6 @@ class VirtualFolder(models.Model): null=True, max_length=255) - # any changes to the `location` field may require updating the schema - # see migration 0003_case_sensitive_schema.py - location = models.CharField( - _('Location'), - blank=False, - max_length=255, - help_text=_('Root path where this virtual folder is applied.'), - ) filter_rules = models.TextField( # Translators: This is a noun. _('Filter'), @@ -85,9 +77,6 @@ class VirtualFolder(models.Model): db_index=True, related_name='vfolders') - class Meta(object): - unique_together = ('name', 'location') - @cached_property def path_matcher(self): return path_matcher.get(self.__class__)(self) @@ -131,7 +120,7 @@ def all_locations(self): return [self.location] def __unicode__(self): - return ": ".join([self.name, self.location]) + return self.name def save(self, *args, **kwargs): # Force validation of fields. @@ -149,14 +138,6 @@ def clean_fields(self): """Validate virtual folder fields.""" if self.priority <= 0: raise ValidationError(u'Priority must be greater than zero.') - - elif self.location == "/": - raise ValidationError(u'The "/" location is not allowed. Use ' - u'"/{LANG}/{PROJ}/" instead.') - elif self.location.startswith("/projects/"): - raise ValidationError(u'Locations starting with "/projects/" are ' - u'not allowed. Use "/{LANG}/" instead.') - if not self.filter_rules: raise ValidationError(u'Some filtering rule must be specified.') From 9a13900d5e959aaf9c7d87940633ebc30dfcaf2d Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 20:10:54 +0100 Subject: [PATCH 06/19] Remove location methods from virtualfolder --- pootle/apps/virtualfolder/models.py | 102 ---------------------------- 1 file changed, 102 deletions(-) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index 441881bf0ba..9f44fb4ca6a 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -81,44 +81,6 @@ class VirtualFolder(models.Model): def path_matcher(self): return path_matcher.get(self.__class__)(self) - @property - def all_locations(self): - """Return a list with all the locations this virtual folder applies. - - If the virtual folder location has no {LANG} nor {PROJ} placeholders - then the list only contains its location. If any of the placeholders is - present, then they get expanded to match all the existing languages and - projects. - """ - if "{LANG}/{PROJ}" in self.location: - locations = [] - for lang in Language.objects.all(): - temp = self.location.replace("{LANG}", lang.code) - for proj in Project.objects.all(): - locations.append(temp.replace("{PROJ}", proj.code)) - return locations - elif "{LANG}" in self.location: - try: - project = Project.objects.get(code=self.location.split("/")[2]) - languages = project.languages.iterator() - except Exception: - languages = Language.objects.iterator() - - return [self.location.replace("{LANG}", lang.code) - for lang in languages] - elif "{PROJ}" in self.location: - try: - projects = Project.objects.filter( - translationproject__language__code=self.location.split("/")[1] - ).iterator() - except Exception: - projects = Project.objects.iterator() - - return [self.location.replace("{PROJ}", proj.code) - for proj in projects] - - return [self.location] - def __unicode__(self): return self.name @@ -141,70 +103,6 @@ def clean_fields(self): if not self.filter_rules: raise ValidationError(u'Some filtering rule must be specified.') - def get_adjusted_location(self, pootle_path): - """Return the virtual folder location adjusted to the given path. - - The virtual folder location might have placeholders, which affect the - actual filenames since those have to be concatenated to the virtual - folder location. - """ - count = self.location.count("/") - - if pootle_path.count("/") < count: - raise ValueError("%s is not applicable in %s" % (self, - pootle_path)) - - pootle_path_parts = pootle_path.strip("/").split("/") - location_parts = self.location.strip("/").split("/") - - try: - if (location_parts[0] != pootle_path_parts[0] and - location_parts[0] != "{LANG}"): - raise ValueError("%s is not applicable in %s" % (self, - pootle_path)) - - if (location_parts[1] != pootle_path_parts[1] and - location_parts[1] != "{PROJ}"): - raise ValueError("%s is not applicable in %s" % (self, - pootle_path)) - except IndexError: - pass - - return "/".join(pootle_path.split("/")[:count]) - - def get_adjusted_pootle_path(self, pootle_path): - """Adjust the given pootle path to this virtual folder. - - The provided pootle path is converted to a path that includes the - virtual folder name in the right place. - - For example a virtual folder named vfolder8, with a location - /{LANG}/firefox/browser/ in a path - /af/firefox/browser/chrome/overrides/ gets converted to - /af/firefox/browser/vfolder8/chrome/overrides/ - """ - count = self.location.count('/') - - if pootle_path.count('/') < count: - # The provided pootle path is above the virtual folder location. - path_parts = pootle_path.rstrip('/').split('/') - pootle_path = '/'.join(path_parts + - self.location.split('/')[len(path_parts):]) - - if count < 3: - # If the virtual folder location is not long as a translation - # project pootle path then the returned adjusted location is too - # short, meaning that the returned translate URL will have the - # virtual folder name as the project or language code. - path_parts = pootle_path.split('/') - return '/'.join(path_parts[:3] + [self.name] + path_parts[3:]) - - # If the virtual folder location is as long as a TP pootle path and - # the provided pootle path isn't above the virtual folder location. - lead = self.get_adjusted_location(pootle_path) - trail = pootle_path.replace(lead, '').lstrip('/') - return '/'.join([lead, self.name, trail]) - class VirtualFolderTreeItemManager(models.Manager): use_for_related_fields = True From 7603508de3b38634eaa01804c73599bc0866419e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 11 Sep 2016 06:28:41 +0100 Subject: [PATCH 07/19] Add all_language and all_project flags to virtualfolder --- pootle/apps/virtualfolder/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index 9f44fb4ca6a..138e375974d 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -68,10 +68,12 @@ class VirtualFolder(models.Model): Store, db_index=True, related_name='vfolders') + all_projects = models.BooleanField(default=False) projects = models.ManyToManyField( Project, db_index=True, related_name='vfolders') + all_languages = models.BooleanField(default=False) languages = models.ManyToManyField( Language, db_index=True, From f3a26af933d86a706534e41c41aaf5e9f81a22d5 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 14:34:09 +0100 Subject: [PATCH 08/19] Add migration for vfolder projects and languages m2m tables --- .../migrations/0011_add_projects_languages.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 pootle/apps/virtualfolder/migrations/0011_add_projects_languages.py diff --git a/pootle/apps/virtualfolder/migrations/0011_add_projects_languages.py b/pootle/apps/virtualfolder/migrations/0011_add_projects_languages.py new file mode 100644 index 00000000000..97457819d76 --- /dev/null +++ b/pootle/apps/virtualfolder/migrations/0011_add_projects_languages.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pootle_language', '0002_case_insensitive_schema'), + ('pootle_project', '0010_add_reserved_code_validator'), + ('virtualfolder', '0010_remove_virtualfolder_units'), + ] + + operations = [ + migrations.AddField( + model_name='virtualfolder', + name='languages', + field=models.ManyToManyField(related_name='vfolders', to='pootle_language.Language', db_index=True), + ), + migrations.AddField( + model_name='virtualfolder', + name='projects', + field=models.ManyToManyField(related_name='vfolders', to='pootle_project.Project', db_index=True), + ), + ] From 6630da50be199765a115dcf30c84d6ed76203041 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 11 Sep 2016 06:36:03 +0100 Subject: [PATCH 09/19] Migration to add all_languages and all_projects fields to virtualfolder --- .../0012_add_all_proj_lang_flags.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 pootle/apps/virtualfolder/migrations/0012_add_all_proj_lang_flags.py diff --git a/pootle/apps/virtualfolder/migrations/0012_add_all_proj_lang_flags.py b/pootle/apps/virtualfolder/migrations/0012_add_all_proj_lang_flags.py new file mode 100644 index 00000000000..b63b515ed86 --- /dev/null +++ b/pootle/apps/virtualfolder/migrations/0012_add_all_proj_lang_flags.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('virtualfolder', '0011_add_projects_languages'), + ] + + operations = [ + migrations.AddField( + model_name='virtualfolder', + name='all_languages', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='virtualfolder', + name='all_projects', + field=models.BooleanField(default=False), + ), + ] From 7f85554d56131c058360828308ae27bb4c876d7a Mon Sep 17 00:00:00 2001 From: Leandro Regueiro Date: Sat, 10 Sep 2016 17:09:59 +0100 Subject: [PATCH 10/19] Add data migration to set projects, langs and rules for vfolders --- .../migrations/0013_set_projects_languages.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py diff --git a/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py b/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py new file mode 100644 index 00000000000..0109aadcc4c --- /dev/null +++ b/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + +from pootle.core.url_helpers import split_pootle_path + + +def parse_vfolder_rules(vf): + languages = set() + projects = set() + new_rules = set() + + full_rules = [vf.location + rule for rule in vf.filter_rules.split(",")] + + for full_rule in full_rules: + lang_code, proj_code, dir_path, filename = split_pootle_path(full_rule) + new_rules.add(dir_path + filename) + languages.add(lang_code) + projects.add(proj_code) + + if "{LANG}" in languages: + languages = set() + + if "{PROJ}" in projects: + projects = set() + + new_rules=",".join(new_rules) + + return languages, projects, new_rules + + +def set_projects_and_languages(app, schema): + VirtualFolder = app.get_model("virtualfolder.VirtualFolder") + Project = app.get_model("pootle_project.Project") + Language = app.get_model("pootle_language.Language") + for vf in VirtualFolder.objects.all(): + languages, projects, new_rules = parse_vfolder_rules(vf) + if projects: + vf.projects.add(*Project.objects.filter(code__in=projects)) + if languages: + vf.languages.add(*Language.objects.filter(code__in=languages)) + vf.filter_rules = new_rules + vf.all_projects = not projects + vf.all_languages = not languages + vf.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('virtualfolder', '0012_add_all_proj_lang_flags'), + ] + + operations = [ + migrations.RunPython(set_projects_and_languages) + ] From e870586e1be8ce5bcdd4ad773664eb585f631431 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 11 Sep 2016 06:37:38 +0100 Subject: [PATCH 11/19] Migration to remove location field --- .../migrations/0014_remove_location.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 pootle/apps/virtualfolder/migrations/0014_remove_location.py diff --git a/pootle/apps/virtualfolder/migrations/0014_remove_location.py b/pootle/apps/virtualfolder/migrations/0014_remove_location.py new file mode 100644 index 00000000000..13ab61ef809 --- /dev/null +++ b/pootle/apps/virtualfolder/migrations/0014_remove_location.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('virtualfolder', '0013_set_projects_languages'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='virtualfolder', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='virtualfolder', + name='location', + ), + ] From 24ff3000cfb67b149b18cb8a06ca6977ce9bdac7 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 4 Sep 2016 12:51:42 +0100 Subject: [PATCH 12/19] Add new virtualfolder signal handlers --- pootle/apps/virtualfolder/receivers.py | 99 +++++--------------------- 1 file changed, 16 insertions(+), 83 deletions(-) diff --git a/pootle/apps/virtualfolder/receivers.py b/pootle/apps/virtualfolder/receivers.py index 416182bfa6e..9f6326a135f 100644 --- a/pootle/apps/virtualfolder/receivers.py +++ b/pootle/apps/virtualfolder/receivers.py @@ -6,98 +6,31 @@ # or later license. See the LICENSE file for a copy of the license and the # AUTHORS file for copyright and authorship information. -from django.db.models.signals import post_save +from django.db.models.signals import post_save, pre_delete from django.dispatch import receiver from pootle_store.models import Store -from .models import VirtualFolder, VirtualFolderTreeItem +from .delegate import vfolder_finder +from .models import VirtualFolder -def update_vfolder_tree(vf, store): - """For a given VirtualFolder and Store update the VirtualFolderTreeItem - """ - # Create missing VirtualFolderTreeItem tree structure for affected Stores - # after adding or unobsoleting Units. - vfolder_treeitem, created = ( - VirtualFolderTreeItem.objects.get_or_create( - directory=store.parent, vfolder=vf)) - +@receiver(post_save, sender=Store) +def handle_store_save(sender, instance, created, **kwargs): if not created: - # The VirtualFolderTreeItem already existed, so - # calculate again the stats up to the root. - vfolder_treeitem.update_all_cache() - - -def add_store_to_vfolders(store): - """For a given Unit check for membership of any VirtualFolders - """ - pootle_path = store.pootle_path - - for vf in VirtualFolder.objects.iterator(): - store_added = False - for location in vf.all_locations: - if not pootle_path.startswith(location): - continue - - for filename in vf.filter_rules.split(","): - if pootle_path == "".join([location, filename]): - vf.stores.add(store) - store_added = True - break - - if store_added: - break - - if store_added: - update_vfolder_tree(vf, store) + return + vfolder_finder.get( + instance.__class__)(instance).add_to_vfolders() @receiver(post_save, sender=VirtualFolder) -def vfolder_save_handler(sender, instance, created, **kwargs): - """Remove Units from VirtualFolder when vfolder changes - - - Check the original VirtualFolder object's locations - - Check the new VirtualFolders object's locations - - Remove any units from the VirtualFolder that are only in original's - locations and reset Unit priority - - Update VirtualFolderTree for any Stores that may have been affected - """ - locations = set() - for location in instance.all_locations: - for filename in instance.filter_rules.split(","): - locations.add("".join([location, filename])) - - stores_we_want = Store.objects.none() - for location in locations: - stores_we_want = stores_we_want | Store.objects.filter( - pootle_path__startswith=location) - to_remove = list( - instance.stores.exclude( - pk__in=stores_we_want.values_list("pk", flat=True))) - to_add = list( - stores_we_want.exclude( - pk__in=instance.stores.values_list("pk", flat=True))) - instance.stores.remove(*to_remove) - instance.stores.add(*to_add) - for store in to_add: - update_vfolder_tree(instance, store) - for store in to_remove: - store.set_priority() - for store in stores_we_want: - store.set_priority() - - -@receiver(post_save, sender=Store) -def vfolder_store_postsave_handler(**kwargs): - """Match VirtualFolders to Unit and update Unit.priority +def handle_vfolder_save(sender, instance, created, **kwargs): + instance.path_matcher.update_stores() - - If unit was newly created, then check vfolders for membership - - Update Unit priority from vfolder membership on Unit.save or create - """ - instance = kwargs["instance"] - created = kwargs.get("created", False) - if created: - add_store_to_vfolders(instance) - instance.set_priority() +@receiver(pre_delete, sender=VirtualFolder) +def handle_vfolder_delete(sender, instance, **kwargs): + for store in instance.stores.all(): + instance.stores.remove(store) + if store.priority == instance.priority: + store.set_priority() From 0ad046b699f0d3a031b333a983a000dda2c2296b Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 19:53:12 +0100 Subject: [PATCH 13/19] Adjust vfolder tests to use new associations or remove if they are no longer relevant --- tests/models/virtualfolder.py | 131 +++++++++++++++------------------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/tests/models/virtualfolder.py b/tests/models/virtualfolder.py index 92a0077272d..5ad4328bdb9 100644 --- a/tests/models/virtualfolder.py +++ b/tests/models/virtualfolder.py @@ -14,12 +14,13 @@ from pytest_pootle.factories import VirtualFolderDBFactory +from pootle_language.models import Language from pootle_store.models import Store from virtualfolder.models import VirtualFolder, VirtualFolderTreeItem @pytest.mark.django_db -def test_vfolder_priority_not_greater_than_zero(): +def test_vfolder_priority_not_greater_than_zero(tp0): """Tests that the creation of a virtual folder fails if the provided priority is not greater than zero. """ @@ -27,83 +28,27 @@ def test_vfolder_priority_not_greater_than_zero(): # Test priority less than zero. vfolder_item = { 'name': "whatever", - 'location': "/af/vfolder_test/", 'priority': -3, 'is_public': True, 'filter_rules': "browser/defines.po", } - vfolder = VirtualFolder(**vfolder_item) with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() + VirtualFolder.objects.create(**vfolder_item) assert u'Priority must be greater than zero.' in str(excinfo.value) # Test zero priority. - vfolder_item['priority'] = 0 - vfolder = VirtualFolder(**vfolder_item) + vfolder_item['priority'] = 1 + vfolder = VirtualFolder.objects.create(**vfolder_item) + vfolder.priority = 0 with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() + vfolder.save() assert u'Priority must be greater than zero.' in str(excinfo.value) -@pytest.mark.django_db -def test_vfolder_root_location(): - """Tests that the creation of a virtual folder fails if it uses location / - instead of /{LANG}/{PROJ}/. - """ - - vfolder_item = { - 'name': "whatever", - 'location': "/", - 'priority': 4, - 'is_public': True, - 'filter_rules': "browser/defines.po", - } - vfolder = VirtualFolder(**vfolder_item) - - with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() - - assert (u'The "/" location is not allowed. Use "/{LANG}/{PROJ}/" instead.' - in str(excinfo.value)) - - -@pytest.mark.django_db -def test_vfolder_location_starts_with_projects(): - """Tests that the creation of a virtual folder fails if it uses a location - that starts with /projects/. - """ - - # Test just /projects/ location. - vfolder_item = { - 'name': "whatever", - 'location': "/projects/", - 'priority': 4, - 'is_public': True, - 'filter_rules': "browser/defines.po", - } - vfolder = VirtualFolder(**vfolder_item) - - with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() - - assert (u'Locations starting with "/projects/" are not allowed. Use ' - u'"/{LANG}/" instead.') in str(excinfo.value) - - # Test /projects/tutorial/ location. - vfolder_item['location'] = "/projects/tutorial/" - vfolder = VirtualFolder(**vfolder_item) - - with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() - - assert (u'Locations starting with "/projects/" are not allowed. Use ' - u'"/{LANG}/" instead.') in str(excinfo.value) - - @pytest.mark.django_db def test_vfolder_with_no_filter_rules(): """Tests that the creation of a virtual folder fails if it doesn't have any @@ -112,16 +57,19 @@ def test_vfolder_with_no_filter_rules(): vfolder_item = { 'name': "whatever", - 'location': "/af/vfolder_test/", 'priority': 4, 'is_public': True, 'filter_rules': "", } - vfolder = VirtualFolder(**vfolder_item) - with pytest.raises(ValidationError) as excinfo: - vfolder.clean_fields() + VirtualFolder.objects.create(**vfolder_item) + assert u'Some filtering rule must be specified.' in str(excinfo.value) + vfolder_item["filter_rules"] = "FOO" + vf = VirtualFolder.objects.create(**vfolder_item) + vf.filter_rules = "" + with pytest.raises(ValidationError) as excinfo: + vf.save() assert u'Some filtering rule must be specified.' in str(excinfo.value) @@ -133,16 +81,20 @@ def test_vfolder_membership(tp0, store0): vf0 = VirtualFolder.objects.create( name="vf0", title="the vf0", - location=tp0.pootle_path, filter_rules=store0.name) + vf0.projects.add(tp0.project) + vf0.languages.add(tp0.language) + vf0.save() assert vf0.stores.count() == 1 assert vf0.stores.first() == store0 vf1 = VirtualFolder.objects.create( name="vf1", title="the vf1", - location=tp0.pootle_path, filter_rules=tp0_stores) + vf1.projects.add(tp0.project) + vf1.languages.add(tp0.language) + vf1.save() assert ( list(vf1.stores.order_by("pk")) == list(tp0.stores.order_by("pk"))) @@ -156,13 +108,12 @@ def test_vfolder_membership(tp0, store0): assert store in vf1.stores.all() +@pytest.mark.pootle_vfolders @pytest.mark.django_db -def test_vfolder_store_priorities(): +def test_vfolder_store_priorities(project0): # remove the default vfolders and update units to reset priorities VirtualFolder.objects.all().delete() - [store.save() for store in Store.objects.all()] - assert all( priority == 1 for priority @@ -198,8 +149,9 @@ def test_vfolder_store_priorities(): .values_list("priority", flat=True)) vfolder1 = VirtualFolderDBFactory( - location='/{LANG}/project0/', filter_rules="store1.po") + vfolder1.languages.add(*Language.objects.all()) + vfolder1.projects.add(project0) vfolder1.priority = 4 vfolder1.save() vfolder1_stores = vfolder1.stores.values_list("pk", flat=True) @@ -229,7 +181,7 @@ def test_vfolder_store_priorities(): def test_virtualfolder_repr(): vf = VirtualFolderDBFactory(filter_rules="store0.po") assert ( - "" % (vf.name, vf.location) + "" % (vf.name) == repr(vf)) @@ -241,6 +193,7 @@ def test_virtualfoldertreeitem_repr(): == repr(vfti)) +@pytest.mark.pootle_vfolders @pytest.mark.django_db def test_vfti_rm(): original_vftis = VirtualFolderTreeItem.objects.values_list("pk", flat=True) @@ -261,12 +214,40 @@ def test_vfti_rm(): assert not VirtualFolderTreeItem.objects.exists() +@pytest.mark.pootle_vfolders @pytest.mark.django_db def test_vfolder_calc_priority(settings, store0): - vf = VirtualFolderDBFactory( - filter_rules=store0.name) + vf = store0.vfolders.first() vf.priority = 5 vf.save() assert store0.calculate_priority() == 5.0 settings.INSTALLED_APPS.remove("virtualfolder") assert store0.calculate_priority() == 1.0 + settings.INSTALLED_APPS.append("virtualfolder") + + +@pytest.mark.pootle_vfolder +@pytest.mark.django_db +def test_vfolder_membership_new_store(tp0): + vf0 = VirtualFolder.objects.create( + name="vf0", + title="the vf0", + priority=7.0, + all_languages=True, + all_projects=True, + filter_rules="wierd.po") + wierd_store = Store.objects.create( + parent=tp0.directory, + translation_project=tp0, + name="wierd.po") + wierd_store.set_priority() + assert wierd_store in vf0.stores.all() + assert Store.objects.get(pk=wierd_store.pk).priority == 7 + normal_store = Store.objects.create( + parent=tp0.directory, + translation_project=tp0, + name="normal.po") + assert normal_store not in vf0.stores.all() + assert Store.objects.get(pk=normal_store.pk).priority == 1.0 + vf0.delete() + assert Store.objects.get(pk=wierd_store.pk).priority == 1.0 From 70b43f22eda08d50aff54dcf7d2f75736f7ab6be Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 19:53:57 +0100 Subject: [PATCH 14/19] Update env fixtures with new vfolder associations --- pytest_pootle/env.py | 29 ++++++++++++++++------------- pytest_pootle/factories.py | 2 -- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pytest_pootle/env.py b/pytest_pootle/env.py index 356869f4048..e1fc3f543d0 100644 --- a/pytest_pootle/env.py +++ b/pytest_pootle/env.py @@ -473,6 +473,8 @@ def setup_vfolders(self): from django.apps import apps from pootle.core.utils.db import set_mysql_collation_for_column + from pootle_language.models import Language + from pootle_project.models import Project cursor = connection.cursor() @@ -493,24 +495,25 @@ def setup_vfolders(self): "name", "utf8_bin", "varchar(70)") - set_mysql_collation_for_column( - apps, - cursor, - "virtualfolder.VirtualFolder", - "location", - "utf8_bin", - "varchar(255)") + project0 = Project.objects.get(code="project0") + language0 = Language.objects.get(code="language0") VirtualFolderDBFactory(filter_rules="store0.po") VirtualFolderDBFactory(filter_rules="store1.po") - VirtualFolderDBFactory( - location='/{LANG}/project0/', + vf = VirtualFolderDBFactory( + all_languages=True, is_public=False, filter_rules="store0.po") - VirtualFolderDBFactory( - location='/{LANG}/project0/', + vf.projects.add(project0) + vf.save() + vf = VirtualFolderDBFactory( + all_languages=True, is_public=False, filter_rules="store1.po") - VirtualFolderDBFactory( - location='/language0/project0/', + vf.projects.add(project0) + vf.save() + vf = VirtualFolderDBFactory( filter_rules="subdir0/store4.po") + vf.languages.add(language0) + vf.projects.add(project0) + vf.save() diff --git a/pytest_pootle/factories.py b/pytest_pootle/factories.py index d827abfc849..10bb6db3275 100644 --- a/pytest_pootle/factories.py +++ b/pytest_pootle/factories.py @@ -208,11 +208,9 @@ class VirtualFolderDBFactory(factory.django.DjangoModelFactory): class Meta(object): model = 'virtualfolder.VirtualFolder' - django_get_or_create = ("location", "is_public", "filter_rules") priority = 2 is_public = True - location = "/{LANG}/{PROJ}/" @factory.lazy_attribute def name(self): From 9a1fb536c6b0e12111a4b79be317d45fb03f3a6f Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 10 Sep 2016 19:56:41 +0100 Subject: [PATCH 15/19] Adjust vftreeitem creation algorithm to work with new vfolder assocs --- pootle/apps/virtualfolder/models.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pootle/apps/virtualfolder/models.py b/pootle/apps/virtualfolder/models.py index 138e375974d..43dbde88296 100644 --- a/pootle/apps/virtualfolder/models.py +++ b/pootle/apps/virtualfolder/models.py @@ -194,19 +194,19 @@ def __unicode__(self): return self.pootle_path def save(self, *args, **kwargs): - self.pootle_path = self.vfolder.get_adjusted_pootle_path( - self.directory.pootle_path - ) + parts = split_pootle_path(self.directory.pootle_path) + path_parts = ["", parts[0], parts[1], self.vfolder.name] + if parts[2]: + path_parts.append(parts[2].strip("/")) + path_parts.append(parts[3]) + self.pootle_path = "/".join(path_parts) # Trigger the creation of the whole parent tree up to the vfolder - # adjusted location. - if (self.directory.pootle_path.count('/') > - self.vfolder.location.count('/')): - parent = VirtualFolderTreeItem.objects.get_or_create( + # tp + if self.directory.pootle_path.count('/') > 3: + self.parent = VirtualFolderTreeItem.objects.get_or_create( directory=self.directory.parent, - vfolder=self.vfolder, - )[0] - self.parent = parent + vfolder=self.vfolder)[0] super(VirtualFolderTreeItem, self).save(*args, **kwargs) @@ -269,6 +269,6 @@ def all_pootle_paths(self): don't want to mess with regular CachedTreeItem stats. """ return [p for p in get_all_pootle_paths(self.get_cachekey()) - if p.count('/') > self.vfolder.location.count('/')] + if p.count('/') > 3] # # # /TreeItem From 7a6cfa35aaf866866c46946bab1c35f102167531 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 14 Sep 2016 21:47:11 +0100 Subject: [PATCH 16/19] Adjust pootle_store get_all_pootle_paths method to not use location --- pootle/apps/pootle_store/models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pootle/apps/pootle_store/models.py b/pootle/apps/pootle_store/models.py index 8ac1296d12b..d2e56ed7581 100644 --- a/pootle/apps/pootle_store/models.py +++ b/pootle/apps/pootle_store/models.py @@ -1624,13 +1624,12 @@ def all_pootle_paths(self): """ pootle_paths = super(Store, self).all_pootle_paths() if 'virtualfolder' in settings.INSTALLED_APPS: - vftis = self.parent_vf_treeitems.values_list( - "vfolder__location", "pootle_path") - for location, pootle_path in vftis: + vftis = self.parent_vf_treeitems + for pootle_path in vftis.values_list("pootle_path", flat=True): pootle_paths.extend( [p for p in get_all_pootle_paths(pootle_path) - if p.count('/') > location.count('/')]) + if p.count('/') > 3]) return pootle_paths # # # /TreeItem From 175b7d4cb4b226b3e237625dc10cc4598e9f6bf9 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 15 Sep 2016 11:42:36 +0100 Subject: [PATCH 17/19] Add tests for vfolder store matching --- tests/vfolders/path_matcher.py | 198 +++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 tests/vfolders/path_matcher.py diff --git a/tests/vfolders/path_matcher.py b/tests/vfolders/path_matcher.py new file mode 100644 index 00000000000..fd71edaae2c --- /dev/null +++ b/tests/vfolders/path_matcher.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +from collections import OrderedDict + +import pytest + +from pootle_fs.utils import PathFilter +from pootle_store.models import Store +from virtualfolder.models import VirtualFolder +from virtualfolder.utils import VirtualFolderPathMatcher + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher(): + return + vfolder = VirtualFolder.objects.create( + name="avfolder", + filter_rules="FOO,BAR") + path_matcher = VirtualFolderPathMatcher(vfolder) + assert path_matcher.vf == vfolder + assert path_matcher.tp_path == "/[^/]*/[^/]*/" + assert isinstance( + vfolder.path_matcher, + VirtualFolderPathMatcher) + # no projects/languages set by default + assert ( + list(vfolder.path_matcher.languages) + == list(vfolder.path_matcher.projects) + == list(vfolder.path_matcher.existing_stores) + == []) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_languages(language0, language1): + vfolder = VirtualFolder.objects.create( + name="avfolder", + filter_rules="FOO,BAR") + path_matcher = VirtualFolderPathMatcher(vfolder) + vfolder.languages.add(language0) + vfolder.languages.add(language1) + # no projects set so no stores + assert list(vfolder.path_matcher.existing_stores) == [] + assert ( + sorted(vfolder.path_matcher.languages) + == [language0.pk, language1.pk]) + vfolder.all_languages = True + vfolder.save() + assert vfolder.path_matcher.languages is None + assert list(vfolder.path_matcher.projects) == [] + # no projects set so no stores + assert list(vfolder.path_matcher.existing_stores) == [] + vfolder.all_languages = False + vfolder.save() + # didnt forget lang preferences + vfolder.all_languages = False + vfolder.save() + assert ( + sorted(vfolder.path_matcher.languages.values_list("code", flat=True)) + == ["language0", "language1"]) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_projects(project0, project1): + vfolder = VirtualFolder.objects.create( + name="avfolder", + filter_rules="FOO,BAR") + path_matcher = VirtualFolderPathMatcher(vfolder) + vfolder.projects.add(project0) + vfolder.projects.add(project1) + # no languages set so no stores + assert list(vfolder.path_matcher.existing_stores) == [] + assert ( + sorted(vfolder.path_matcher.projects.values_list("code", flat=True)) + == ["project0", "project1"]) + vfolder.all_projects = True + vfolder.save() + assert vfolder.path_matcher.projects is None + assert list(vfolder.path_matcher.languages) == [] + # no languages set so no stores + assert list(vfolder.path_matcher.existing_stores) == [] + vfolder.all_projects = False + vfolder.save() + # didnt forget lang preferences + vfolder.all_projects = False + vfolder.save() + assert ( + sorted(vfolder.path_matcher.projects.values_list("code", flat=True)) + == ["project0", "project1"]) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_all_proj_lang(): + vfolder = VirtualFolder.objects.create( + name="avfolder", + all_projects=True, + all_languages=True, + filter_rules="*") + return + assert ( + list(vfolder.path_matcher.existing_stores.order_by("pk")) + == list(Store.objects.order_by("pk"))) + vfolder.all_languages = False + vfolder.save() + assert ( + list(vfolder.path_matcher.existing_stores.order_by("pk")) + == []) + vfolder.all_languages = True + vfolder.all_languages = False + vfolder.save() + assert ( + list(vfolder.path_matcher.existing_stores.order_by("pk")) + == []) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_some_proj_lang(tp0, language1): + vfolder = VirtualFolder.objects.create( + name="avfolder", + filter_rules="*") + vfolder.languages.add(tp0.language) + vfolder.projects.add(tp0.project) + # m2m handler? + vfolder.save() + assert ( + list(vfolder.path_matcher.existing_stores.order_by("pk")) + == list(tp0.stores.order_by("pk"))) + vfolder.languages.add(language1) + vfolder.save() + assert ( + list(vfolder.path_matcher.existing_stores.order_by("pk")) + == list( + Store.objects.filter( + translation_project__project=tp0.project, + translation_project__language__in=[ + tp0.language, + language1]).order_by("pk"))) + +VF_RULE_TESTS = [ + "*", + "store.po", + "*/subdir0/*"] + + +@pytest.fixture(params=VF_RULE_TESTS) +def vf_rules(request): + return request.param + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_get_rule_regex(vfolder0, vf_rules): + rule = vf_rules + assert ( + vfolder0.path_matcher.get_rule_regex(rule) + == ("^%s%s" + % (vfolder0.path_matcher.tp_path, + PathFilter().path_regex(rule)))) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_rules(vfolder0): + vfolder0.filter_rules = "foo,bar" + assert ( + list(vfolder0.path_matcher.rules) + == ["foo", "bar"]) + + # convert to json field? for now be ws tolerant + vfolder0.filter_rules = "foo, bar" + assert ( + list(vfolder0.path_matcher.rules) + == ["foo", "bar"]) + + +@pytest.mark.pootle_vfolders +@pytest.mark.django_db +def test_vfolder_path_matcher_paths_regex(vfolder0): + vfolder0.filter_rules = "foo,bar" + assert ( + list(vfolder0.path_matcher.rules) + == ["foo", "bar"]) + + # convert to json field? for now be ws tolerant + vfolder0.filter_rules = "foo, bar" + assert ( + list(vfolder0.path_matcher.rules) + == ["foo", "bar"]) From 52d50961250df47adb530f1913e54b5f3d0eb7c1 Mon Sep 17 00:00:00 2001 From: Leandro Regueiro Date: Thu, 15 Sep 2016 13:27:05 +0200 Subject: [PATCH 18/19] Squash: Append * to new rules if it is a directory path --- .../migrations/0013_set_projects_languages.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py b/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py index 0109aadcc4c..626e3ee442b 100644 --- a/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py +++ b/pootle/apps/virtualfolder/migrations/0013_set_projects_languages.py @@ -11,11 +11,15 @@ def parse_vfolder_rules(vf): projects = set() new_rules = set() - full_rules = [vf.location + rule for rule in vf.filter_rules.split(",")] + full_rules = [vf.location.strip() + rule.strip() + for rule in vf.filter_rules.split(",")] for full_rule in full_rules: lang_code, proj_code, dir_path, filename = split_pootle_path(full_rule) - new_rules.add(dir_path + filename) + if filename: + new_rules.add(dir_path + filename) + else: + new_rules.add(dir_path + "*") languages.add(lang_code) projects.add(proj_code) From 1662c52f5ef1d8281ce8dce34eda5467b412ba97 Mon Sep 17 00:00:00 2001 From: Leandro Regueiro Date: Thu, 15 Sep 2016 14:04:22 +0200 Subject: [PATCH 19/19] Adjust add_vfolders command to new reality --- .../management/commands/add_vfolders.py | 74 +++++++++++++++++-- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/pootle/apps/virtualfolder/management/commands/add_vfolders.py b/pootle/apps/virtualfolder/management/commands/add_vfolders.py index b957753bf87..138875079b3 100644 --- a/pootle/apps/virtualfolder/management/commands/add_vfolders.py +++ b/pootle/apps/virtualfolder/management/commands/add_vfolders.py @@ -16,6 +16,7 @@ from django.core.exceptions import ValidationError from django.core.management.base import BaseCommand, CommandError +from pootle.core.url_helpers import split_pootle_path from virtualfolder.models import VirtualFolder @@ -29,6 +30,33 @@ def add_arguments(self, parser): help="JSON vfolder configuration file", ) + def parse_vfolder_rules(self, location, old_rules): + """Extract languages, projects and new rules from location and rules.""" + languages = set() + projects = set() + new_rules = set() + + full_rules = [location + old_rule.strip() for old_rule in old_rules] + + for full_rule in full_rules: + lang_code, proj_code, dir_path, fname = split_pootle_path(full_rule) + if fname: + new_rules.add(dir_path + fname) + else: + new_rules.add(dir_path + "*") + languages.add(lang_code) + projects.add(proj_code) + + if "{LANG}" in languages: + languages = set() + + if "{PROJ}" in projects: + projects = set() + + new_rules=",".join(new_rules) + + return languages, projects, new_rules + def handle(self, **options): """Add virtual folders from file.""" @@ -57,12 +85,16 @@ def handle(self, **options): errored_count = 0 for vfolder_item in vfolders: - vfolder_item['name'] = vfolder_item['name'].lower() + vfolder_item['name'] = vfolder_item['name'].strip().lower() # Put all the files for each virtual folder as a list and save it # as its filter rules. - vfolder_item['filter_rules'] = ','.join( - vfolder_item['filters']['files']) + languages, projects, new_rules = self.parse_vfolder_rules( + vfolder_item['location'].strip(), + vfolder_item['filters']['files'] + ) + + vfolder_item['filter_rules'] = new_rules if 'filters' in vfolder_item: del vfolder_item['filters'] @@ -70,15 +102,14 @@ def handle(self, **options): # Now create or update the virtual folder. try: # Retrieve the virtual folder if it exists. - vfolder = VirtualFolder.objects.get( - name=vfolder_item['name'], - location=vfolder_item['location'], - ) + vfolder = VirtualFolder.objects.get(name=vfolder_item['name']) except VirtualFolder.DoesNotExist: # If the virtual folder doesn't exist yet then create it. try: self.stdout.write(u'Adding new virtual folder %s...' % vfolder_item['name']) + vfolder_item['all_projects'] = not projects + vfolder_item['all_languages'] = not languages vfolder = VirtualFolder(**vfolder_item) vfolder.save() except ValidationError as e: @@ -86,12 +117,41 @@ def handle(self, **options): self.stdout.write('FAILED') self.stderr.write(e) else: + if projects: + vfolder.projects.add( + *Project.objects.filter(code__in=projects) + ) + if languages: + vfolder.languages.add( + *Language.objects.filter(code__in=languages) + ) self.stdout.write('DONE') added_count += 1 else: # Update the already existing virtual folder. changed = False + if vfolder.all_projects != not projects: + vfolder.all_projects = not projects + changed = True + logging.debug("'All projects' for virtual folder '%s' " + "will be changed.", vfolder.name) + + if vfolder.all_languages != not languages: + vfolder.all_languages = not languages + changed = True + logging.debug("'All languages' for virtual folder '%s' " + "will be changed.", vfolder.name) + + if projects: + vfolder.projects.set( + *Project.objects.filter(code__in=projects) + ) + if languages: + vfolder.languages.set( + *Language.objects.filter(code__in=languages) + ) + if vfolder.filter_rules != vfolder_item['filter_rules']: vfolder.filter_rules = vfolder_item['filter_rules'] changed = True