Custom Logic Feature Branch

entity/exceptions.py

@@ -0,0 +1,5 @@
+
+
+class InvalidLogicStringException(Exception):
+    def __str__(self):
+        return 'Invalid logic string'

entity/migrations/0002_entitygroup_logic_string.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.4 on 2023-08-30 18:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entity', '0001_0010_squashed'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entitygroup',
+            name='logic_string',
+            field=models.TextField(blank=True, default=None, null=True),
+        ),
+        migrations.AddField(
+            model_name='entitygroupmembership',
+            name='sort_order',
+            field=models.IntegerField(default=0),
+        ),
+    ]

entity/models.py

@@ -1,14 +1,18 @@
-from itertools import compress
+import ast
+from itertools import compress, chain
 
 from activatable_model.models import BaseActivatableModel, ActivatableManager, ActivatableQuerySet
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.db.models import Count, Q, JSONField
 from python3_utils import compare_on_attr
 from functools import reduce
 
+from entity.exceptions import InvalidLogicStringException
+
 
 class AllEntityKindManager(ActivatableManager):
     """
@@ -334,6 +338,8 @@ def get_membership_cache(self, group_ids=None, is_active=True):
         if group_ids:
             membership_queryset = membership_queryset.filter(entity_group_id__in=group_ids)
 
+        membership_queryset = membership_queryset.order_by('sort_order', 'id')
+
         membership_queryset = membership_queryset.values_list('entity_group_id', 'entity_id', 'sub_entity_kind_id')
 
         # Iterate over the query results and build the cache dict
@@ -363,6 +369,8 @@ class EntityGroup(models.Model):
 
     objects = EntityGroupManager()
 
+    logic_string = models.TextField(default=None, null=True, blank=True)
+
     def all_entities(self, is_active=True):
         """
         Return all the entities in the group.
@@ -373,6 +381,96 @@ def all_entities(self, is_active=True):
         """
         return self.get_all_entities(return_models=True, is_active=is_active)
 
+    def get_filter_indices(self, node):
+        """
+        Makes sure that each filter referenced actually exists
+        """
+        if hasattr(node, 'op'):
+            # multi-operand operators
+            if hasattr(node, 'values'):
+                return list(chain(*[self.get_filter_indices(value) for value in node.values]))
+            # unary operators
+            elif hasattr(node, 'operand'):
+                return list(chain(*[self.get_filter_indices(node.operand)]))
+        elif hasattr(node, 'n'):
+            return [node.n]
+        return None
+
+    def validate_filter_indices(self, indices, memberships):
+        """
+        Raises an error if an invalid filter index is referenced or if an index is not referenced
+        """
+        for index in indices:
+            if hasattr(index, '__iter__'):
+                return self.validate_filter_indices(index, memberships)
+            if index < 1 or index > len(memberships):
+                raise ValidationError('Filter logic contains an invalid filter index ({0})'.format(index))
+
+        for i in range(1, len(memberships) + 1):
+            if i not in indices:
+                raise ValidationError('Filter logic is missing a filter index ({0})'.format(i))
+
+        return True
+
+    def _node_to_kmatch(self, node):
+        """
+        Looks at an ast node and either returns the value or recursively returns the kmatch syntax. This is meant
+        to convert the boolean logic like "1 AND 2" to kmatch syntax like ['&', [1, 2]]
+        :return: kmatch syntax where memberships are represented by numbers
+        :rtype: list
+        """
+        if hasattr(node, 'op'):
+            if hasattr(node, 'values'):
+                return [node.op, [self._node_to_kmatch(value) for value in node.values]]
+            elif hasattr(node, 'operand'):
+                return [node.op, self._node_to_kmatch(node.operand)]
+        elif hasattr(node, 'n'):
+            return node.n
+        return None
+
+    def _map_kmatch_values(self, kmatch, memberships):
+        """
+        Replaces index placeholders in the kmatch with the actual memberships. Any memberships that could not be matched
+        up with a field will be replaced with None
+        :return: the complete kmatch pattern
+        :rtype: list
+        """
+        # Check if single item
+        if isinstance(kmatch, int):
+            return memberships[kmatch - 1]
+        if hasattr(kmatch, '__iter__'):
+            return [self._map_kmatch_values(value, memberships) for value in kmatch]
+
+        cls = getattr(kmatch, '__class__')
+        if cls == ast.And:
+            return '&'
+        elif cls == ast.Or:
+            return '|'
+        elif cls == ast.Not:
+            return '!'
+
+    def _process_kmatch(self, kmatch, full_set):
+        """
+        Every item is 2 elements - the operator and the value or list of values
+        """
+        entity_ids = set()
+        operators = {'&', '|', '!'}
+
+        if isinstance(kmatch, set):
+            return kmatch
+
+        if len(kmatch) == 2 and kmatch[0] not in operators:
+            return kmatch
+
+        if kmatch[0] == '&':
+            entity_ids = self._process_kmatch(kmatch[1][0], full_set) & self._process_kmatch(kmatch[1][1], full_set)
+        elif kmatch[0] == '|':
+            entity_ids = self._process_kmatch(kmatch[1][0], full_set) | self._process_kmatch(kmatch[1][1], full_set)
+        elif kmatch[0] == '!':
+            entity_ids = full_set - self._process_kmatch(kmatch[1], full_set)
+
+        return entity_ids
+
     def get_all_entities(self, membership_cache=None, entities_by_kind=None, return_models=False, is_active=True):
         """
         Returns a list of all entity ids in this group or optionally returns a queryset for all entity models.
@@ -401,27 +499,60 @@ def get_all_entities(self, membership_cache=None, entities_by_kind=None, return_
         entity_ids = set()
 
         # This group does have entities
-        if membership_cache.get(self.id):
-
-            # Loop over each membership in this group
-            for entity_id, entity_kind_id in membership_cache[self.id]:
-                if entity_id:
-                    if entity_kind_id:
-                        # All sub entities of this kind under this entity
-                        entity_ids.update(entities_by_kind[entity_kind_id][entity_id])
+        memberships = membership_cache.get(self.id)
+        if memberships:
+            if self.logic_string:
+                entity_ids = self.get_entity_ids_from_logic_string(entities_by_kind, memberships)
+            else:
+                # Loop over each membership in this group
+                for entity_id, entity_kind_id in membership_cache[self.id]:
+                    if entity_id:
+                        if entity_kind_id:
+                            # All sub entities of this kind under this entity
+                            entity_ids.update(entities_by_kind[entity_kind_id][entity_id])
+                        else:
+                            # Individual entity
+                            entity_ids.add(entity_id)
                     else:
-                        # Individual entity
-                        entity_ids.add(entity_id)
-                else:
-                    # All entities of this kind
-                    entity_ids.update(entities_by_kind[entity_kind_id]['all'])
+                        # All entities of this kind
+                        entity_ids.update(entities_by_kind[entity_kind_id]['all'])
 
         # Check if a queryset needs to be returned
         if return_models:
             return Entity.objects.filter(id__in=entity_ids)
 
         return entity_ids
 
+    def get_entity_ids_from_logic_string(self, entities_by_kind, memberships):
+        entity_kind_id = memberships[0][1]
+        full_set = set(entities_by_kind[entity_kind_id]['all'])
+        try:
+            filter_tree = ast.parse(self.logic_string.lower())
+        except:
+            raise InvalidLogicStringException()
+
+        expanded_memberships = []
+        for entity_id, entity_kind_id in memberships:
+            if entity_id:
+                if entity_kind_id:
+                    # All sub entities of this kind under this entity
+                    expanded_memberships.append(set(entities_by_kind[entity_kind_id][entity_id]))
+                else:
+                    # Individual entity
+                    expanded_memberships.append({entity_id})
+            else:
+                # All entities of this kind
+                expanded_memberships.append(set(entities_by_kind[entity_kind_id]['all']))
+
+        # Make sure each index is valid
+        indices = self.get_filter_indices(filter_tree.body[0].value)
+        self.validate_filter_indices(indices, expanded_memberships)
+        kmatch = self._node_to_kmatch(filter_tree.body[0].value)
+        kmatch = self._map_kmatch_values(kmatch, expanded_memberships)
+        entity_ids = self._process_kmatch(kmatch, full_set=full_set)
+
+        return entity_ids
+
     def add_entity(self, entity, sub_entity_kind=None):
         """
         Add an entity, or sub-entity group to this EntityGroup.
@@ -543,6 +674,7 @@ class EntityGroupMembership(models.Model):
     entity_group = models.ForeignKey(EntityGroup, on_delete=models.CASCADE)
     entity = models.ForeignKey(Entity, null=True, on_delete=models.CASCADE)
     sub_entity_kind = models.ForeignKey(EntityKind, null=True, on_delete=models.CASCADE)
+    sort_order = models.IntegerField(default=0)
 
 
 def get_entities_by_kind(membership_cache=None, is_active=True):
@@ -569,6 +701,12 @@ def get_entities_by_kind(membership_cache=None, is_active=True):
     kinds_with_supers = set()
     super_ids = set()
 
+    # Determine if we need to include the "universal set" aka all for a kind based on the presence of a logic_string
+    group_ids_with_logic_string = set(EntityGroup.objects.filter(
+        id__in=membership_cache.keys(),
+        logic_string__isnull=False,
+    ).values_list('id', flat=True))
+
     # Loop over each group
     for group_id, memberships in membership_cache.items():
 
@@ -581,6 +719,11 @@ def get_entities_by_kind(membership_cache=None, is_active=True):
                 # Make sure a dict exists for this kind
                 entities_by_kind.setdefault(entity_kind_id, {})
 
+                # Always include all if there is a logic string
+                if group_id in group_ids_with_logic_string:
+                    entities_by_kind[entity_kind_id]['all'] = []
+                    kinds_with_all.add(entity_kind_id)
+
                 # Check if this is all entities of a kind under a specific entity
                 if entity_id:
                     entities_by_kind[entity_kind_id][entity_id] = []