Explorer revamp

backend/lib/processor.py

@@ -400,7 +400,7 @@ def add_field_to_parent(self, field_name, new_data, which_parent=source_dataset,
 
 		TODO: could be improved by accepting different types of data depending on csv or ndjson.
 
-		:param str field_name: 	name of the desired
+		:param str field_name: 	Name of the desired new field
 		:param List new_data: 	List of data to be added to parent dataset
 		:param DataSet which_parent: 	DataSet to be updated (e.g., self.source_dataset, self.dataset.get_parent(), self.dataset.top_parent())
 		:param bool update_existing: 	False (default) will raise an error if the field_name already exists
@@ -418,7 +418,7 @@ def add_field_to_parent(self, field_name, new_data, which_parent=source_dataset,
 		parent_path = which_parent.get_results_path()
 
 		if len(new_data) != which_parent.num_rows:
-			raise ProcessorException('Must have new data point for each record: parent dataset: %i, new data points: %i' % (which_parent.num_rows, len(new_data)))
+			self.dataset.update_status('The amount of new data points and existing records don\'t match; data may be misaligned (parent dataset: %i, new data points: %i)' % (which_parent.num_rows, len(new_data)))
 
 		self.dataset.update_status("Adding new field %s to the source file" % field_name)
 

common/lib/config_definition.py

@@ -102,8 +102,8 @@
     "privileges.can_use_explorer": {
         "type": UserInput.OPTION_TOGGLE,
         "default": True,
-        "help": "Can use explorer",
-        "tooltip": "Controls whether users can use the Explorer feature to navigate datasets."
+        "help": "Can use Explorer",
+        "tooltip": "Controls whether users can use the Explorer feature to analyse and annotate datasets."
     },
     "privileges.can_export_datasets": {
         "type": UserInput.OPTION_TOGGLE,
@@ -305,22 +305,50 @@
         "global": True
     },
     # Explorer settings
-    # The maximum allowed amount of rows (prevents timeouts and memory errors)
-    "explorer.max_posts": {
+    "explorer.__basic-explanation": {
+        "type": UserInput.OPTION_INFO,
+        "help": "4CAT's Explorer feature lets you navigate and annotate datasets as if they "
+                "appared on their original platform. This is intended to facilitate qualitative "
+                "exploration and manual coding."
+    },
+    "explorer.__max_posts": {
         "type": UserInput.OPTION_TEXT,
         "default": 100000,
         "help": "Amount of posts",
         "coerce_type": int,
-        "tooltip": "Amount of posts to show in Explorer. The maximum allowed amount of rows (prevents timeouts and "
+        "tooltip": "Maximum number of posts to be considered by the Explorer (prevents timeouts and "
                    "memory errors)"
     },
-    "explorer.posts_per_page": {
+    "explorer.__posts_per_page": {
         "type": UserInput.OPTION_TEXT,
         "default": 50,
         "help": "Posts per page",
         "coerce_type": int,
-        "tooltip": "Posts to display per page"
+        "tooltip": "Number of posts to display per page"
     },
+    "explorer._config_explanation": {
+        "type": UserInput.OPTION_INFO,
+        "help": "Per data source, you can enable or disable the Explorer. Posts will be formatted through a <em>generic</em> template "
+                "made of [this HTML file](https://github.com/digitalmethodsinitiative/4cat/tree/master/webtool/templates/explorer/"
+                "templates/generic.html) and [this CSS file](https://github.com/digitalmethodsinitiative/4cat/tree/master/webtool/"
+                "static/css/explorer/generic.css). For various data sources, <em>data source-specific</em> templates are also available. "
+                "These are made of a custom HTML template in [this directory](https://github.com/digitalmethodsinitiative/4cat/tree/master/"
+                "webtool/datasource-templates/explorer/templates) and a custom CSS file [in this directory](https://github.com/digitalmethodsinitiative/4cat/tree/master/webtool/static/css/explorer)."
+    },
+    "explorer.config": {
+        "type": UserInput.OPTION_DATASOURCES_TABLE,
+        "help": "Explorer settings per data source",
+        "default": {"fourchan": {"enabled": True}, "eightchan": {"enabled": True}, "eightkun": {"enabled": True}, "ninegag": {"enabled": True}, "bitchute": {"enabled": True}, "dmi-tcat": {"enabled": True}, "dmi-tcatv2": {"enabled": True}, "douban": {"enabled": True}, "douyin": {"enabled": False}, "imgur": {"enabled": True}, "upload": {"enabled": True}, "instagram": {"enabled": True}, "linkedin": {"enabled": True}, "parler": {"enabled": True}, "reddit": {"enabled": True}, "telegram": {"enabled": True}, "tiktok": {"enabled": True}, "tiktok-urls": {"enabled": True}, "tumblr": {"enabled": True}, "twitter": {"enabled": True}, "twitterv2": {"enabled": True}, "usenet": {"enabled": True}, "vk": {"enabled": True}},
+        "columns": {
+            "enabled": {
+                "type": UserInput.OPTION_TOGGLE,
+                "help": "Enable Explorer",
+                "tooltip": "Whether the Explorer is available for this data source",
+                "default": True
+            }
+        }
+    },
+    "explorer"
     # Web tool settings
     # These are used by the FlaskConfig class in config.py
     # Flask may require a restart to update them
@@ -515,7 +543,7 @@
     "4cat": "4CAT Tool settings",
     "api": "API credentials",
     "flask": "Flask settings",
-    "explorer": "Data Explorer",
+    "explorer": "Explorer",
     "datasources": "Data sources",
     "expire": "Dataset expiration settings",
     "mail": "Mail settings & credentials",

common/lib/dataset.py

@@ -15,7 +15,7 @@
 import backend
 from common.config_manager import config
 from common.lib.job import Job, JobNotFoundException
-from common.lib.helpers import get_software_commit, NullAwareTextIOWrapper, convert_to_int
+from common.lib.helpers import get_software_commit, NullAwareTextIOWrapper, convert_to_int, convert_to_float, flatten_dict
 from common.lib.item_mapping import MappedItem, MissingMappedField, DatasetItem
 from common.lib.fourcat_module import FourcatModule
 from common.lib.exceptions import (ProcessorInterruptedException, DataSetException, DataSetNotFoundException,
@@ -275,8 +275,9 @@ def _iterate_items(self, processor=None):
 					yield item
 
 		elif path.suffix.lower() == ".ndjson":
-			# In NDJSON format each line in the file is a self-contained JSON
+
 			with path.open(encoding="utf-8") as infile:
+
 				for line in infile:
 					if hasattr(processor, "interrupted") and processor.interrupted:
 						raise ProcessorInterruptedException("Processor interrupted while iterating through NDJSON file")
@@ -337,6 +338,10 @@ def iterate_items(self, processor=None, warn_unmappable=True, map_missing="defau
 		if own_processor and own_processor.map_item_method_available(dataset=self):
 			item_mapper = True
 
+		# Annotation fields are dynamically added,
+		# so we're always going to accept these.
+		annotation_fields = self.get_annotation_fields()
+
 		# Loop through items
 		for i, item in enumerate(self._iterate_items(processor)):
 			# Save original to yield
@@ -381,6 +386,15 @@ def iterate_items(self, processor=None, warn_unmappable=True, map_missing="defau
 
 			else:
 				mapped_item = original_item
+
+			# Re-add annotation fields to a mapped item.
+			if annotation_fields:
+				for annotation_field in annotation_fields.values():
+					label = annotation_field["label"]
+					if type(mapped_item) is MappedItem:
+						mapped_item.data[label] = original_item.get(label, "")
+					else:
+						mapped_item[label] = original_item.get(label, "")
 
 			# yield a DatasetItem, which is a dict with some special properties
 			yield DatasetItem(mapper=item_mapper, original=original_item, mapped_object=mapped_item, **(mapped_item.get_item_data() if type(mapped_item) is MappedItem else mapped_item))

common/lib/helpers.py

@@ -206,6 +206,22 @@ def convert_to_int(value, default=0):
     except (ValueError, TypeError):
         return default
 
+def convert_to_float(value, default=0):
+    """
+    Convert a value to a floating point, with a fallback
+
+    The fallback is used if an Error is thrown during converstion to float.
+    This is a convenience function, but beats putting try-catches everywhere
+    we're using user input as a floating point number.
+
+    :param value:  Value to convert
+    :param int default:  Default value, if conversion not possible
+    :return int:  Converted value
+    """
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        return default
 
 def timify_long(number):
     """
@@ -789,7 +805,7 @@ def flatten_dict(d: MutableMapping, parent_key: str = '', sep: str = '.'):
     Lists will be converted to json strings via json.dumps()
 
     :param MutableMapping d:  Dictionary like object
-    :param str partent_key: The original parent key prepending future nested keys
+    :param str parent_key: The original parent key prepending future nested keys
     :param str sep: A seperator string used to combine parent and child keys
     :return dict:  A new dictionary with the no nested values
     """

common/lib/user_input.py

@@ -35,6 +35,8 @@ class UserInput:
     OPTION_FILE = "file"  # file upload
     OPTION_HUE = "hue"  # colour hue
     OPTION_DATASOURCES = "datasources"  # data source toggling
+    OPTION_DATASOURCES_TABLE = "datasources_table" # a table with settings per data source
+    OPTION_DATASOURCES_TEXT = "datasources_text" # text input per data source (via dropdown)
 
     OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER)
 
@@ -143,6 +145,21 @@ def parse_all(options, input, silently_correct=True):
                 parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]]
                 parsed_input[option.split(".")[0] + ".expiration"] = datasources
 
+            elif settings.get("type") == UserInput.OPTION_DATASOURCES_TABLE:
+                # special case, parse table values to generate a dict
+                columns = list(settings["columns"].keys())
+                table_input = {}
+
+                for datasource in list(settings["default"].keys()):
+                    table_input[datasource] = {}
+                    for column in columns:
+
+                        choice = input.get(option + "-" + datasource + "-" + column, False)
+                        column_settings = settings["columns"][column] # sub-settings per column
+                        table_input[datasource][column] = UserInput.parse_value(column_settings, choice, table_input, silently_correct=True)
+
+                parsed_input[option] = table_input
+
             elif option not in input:
                 # not provided? use default
                 parsed_input[option] = settings.get("default", None)