Merge branch 'kitsune-new-fields' of 'https://github.com/jjmerchante/…

…grimoirelab-elk'

Merges #1148
Closes #1148
Fixes #1144
Fixes #1145
Fixes #1146
Fixes #1147

grimoire_elk/enriched/kitsune.py

@@ -111,7 +111,7 @@ def get_identities(self, item):
                     yield user
 
     @metadata
-    def get_rich_item(self, item, kind='question'):
+    def get_rich_item(self, item, kind='question', equestion=None):
         eitem = {}
 
         # Fields common in questions and answers
@@ -137,6 +137,11 @@ def get_rich_item(self, item, kind='question'):
                     eitem[f] = None
             eitem["content_analyzed"] = question['content']
 
+            if question['solution']:
+                eitem['is_kitsune_question_solved'] = 1
+            else:
+                eitem['is_kitsune_question_solved'] = 0
+
             # Fields which names are translated
             map_fields = {"id": "question_id",
                           "num_votes": "score"
@@ -158,10 +163,26 @@ def get_rich_item(self, item, kind='question'):
             eitem['lifetime_days'] = \
                 get_time_diff_days(question['created'], question['updated'])
 
+            # Time to first response
+            if 'answers_data' in question:
+                first_response = None
+                for answer in question['answers_data']:
+                    answer_date = str_to_datetime(answer['created'])
+                    if not first_response or answer_date < first_response:
+                        first_response = answer_date
+                if first_response:
+                    ttr = (first_response - str_to_datetime(question["created"]))
+                    hours = ttr.days * 24 + ttr.seconds / 3600.0
+                    eitem['time_to_first_reply_hours'] = hours
+
             # Add id info to allow to coexistence of items of different types in the same index
             eitem['id'] = 'question_{}'.format(question['id'])
             eitem.update(self.get_grimoire_fields(question['created'], "question"))
 
+            # Add URL info
+            origin = item['origin'].rstrip('/')
+            eitem['url'] = f"{origin}/{question['locale']}/questions/{question['id']}"
+
             eitem['author'] = question['creator']['username']
             if question['creator']['display_name']:
                 eitem['author'] = question['creator']['display_name']
@@ -180,8 +201,12 @@ def get_rich_item(self, item, kind='question'):
             eitem['type'] = kind
 
             # data fields to copy
+            for f in common_fields:
+                if f in equestion:
+                    eitem[f] = equestion[f]
+                else:
+                    eitem[f] = None
             copy_fields = ["content", "solution"]
-            copy_fields += common_fields
             for f in copy_fields:
                 if f in answer:
                     eitem[f] = answer[f]
@@ -216,6 +241,10 @@ def get_rich_item(self, item, kind='question'):
             if answer['creator']['display_name']:
                 eitem['author'] = answer['creator']['display_name']
 
+            origin = equestion['origin'].rstrip('/')
+            eitem['url'] = (f"{origin}/{equestion['locale']}/questions/"
+                            f"{equestion['question_id']}#answer-{answer['id']}")
+
             if self.sortinghat:
                 # date field must be the same than in question to share code
                 answer[self.get_field_date()] = answer['updated']
@@ -250,15 +279,15 @@ def enrich_items(self, ocean_backend):
                 (rich_item[self.get_field_unique_id()])
             bulk_json += data_json + "\n"  # Bulk document
             current += 1
-            # Time to enrich also de answers
+            # Time to enrich also the answers
             if 'answers_data' in item['data']:
                 for answer in item['data']['answers_data']:
                     # Add question title in answers
                     answer['title'] = item['data']['title']
                     answer['solution'] = 0
                     if answer['id'] == item['data']['solution']:
                         answer['solution'] = 1
-                    rich_answer = self.get_rich_item(answer, kind='answer')
+                    rich_answer = self.get_rich_item(answer, kind='answer', equestion=rich_item)
                     self.copy_raw_fields(self.RAW_FIELDS_COPY, item, rich_answer)
 
                     data_json = json.dumps(rich_answer)

releases/unreleased/kitsune-fields-updated.yml

@@ -0,0 +1,10 @@
+---
+title: Kitsune fields updated
+category: added
+author: Jose Javier Merchante <[email protected]>
+issue: null
+notes: >
+  Include new fields in Kitsune backend enriched index.
+  Include `product` and `url` fields in answers, and
+  `is_kitsune_question_solved`, `time_to_first_reply` and 
+  `url` in questions.