api: add error document type to the add doc endpoint

change-metrics · Dec 20, 2023 · 5125006 · 5125006
1 parent 5237f2a
commit 5125006
Show file tree

Hide file tree

Showing 5 changed files with 172 additions and 2 deletions.
diff --git a/codegen/Monocle/Protob/Crawler.hs b/codegen/Monocle/Protob/Crawler.hs
diff --git a/doc/openapi.yaml b/doc/openapi.yaml
diff --git a/schemas/monocle/protob/crawler.proto b/schemas/monocle/protob/crawler.proto
@@ -28,6 +28,11 @@ enum EntityType {
   ENTITY_TYPE_USER = 3;
 }
 
+message CrawlerError {
+  string message = 1;
+  string body = 2;
+}
+
 message AddDocRequest {
   string index = 1;
   string crawler = 2;
@@ -45,6 +50,7 @@ message AddDocRequest {
   repeated monocle_issue.Issue issues = 10;
   // issue_events are added when Entity is project_issue_name
   repeated monocle_issue.IssueEvent issue_events = 11;
+  repeated CrawlerError errors = 12;
 }
 
 enum AddDocError {

diff --git a/src/Macroscope/Worker.hs b/src/Macroscope/Worker.hs
@@ -78,6 +78,7 @@ data DocumentType
   | DTChanges (Change, [ChangeEvent])
   | DTTaskData TaskData
   | DTIssues (Issue, [IssueEvent])
+  | DTError CrawlerError
   deriving (Generic, ToJSON)
 
 data ProcessError es
@@ -246,6 +247,7 @@ runStreamError startTime apiKey indexName (CrawlerName crawlerName) documentStre
         addDocRequestTaskDatas = V.fromList $ mapMaybe getTD xs
         addDocRequestIssues = V.fromList $ mapMaybe getIssue xs
         addDocRequestIssueEvents = V.fromList $ concat $ mapMaybe getIssueEvent xs
+        addDocRequestErrors = V.fromList $ mapMaybe getError xs
      in AddDocRequest {..}
    where
     getIssue = \case
@@ -266,6 +268,9 @@ runStreamError startTime apiKey indexName (CrawlerName crawlerName) documentStre
     getTD = \case
       DTTaskData td -> Just td
       _ -> Nothing
+    getError = \case
+      DTError e -> Just e
+      _ -> Nothing
 
   -- 'commitTimestamp' post the commit date.
   commitTimestamp entity = do

diff --git a/src/Monocle/Api/Server.hs b/src/Monocle/Api/Server.hs
@@ -293,6 +293,7 @@ crawlerAddDoc _auth request = do
           taskDatas
           issues
           issuesEvents
+          _errors
         ) = request
 
   let requestE = do