diff --git a/codegen/Monocle/Protob/Crawler.hs b/codegen/Monocle/Protob/Crawler.hs index f379d4ac2..9d8d15ac0 100644 --- a/codegen/Monocle/Protob/Crawler.hs +++ b/codegen/Monocle/Protob/Crawler.hs @@ -346,7 +346,6 @@ data CrawlerError = CrawlerError , crawlerErrorBody :: Hs.Text , crawlerErrorCreatedAt :: Hs.Maybe Google.Protobuf.Timestamp.Timestamp - , crawlerErrorEntity :: Hs.Maybe Monocle.Protob.Crawler.Entity } deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) @@ -364,35 +363,27 @@ instance HsProtobuf.Message CrawlerError where { crawlerErrorMessage = crawlerErrorMessage , crawlerErrorBody = crawlerErrorBody , crawlerErrorCreatedAt = crawlerErrorCreatedAt - , crawlerErrorEntity = crawlerErrorEntity } = ( Hs.mconcat [ ( HsProtobuf.encodeMessageField - (HsProtobuf.FieldNumber 1) + (HsProtobuf.FieldNumber 2) ( Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (crawlerErrorMessage) ) ) , ( HsProtobuf.encodeMessageField - (HsProtobuf.FieldNumber 2) + (HsProtobuf.FieldNumber 3) ( Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (crawlerErrorBody) ) ) , ( HsProtobuf.encodeMessageField - (HsProtobuf.FieldNumber 3) + (HsProtobuf.FieldNumber 4) ( Hs.coerce @(Hs.Maybe Google.Protobuf.Timestamp.Timestamp) @(HsProtobuf.Nested Google.Protobuf.Timestamp.Timestamp) (crawlerErrorCreatedAt) ) ) - , ( HsProtobuf.encodeMessageField - (HsProtobuf.FieldNumber 4) - ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) - @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) - (crawlerErrorEntity) - ) - ) ] ) decodeMessage _ = @@ -400,26 +391,18 @@ instance HsProtobuf.Message CrawlerError where <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) ( HsProtobuf.at HsProtobuf.decodeMessageField - (HsProtobuf.FieldNumber 1) + (HsProtobuf.FieldNumber 2) ) ) <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) ( HsProtobuf.at HsProtobuf.decodeMessageField - (HsProtobuf.FieldNumber 2) + (HsProtobuf.FieldNumber 3) ) ) <*> ( HsProtobuf.coerceOver @(HsProtobuf.Nested Google.Protobuf.Timestamp.Timestamp) @(Hs.Maybe Google.Protobuf.Timestamp.Timestamp) - ( HsProtobuf.at - HsProtobuf.decodeMessageField - (HsProtobuf.FieldNumber 3) - ) - ) - <*> ( HsProtobuf.coerceOver - @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) - @(Hs.Maybe Monocle.Protob.Crawler.Entity) ( HsProtobuf.at HsProtobuf.decodeMessageField (HsProtobuf.FieldNumber 4) @@ -427,21 +410,21 @@ instance HsProtobuf.Message CrawlerError where ) dotProto _ = [ ( HsProtobufAST.DotProtoField - (HsProtobuf.FieldNumber 1) + (HsProtobuf.FieldNumber 2) (HsProtobufAST.Prim HsProtobufAST.String) (HsProtobufAST.Single "message") [] "" ) , ( HsProtobufAST.DotProtoField - (HsProtobuf.FieldNumber 2) + (HsProtobuf.FieldNumber 3) (HsProtobufAST.Prim HsProtobufAST.String) (HsProtobufAST.Single "body") [] "" ) , ( HsProtobufAST.DotProtoField - (HsProtobuf.FieldNumber 3) + (HsProtobuf.FieldNumber 4) ( HsProtobufAST.Prim ( HsProtobufAST.Named ( HsProtobufAST.Dots @@ -453,48 +436,29 @@ instance HsProtobuf.Message CrawlerError where [] "" ) - , ( HsProtobufAST.DotProtoField - (HsProtobuf.FieldNumber 4) - ( HsProtobufAST.Prim - (HsProtobufAST.Named (HsProtobufAST.Single "Entity")) - ) - (HsProtobufAST.Single "entity") - [] - "" - ) ] instance HsJSONPB.ToJSONPB CrawlerError where - toJSONPB (CrawlerError f1 f2 f3 f4) = + toJSONPB (CrawlerError f2 f3 f4) = ( HsJSONPB.object [ "message" - .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) - , "body" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f2)) + .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f2)) + , "body" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f3)) , "created_at" .= ( Hs.coerce @(Hs.Maybe Google.Protobuf.Timestamp.Timestamp) @(HsProtobuf.Nested Google.Protobuf.Timestamp.Timestamp) - (f3) - ) - , "entity" - .= ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) - @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) (f4) ) ] ) - toEncodingPB (CrawlerError f1 f2 f3 f4) = + toEncodingPB (CrawlerError f2 f3 f4) = ( HsJSONPB.pairs [ "message" - .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) - , "body" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f2)) + .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f2)) + , "body" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f3)) , "created_at" .= ( Hs.coerce @(Hs.Maybe Google.Protobuf.Timestamp.Timestamp) @(HsProtobuf.Nested Google.Protobuf.Timestamp.Timestamp) - (f3) - ) - , "entity" - .= ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) - @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) (f4) ) ] @@ -517,19 +481,177 @@ instance HsJSONPB.FromJSONPB CrawlerError where @(Hs.Maybe Google.Protobuf.Timestamp.Timestamp) (obj .: "created_at") ) + ) + ) + +instance HsJSONPB.ToJSON CrawlerError where + toJSON = HsJSONPB.toAesonValue + toEncoding = HsJSONPB.toAesonEncoding + +instance HsJSONPB.FromJSON CrawlerError where + parseJSON = HsJSONPB.parseJSONPB + +data CrawlerErrorList = CrawlerErrorList + { crawlerErrorListCrawler :: + Hs.Text + , crawlerErrorListEntity :: + Hs.Maybe Monocle.Protob.Crawler.Entity + , crawlerErrorListErrors :: + Hs.Vector Monocle.Protob.Crawler.CrawlerError + } + deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) + +instance Hs.NFData CrawlerErrorList + +instance HsProtobuf.Named CrawlerErrorList where + nameOf _ = (Hs.fromString "CrawlerErrorList") + +instance HsProtobuf.HasDefault CrawlerErrorList + +instance HsProtobuf.Message CrawlerErrorList where + encodeMessage + _ + CrawlerErrorList + { crawlerErrorListCrawler = crawlerErrorListCrawler + , crawlerErrorListEntity = crawlerErrorListEntity + , crawlerErrorListErrors = crawlerErrorListErrors + } = + ( Hs.mconcat + [ ( HsProtobuf.encodeMessageField + (HsProtobuf.FieldNumber 1) + ( Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) + (crawlerErrorListCrawler) + ) + ) + , ( HsProtobuf.encodeMessageField + (HsProtobuf.FieldNumber 2) + ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) + @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) + (crawlerErrorListEntity) + ) + ) + , ( HsProtobuf.encodeMessageField + (HsProtobuf.FieldNumber 3) + ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + (crawlerErrorListErrors) + ) + ) + ] + ) + decodeMessage _ = + (Hs.pure CrawlerErrorList) + <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) + ( HsProtobuf.at + HsProtobuf.decodeMessageField + (HsProtobuf.FieldNumber 1) + ) + ) + <*> ( HsProtobuf.coerceOver + @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) + @(Hs.Maybe Monocle.Protob.Crawler.Entity) + ( HsProtobuf.at + HsProtobuf.decodeMessageField + (HsProtobuf.FieldNumber 2) + ) + ) + <*> ( HsProtobuf.coerceOver + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + ( HsProtobuf.at + HsProtobuf.decodeMessageField + (HsProtobuf.FieldNumber 3) + ) + ) + dotProto _ = + [ ( HsProtobufAST.DotProtoField + (HsProtobuf.FieldNumber 1) + (HsProtobufAST.Prim HsProtobufAST.String) + (HsProtobufAST.Single "crawler") + [] + "" + ) + , ( HsProtobufAST.DotProtoField + (HsProtobuf.FieldNumber 2) + ( HsProtobufAST.Prim + (HsProtobufAST.Named (HsProtobufAST.Single "Entity")) + ) + (HsProtobufAST.Single "entity") + [] + "" + ) + , ( HsProtobufAST.DotProtoField + (HsProtobuf.FieldNumber 3) + ( HsProtobufAST.Repeated + (HsProtobufAST.Named (HsProtobufAST.Single "CrawlerError")) + ) + (HsProtobufAST.Single "errors") + [] + "" + ) + ] + +instance HsJSONPB.ToJSONPB CrawlerErrorList where + toJSONPB (CrawlerErrorList f1 f2 f3) = + ( HsJSONPB.object + [ "crawler" + .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) + , "entity" + .= ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) + @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) + (f2) + ) + , "errors" + .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + (f3) + ) + ] + ) + toEncodingPB (CrawlerErrorList f1 f2 f3) = + ( HsJSONPB.pairs + [ "crawler" + .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) + , "entity" + .= ( Hs.coerce @(Hs.Maybe Monocle.Protob.Crawler.Entity) + @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) + (f2) + ) + , "errors" + .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + (f3) + ) + ] + ) + +instance HsJSONPB.FromJSONPB CrawlerErrorList where + parseJSONPB = + ( HsJSONPB.withObject + "CrawlerErrorList" + ( \obj -> + (Hs.pure CrawlerErrorList) + <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) + (obj .: "crawler") + ) <*> ( HsProtobuf.coerceOver @(HsProtobuf.Nested Monocle.Protob.Crawler.Entity) @(Hs.Maybe Monocle.Protob.Crawler.Entity) (obj .: "entity") ) + <*> ( HsProtobuf.coerceOver + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + (obj .: "errors") + ) ) ) -instance HsJSONPB.ToJSON CrawlerError where +instance HsJSONPB.ToJSON CrawlerErrorList where toJSON = HsJSONPB.toAesonValue toEncoding = HsJSONPB.toAesonEncoding -instance HsJSONPB.FromJSON CrawlerError where +instance HsJSONPB.FromJSON CrawlerErrorList where parseJSON = HsJSONPB.parseJSONPB data ErrorsRequest = ErrorsRequest @@ -640,7 +762,7 @@ instance HsJSONPB.FromJSON ErrorsRequest where newtype ErrorsList = ErrorsList { errorsListErrors :: - Hs.Vector Monocle.Protob.Crawler.CrawlerError + Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList } deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) @@ -656,8 +778,8 @@ instance HsProtobuf.Message ErrorsList where ( Hs.mconcat [ ( HsProtobuf.encodeMessageField (HsProtobuf.FieldNumber 1) - ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) - @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerErrorList) (errorsListErrors) ) ) @@ -666,8 +788,8 @@ instance HsProtobuf.Message ErrorsList where decodeMessage _ = (Hs.pure ErrorsList) <*> ( HsProtobuf.coerceOver - @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) - @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerErrorList) + @(Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList) ( HsProtobuf.at HsProtobuf.decodeMessageField (HsProtobuf.FieldNumber 1) @@ -677,7 +799,7 @@ instance HsProtobuf.Message ErrorsList where [ ( HsProtobufAST.DotProtoField (HsProtobuf.FieldNumber 1) ( HsProtobufAST.Repeated - (HsProtobufAST.Named (HsProtobufAST.Single "CrawlerError")) + (HsProtobufAST.Named (HsProtobufAST.Single "CrawlerErrorList")) ) (HsProtobufAST.Single "errors") [] @@ -689,8 +811,8 @@ instance HsJSONPB.ToJSONPB ErrorsList where toJSONPB (ErrorsList f1) = ( HsJSONPB.object [ "errors" - .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) - @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerErrorList) (f1) ) ] @@ -698,8 +820,8 @@ instance HsJSONPB.ToJSONPB ErrorsList where toEncodingPB (ErrorsList f1) = ( HsJSONPB.pairs [ "errors" - .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) - @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) + .= ( Hs.coerce @(Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerErrorList) (f1) ) ] @@ -712,8 +834,8 @@ instance HsJSONPB.FromJSONPB ErrorsList where ( \obj -> (Hs.pure ErrorsList) <*> ( HsProtobuf.coerceOver - @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerError) - @(Hs.Vector Monocle.Protob.Crawler.CrawlerError) + @(HsProtobuf.NestedVec Monocle.Protob.Crawler.CrawlerErrorList) + @(Hs.Vector Monocle.Protob.Crawler.CrawlerErrorList) (obj .: "errors") ) ) diff --git a/doc/openapi.yaml b/doc/openapi.yaml index 370664ed5..9bd8404c6 100644 --- a/doc/openapi.yaml +++ b/doc/openapi.yaml @@ -753,8 +753,16 @@ components: created_at: type: string format: RFC3339 + monocle_crawler_CrawlerErrorList: + properties: + crawler: + type: string entity: $ref: '#/components/schemas/monocle_crawler_Entity' + errors: + type: array + items: + $ref: '#/components/schemas/monocle_crawler_CrawlerError' monocle_crawler_Entity: properties: organization_name: @@ -773,7 +781,7 @@ components: errors: type: array items: - $ref: '#/components/schemas/monocle_crawler_CrawlerError' + $ref: '#/components/schemas/monocle_crawler_CrawlerErrorList' monocle_crawler_ErrorsRequest: properties: index: diff --git a/schemas/monocle/protob/crawler.proto b/schemas/monocle/protob/crawler.proto index ebcc47201..f573b337d 100644 --- a/schemas/monocle/protob/crawler.proto +++ b/schemas/monocle/protob/crawler.proto @@ -29,10 +29,15 @@ enum EntityType { } message CrawlerError { - string message = 1; - string body = 2; - google.protobuf.Timestamp created_at = 3; - Entity entity = 4; + string message = 2; + string body = 3; + google.protobuf.Timestamp created_at = 4; +} + +message CrawlerErrorList { + string crawler = 1; + Entity entity = 2; + repeated CrawlerError errors = 3; } message ErrorsRequest { @@ -41,7 +46,7 @@ message ErrorsRequest { } message ErrorsList { - repeated CrawlerError errors = 1; + repeated CrawlerErrorList errors = 1; } message ErrorsResponse { diff --git a/src/Macroscope/Test.hs b/src/Macroscope/Test.hs index b28c84def..95044d4a8 100644 --- a/src/Macroscope/Test.hs +++ b/src/Macroscope/Test.hs @@ -1,6 +1,7 @@ -- | Tests for the macroscope process module Macroscope.Test where +import Data.Vector qualified as V import Effectful.Env import Effectful.Prometheus import Effectful.Reader.Static qualified as E @@ -60,9 +61,11 @@ testCrawlingPoint = do errorResponse <- crawlerErrors client (CrawlerPB.ErrorsRequest (from indexName) "from:2020") case errorResponse of CrawlerPB.ErrorsResponse (Just (CrawlerPB.ErrorsResponseResultSuccess (CrawlerPB.ErrorsList (toList -> [e])))) -> liftIO do - e.crawlerErrorMessage @?= "decode" - e.crawlerErrorBody @?= "[\"Oops\"]" - (from <$> e.crawlerErrorEntity) @?= Just (Project "opendev/neutron") + length e.crawlerErrorListErrors @?= 1 + let err = V.head e.crawlerErrorListErrors + err.crawlerErrorMessage @?= "decode" + err.crawlerErrorBody @?= "[\"Oops\"]" + (from <$> e.crawlerErrorListEntity) @?= Just (Project "opendev/neutron") _ -> error $ "Expected one error, got: " <> show errorResponse Macroscope.runStream apiKey indexName (CrawlerName crawlerName) (Macroscope.Changes $ goodStream currentOldestAge) diff --git a/src/Macroscope/Worker.hs b/src/Macroscope/Worker.hs index 310f86b62..e19776a1a 100644 --- a/src/Macroscope/Worker.hs +++ b/src/Macroscope/Worker.hs @@ -83,7 +83,6 @@ data ProcessError es -- | 'processStream' read the stream of document and post to the monocle API processStream :: forall es. - Entity -> -- | Funtion to log about the processing (Int -> Eff es ()) -> -- | Function to post on the Monocle API @@ -92,7 +91,7 @@ processStream :: LentilleStream es DocumentType -> -- | The processing results Eff es [Maybe (ProcessError es)] -processStream entity logFunc postFunc = go (0 :: Word) [] [] +processStream logFunc postFunc = go (0 :: Word) [] [] where go count acc results stream = do eDocument <- S.next stream @@ -113,10 +112,10 @@ processStream entity logFunc postFunc = go (0 :: Word) [] [] go 0 [] (res : results) rest else go (count + 1) newAcc results rest - toCrawlerError (LentilleError ts err) = - CrawlerError msg body (Just $ from ts) (Just $ from entity) + toCrawlerError (LentilleError ts err) = CrawlerError {..} where - (msg, body) = case err of + crawlerErrorCreatedAt = Just $ from ts + (crawlerErrorMessage, crawlerErrorBody) = case err of DecodeError xs -> ("decode", encodeJSON xs) RequestError e -> ("graph", encodeJSON e) PageInfoError e -> ("page-info", encodeJSON e) @@ -189,7 +188,6 @@ runStreamError startTime apiKey indexName (CrawlerName crawlerName) documentStre -- Run the document stream for that entity postResult <- processStream - entity (\c -> logInfo "Posting documents" ["count" .= c]) (httpRetry "api/commit/add" . mCrawlerAddDoc . mkRequest entity) (getStream oldestAge entity) diff --git a/src/Monocle/Api/Server.hs b/src/Monocle/Api/Server.hs index 92117a8cf..0df45b4c5 100644 --- a/src/Monocle/Api/Server.hs +++ b/src/Monocle/Api/Server.hs @@ -572,7 +572,7 @@ crawlerErrors auth request = checkAuth auth response case requestE of Right (tenant, query) -> runQueryM tenant (Q.ensureMinBound query) $ do logInfo "ListingErrors" ["index" .= request.errorsRequestIndex] - errors <- fmap from <$> Q.crawlerErrors + errors <- toErrorsList <$> Q.crawlerErrors pure $ CrawlerPB.ErrorsResponse $ Just $ CrawlerPB.ErrorsResponseResultSuccess $ CrawlerPB.ErrorsList $ fromList errors Left (ParseError msg offset) -> pure @@ -580,6 +580,23 @@ crawlerErrors auth request = checkAuth auth response $ Just $ CrawlerPB.ErrorsResponseResultError (show offset <> ":" <> from msg) + -- Group eerror by crawler name and entity + toErrorsList :: [EError] -> [CrawlerPB.CrawlerErrorList] + toErrorsList = fmap mkErrorList . Map.toList . mergeErrors + + mkErrorList :: ((LText, CrawlerPB.Entity), [EError]) -> CrawlerPB.CrawlerErrorList + mkErrorList ((crawlerErrorListCrawler, entity), errors) = CrawlerPB.CrawlerErrorList {..} + where + crawlerErrorListEntity = Just (from entity) + crawlerErrorListErrors = fromList (from <$> errors) + + mergeErrors :: [EError] -> Map (LText, CrawlerPB.Entity) [EError] + mergeErrors = Map.fromListWith (<>) . fmap toKVList + where + toKVList eerror = + let k = (from eerror.erCrawlerName, from eerror.erEntity) + in (k, [eerror]) + -- | /search/query endpoint searchQuery :: ApiEffects es => AuthResult AuthenticatedUser -> SearchPB.QueryRequest -> Eff es SearchPB.QueryResponse searchQuery auth request = checkAuth auth response diff --git a/src/Monocle/Backend/Documents.hs b/src/Monocle/Backend/Documents.hs index 39fb0ad95..cb0e2b86e 100644 --- a/src/Monocle/Backend/Documents.hs +++ b/src/Monocle/Backend/Documents.hs @@ -245,7 +245,6 @@ instance From EError CrawlerError where { crawlerErrorBody = from eerror.erBody , crawlerErrorMessage = from eerror.erMessage , crawlerErrorCreatedAt = Just $ from eerror.erCreatedAt - , crawlerErrorEntity = Just $ from eerror.erEntity } -- Custom encoder to manually serialize the entity type diff --git a/web/src/App.res b/web/src/App.res index 11822d9e1..2e91152c1 100644 --- a/web/src/App.res +++ b/web/src/App.res @@ -294,10 +294,8 @@ module Errors = { module CrawlerError = { @react.component let make = (~err: CrawlerTypes.crawler_error) => { - let entity: option = err.entity->Belt.Option.flatMap(Js.Json.stringifyAny)
getDate} /> -
{("entity: " ++ entity->Belt.Option.getWithDefault(""))->str}
{("message: " ++ err.message)->str}
{("body: " ++ err.body)->str}

@@ -305,6 +303,19 @@ module Errors = { } } + module CrawlerErrors = { + @react.component + let make = (~err: CrawlerTypes.crawler_error_list) => { + let entity: option = err.entity->Belt.Option.flatMap(Js.Json.stringifyAny) +
+
{("entity: " ++ entity->Belt.Option.getWithDefault(""))->str}
+
{("crawler: " ++ err.crawler)->str}
+ {err.errors->Belt.List.map(e => )->Belt.List.toArray->React.array} +
+
+ } + } + @react.component let make = (~store: Store.t) => { let (state, _) = store @@ -313,7 +324,7 @@ module Errors = {

{"The following errors happened when updating the index. This is likely causing some data to be missing."->str}

- {state.errors->Belt.List.map(e => )->Belt.List.toArray->React.array} + {state.errors->Belt.List.map(e => )->Belt.List.toArray->React.array} } } diff --git a/web/src/components/Store.res b/web/src/components/Store.res index 606a5f81e..3a20f1c9c 100644 --- a/web/src/components/Store.res +++ b/web/src/components/Store.res @@ -66,7 +66,7 @@ module Store = { about: ConfigTypes.about, dexie: Dexie.Database.t, toasts: list, - errors: list, + errors: list, } type action = | ChangeIndex(string) @@ -75,7 +75,7 @@ module Store = { | SetLimit(int) | SetOrder(option) | SetAuthorScopedTab(authorScopedTab) - | SetErrors(list) + | SetErrors(list) | FetchFields(fieldsRespR) | FetchSuggestions(suggestionsR) | FetchProjects(projectsR) diff --git a/web/src/messages/CrawlerBs.ml b/web/src/messages/CrawlerBs.ml index 8b05db5c4..2029af094 100644 --- a/web/src/messages/CrawlerBs.ml +++ b/web/src/messages/CrawlerBs.ml @@ -4,14 +4,24 @@ type crawler_error_mutable = { mutable message : string; mutable body : string; mutable created_at : TimestampTypes.timestamp option; - mutable entity : CrawlerTypes.entity option; } let default_crawler_error_mutable () : crawler_error_mutable = { message = ""; body = ""; created_at = None; +} + +type crawler_error_list_mutable = { + mutable crawler : string; + mutable entity : CrawlerTypes.entity option; + mutable errors : CrawlerTypes.crawler_error list; +} + +let default_crawler_error_list_mutable () : crawler_error_list_mutable = { + crawler = ""; entity = None; + errors = []; } type errors_request_mutable = { @@ -25,7 +35,7 @@ let default_errors_request_mutable () : errors_request_mutable = { } type errors_list_mutable = { - mutable errors : CrawlerTypes.crawler_error list; + mutable errors : CrawlerTypes.crawler_error_list list; } let default_errors_list_mutable () : errors_list_mutable = { @@ -160,9 +170,6 @@ let rec decode_crawler_error json = | "created_at" -> let json = Js.Dict.unsafeGet json "created_at" in v.created_at <- Some ((TimestampBs.decode_timestamp (Pbrt_bs.string json "crawler_error" "created_at"))) - | "entity" -> - let json = Js.Dict.unsafeGet json "entity" in - v.entity <- Some ((decode_entity (Pbrt_bs.object_ json "crawler_error" "entity"))) | _ -> () (*Unknown fields are ignored*) done; @@ -170,9 +177,38 @@ let rec decode_crawler_error json = CrawlerTypes.message = v.message; CrawlerTypes.body = v.body; CrawlerTypes.created_at = v.created_at; - CrawlerTypes.entity = v.entity; } : CrawlerTypes.crawler_error) +let rec decode_crawler_error_list json = + let v = default_crawler_error_list_mutable () in + let keys = Js.Dict.keys json in + let last_key_index = Array.length keys - 1 in + for i = 0 to last_key_index do + match Array.unsafe_get keys i with + | "crawler" -> + let json = Js.Dict.unsafeGet json "crawler" in + v.crawler <- Pbrt_bs.string json "crawler_error_list" "crawler" + | "entity" -> + let json = Js.Dict.unsafeGet json "entity" in + v.entity <- Some ((decode_entity (Pbrt_bs.object_ json "crawler_error_list" "entity"))) + | "errors" -> begin + let a = + let a = Js.Dict.unsafeGet json "errors" in + Pbrt_bs.array_ a "crawler_error_list" "errors" + in + v.errors <- Array.map (fun json -> + (decode_crawler_error (Pbrt_bs.object_ json "crawler_error_list" "errors")) + ) a |> Array.to_list; + end + + | _ -> () (*Unknown fields are ignored*) + done; + ({ + CrawlerTypes.crawler = v.crawler; + CrawlerTypes.entity = v.entity; + CrawlerTypes.errors = v.errors; + } : CrawlerTypes.crawler_error_list) + let rec decode_errors_request json = let v = default_errors_request_mutable () in let keys = Js.Dict.keys json in @@ -205,7 +241,7 @@ let rec decode_errors_list json = Pbrt_bs.array_ a "errors_list" "errors" in v.errors <- Array.map (fun json -> - (decode_crawler_error (Pbrt_bs.object_ json "errors_list" "errors")) + (decode_crawler_error_list (Pbrt_bs.object_ json "errors_list" "errors")) ) a |> Array.to_list; end @@ -540,6 +576,11 @@ let rec encode_crawler_error (v:CrawlerTypes.crawler_error) = Js.Dict.set json "created_at" (Js.Json.string json'); end; end; + json + +let rec encode_crawler_error_list (v:CrawlerTypes.crawler_error_list) = + let json = Js.Dict.empty () in + Js.Dict.set json "crawler" (Js.Json.string v.CrawlerTypes.crawler); begin match v.CrawlerTypes.entity with | None -> () | Some v -> @@ -548,6 +589,17 @@ let rec encode_crawler_error (v:CrawlerTypes.crawler_error) = Js.Dict.set json "entity" (Js.Json.object_ json'); end; end; + begin (* errors field *) + let (errors':Js.Json.t) = + v.CrawlerTypes.errors + |> Array.of_list + |> Array.map (fun v -> + v |> encode_crawler_error |> Js.Json.object_ + ) + |> Js.Json.array + in + Js.Dict.set json "errors" errors'; + end; json let rec encode_errors_request (v:CrawlerTypes.errors_request) = @@ -563,7 +615,7 @@ let rec encode_errors_list (v:CrawlerTypes.errors_list) = v.CrawlerTypes.errors |> Array.of_list |> Array.map (fun v -> - v |> encode_crawler_error |> Js.Json.object_ + v |> encode_crawler_error_list |> Js.Json.object_ ) |> Js.Json.array in diff --git a/web/src/messages/CrawlerBs.mli b/web/src/messages/CrawlerBs.mli index ad6a614f2..d625e6727 100644 --- a/web/src/messages/CrawlerBs.mli +++ b/web/src/messages/CrawlerBs.mli @@ -12,6 +12,9 @@ val encode_entity_type : CrawlerTypes.entity_type -> string val encode_crawler_error : CrawlerTypes.crawler_error -> Js.Json.t Js.Dict.t (** [encode_crawler_error v dict] encodes [v] int the given JSON [dict] *) +val encode_crawler_error_list : CrawlerTypes.crawler_error_list -> Js.Json.t Js.Dict.t +(** [encode_crawler_error_list v dict] encodes [v] int the given JSON [dict] *) + val encode_errors_request : CrawlerTypes.errors_request -> Js.Json.t Js.Dict.t (** [encode_errors_request v dict] encodes [v] int the given JSON [dict] *) @@ -66,6 +69,9 @@ val decode_entity_type : Js.Json.t -> CrawlerTypes.entity_type val decode_crawler_error : Js.Json.t Js.Dict.t -> CrawlerTypes.crawler_error (** [decode_crawler_error decoder] decodes a [crawler_error] value from [decoder] *) +val decode_crawler_error_list : Js.Json.t Js.Dict.t -> CrawlerTypes.crawler_error_list +(** [decode_crawler_error_list decoder] decodes a [crawler_error_list] value from [decoder] *) + val decode_errors_request : Js.Json.t Js.Dict.t -> CrawlerTypes.errors_request (** [decode_errors_request decoder] decodes a [errors_request] value from [decoder] *) diff --git a/web/src/messages/CrawlerTypes.ml b/web/src/messages/CrawlerTypes.ml index 70a3be931..481404548 100644 --- a/web/src/messages/CrawlerTypes.ml +++ b/web/src/messages/CrawlerTypes.ml @@ -18,7 +18,12 @@ type crawler_error = { message : string; body : string; created_at : TimestampTypes.timestamp option; +} + +type crawler_error_list = { + crawler : string; entity : entity option; + errors : crawler_error list; } type errors_request = { @@ -27,7 +32,7 @@ type errors_request = { } type errors_list = { - errors : crawler_error list; + errors : crawler_error_list list; } type errors_response = @@ -109,12 +114,20 @@ let rec default_crawler_error ?message:((message:string) = "") ?body:((body:string) = "") ?created_at:((created_at:TimestampTypes.timestamp option) = None) - ?entity:((entity:entity option) = None) () : crawler_error = { message; body; created_at; +} + +let rec default_crawler_error_list + ?crawler:((crawler:string) = "") + ?entity:((entity:entity option) = None) + ?errors:((errors:crawler_error list) = []) + () : crawler_error_list = { + crawler; entity; + errors; } let rec default_errors_request @@ -126,7 +139,7 @@ let rec default_errors_request } let rec default_errors_list - ?errors:((errors:crawler_error list) = []) + ?errors:((errors:crawler_error_list list) = []) () : errors_list = { errors; } diff --git a/web/src/messages/CrawlerTypes.mli b/web/src/messages/CrawlerTypes.mli index 14e3338b6..c7777f5f7 100644 --- a/web/src/messages/CrawlerTypes.mli +++ b/web/src/messages/CrawlerTypes.mli @@ -21,7 +21,12 @@ type crawler_error = { message : string; body : string; created_at : TimestampTypes.timestamp option; +} + +type crawler_error_list = { + crawler : string; entity : entity option; + errors : crawler_error list; } type errors_request = { @@ -30,7 +35,7 @@ type errors_request = { } type errors_list = { - errors : crawler_error list; + errors : crawler_error_list list; } type errors_response = @@ -117,11 +122,18 @@ val default_crawler_error : ?message:string -> ?body:string -> ?created_at:TimestampTypes.timestamp option -> - ?entity:entity option -> unit -> crawler_error (** [default_crawler_error ()] is the default value for type [crawler_error] *) +val default_crawler_error_list : + ?crawler:string -> + ?entity:entity option -> + ?errors:crawler_error list -> + unit -> + crawler_error_list +(** [default_crawler_error_list ()] is the default value for type [crawler_error_list] *) + val default_errors_request : ?index:string -> ?query:string -> @@ -130,7 +142,7 @@ val default_errors_request : (** [default_errors_request ()] is the default value for type [errors_request] *) val default_errors_list : - ?errors:crawler_error list -> + ?errors:crawler_error_list list -> unit -> errors_list (** [default_errors_list ()] is the default value for type [errors_list] *)