diff --git a/src/Lentille.hs b/src/Lentille.hs index dfb74b8c6..bf5e54d2a 100644 --- a/src/Lentille.hs +++ b/src/Lentille.hs @@ -114,6 +114,7 @@ data LentilleErrorKind = DecodeError [Text] | RequestError GraphQLError | PageInfoError GraphQLError + | PartialErrors Value deriving (Show, Generic, ToJSON) yieldStreamError :: TimeEffect :> es => LentilleErrorKind -> LentilleStream es a diff --git a/src/Lentille/GraphQL.hs b/src/Lentille/GraphQL.hs index 14c86492d..03a414fc1 100644 --- a/src/Lentille/GraphQL.hs +++ b/src/Lentille/GraphQL.hs @@ -191,6 +191,8 @@ defaultStreamFetchOptParams = StreamFetchOptParams (const $ pure DontRetry) Noth mkGraphQLError :: GraphError a -> GraphQLError mkGraphQLError (req, fe) = GraphQLError (fmapFetchError (const ()) fe) req +data RequestResult a = RequestResult (Maybe Value) a + streamFetch :: forall es a b. (GraphEffects es, Fetch a, Show a) => @@ -215,11 +217,15 @@ streamFetch client@GraphClient {..} mkArgs StreamFetchOptParams {..} transformRe requestWithPageInfo pageInfoM storedRateLimitM = do holdOnIfNeeded storedRateLimitM eResp <- doRequest client mkArgs fpRetryCheck fpDepth pageInfoM + let handleResp mPartial resp = + let (pageInfo, rateLimitM, decodingErrors, xs) = transformResponse resp + in Right (rateLimitM, RequestResult mPartial (pageInfo, rateLimitM, decodingErrors, xs)) pure $ case eResp of + -- This is not a fatal error, it contains the desired response so handle it as a success. + -- The handler below will insert a 'PartialErrors' + Left (_, FetchErrorProducedErrors err (Just resp)) -> handleResp (Just (toJSON err)) resp Left err -> Left err - Right resp -> - let (pageInfo, rateLimitM, decodingErrors, xs) = transformResponse resp - in Right (rateLimitM, (pageInfo, rateLimitM, decodingErrors, xs)) + Right resp -> handleResp Nothing resp logStep pageInfo rateLimitM xs totalFetched = do lift @@ -262,10 +268,14 @@ streamFetch client@GraphClient {..} mkArgs StreamFetchOptParams {..} transformRe Left err -> -- Yield the error and stop the stream yieldStreamError $ RequestError (mkGraphQLError err) - Right (pageInfo, rateLimitM, decodingErrors, xs) -> do + Right (RequestResult mPartial (pageInfo, rateLimitM, decodingErrors, xs)) -> do -- Log crawling status logStep pageInfo rateLimitM xs totalFetched + forM_ mPartial \partial -> do + lift $ logWarn "Fetched partial result" ["err" .= partial] + yieldStreamError $ PartialErrors partial + unless (null decodingErrors) do yieldStreamError $ DecodeError decodingErrors diff --git a/src/Macroscope/Worker.hs b/src/Macroscope/Worker.hs index 8f85191e9..310f86b62 100644 --- a/src/Macroscope/Worker.hs +++ b/src/Macroscope/Worker.hs @@ -120,6 +120,7 @@ processStream entity logFunc postFunc = go (0 :: Word) [] [] DecodeError xs -> ("decode", encodeJSON xs) RequestError e -> ("graph", encodeJSON e) PageInfoError e -> ("page-info", encodeJSON e) + PartialErrors es -> ("partial", encodeJSON es) processBatch :: [DocumentType] -> Eff es (Maybe (ProcessError es)) processBatch [] = pure Nothing