From 56e3c7106a48b06b801ef252fd1059fbf43e77e5 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Wed, 25 Oct 2023 17:30:13 +0000 Subject: [PATCH 1/2] crawler - github - index the mergeCommit's sha Fix #1072 --- CHANGELOG.md | 2 + codegen/Monocle/Protob/Change.hs | 179 ++++++++++++++++++++++++ schemas/monocle/protob/change.proto | 6 + src/Lentille/Gerrit.hs | 2 + src/Lentille/GitHub/GraphQLFragments.hs | 4 + src/Lentille/GitHub/PullRequests.hs | 4 + src/Lentille/GitHub/UserPullRequests.hs | 4 + src/Lentille/GitHub/Utils.hs | 4 + src/Lentille/GitLab/MergeRequests.hs | 2 + src/Monocle/Backend/Documents.hs | 4 + src/Monocle/Backend/Index.hs | 33 ++++- src/Monocle/Backend/Provisioner.hs | 2 + src/Monocle/Backend/Test.hs | 8 ++ 13 files changed, 249 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6f98474..902476ff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. - [crawler] Enable usage of the GitHub user PRs crawler via the Monocle config. - [cli] Add a Janitor command to reset the commit date of a crawler +- [crawler] Github crawler gathers the `mergeCommit`'s SHA. `Change` and `ChangeEvent*` objects + get a new field `merged_commit_sha` in the database. ### Changed diff --git a/codegen/Monocle/Protob/Change.hs b/codegen/Monocle/Protob/Change.hs index 7c3aafbde..b704eb9b6 100644 --- a/codegen/Monocle/Protob/Change.hs +++ b/codegen/Monocle/Protob/Change.hs @@ -680,6 +680,8 @@ data Change = Change , changeApprovals :: Hs.Vector Hs.Text , changeDraft :: Hs.Bool , changeOptionalSelfMerged :: Hs.Maybe ChangeOptionalSelfMerged + , changeOptionalMergedCommitSha :: + Hs.Maybe ChangeOptionalMergedCommitSha } deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) @@ -725,6 +727,7 @@ instance HsProtobuf.Message Change where , changeApprovals = changeApprovals , changeDraft = changeDraft , changeOptionalSelfMerged = changeOptionalSelfMerged + , changeOptionalMergedCommitSha = changeOptionalMergedCommitSha } = ( Hs.mconcat [ ( HsProtobuf.encodeMessageField @@ -923,6 +926,17 @@ instance HsProtobuf.Message Change where (HsProtobuf.FieldNumber 31) (HsProtobuf.ForceEmit y) ) + , case changeOptionalMergedCommitSha of + Hs.Nothing -> Hs.mempty + Hs.Just x -> + case x of + ChangeOptionalMergedCommitShaMergedCommitSha y -> + ( HsProtobuf.encodeMessageField + (HsProtobuf.FieldNumber 32) + ( HsProtobuf.ForceEmit + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (y)) + ) + ) ] ) decodeMessage _ = @@ -1142,6 +1156,19 @@ instance HsProtobuf.Message Change where ) ] ) + <*> ( HsProtobuf.oneof + Hs.Nothing + [ + ( (HsProtobuf.FieldNumber 32) + , ( Hs.pure + (Hs.Just Hs.. ChangeOptionalMergedCommitShaMergedCommitSha) + ) + <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) + (HsProtobuf.decodeMessageField) + ) + ) + ] + ) dotProto _ = [ ( HsProtobufAST.DotProtoField (HsProtobuf.FieldNumber 1) @@ -1383,6 +1410,7 @@ instance HsJSONPB.ToJSONPB Change where f29 f30 f31 + f32 ) = ( HsJSONPB.object [ "id" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) @@ -1526,6 +1554,24 @@ instance HsJSONPB.ToJSONPB Change where options else encodeOptional_self_merged options ) + , ( let encodeOptional_merged_commit_sha = + ( case f32 of + Hs.Just (ChangeOptionalMergedCommitShaMergedCommitSha f32) -> + ( HsJSONPB.pair + "merged_commit_sha" + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f32)) + ) + Hs.Nothing -> Hs.mempty + ) + in \options -> + if HsJSONPB.optEmitNamedOneof options + then + ( "optional_merged_commit_sha" + .= (HsJSONPB.objectOrNull [encodeOptional_merged_commit_sha] options) + ) + options + else encodeOptional_merged_commit_sha options + ) ] ) toEncodingPB @@ -1561,6 +1607,7 @@ instance HsJSONPB.ToJSONPB Change where f29 f30 f31 + f32 ) = ( HsJSONPB.pairs [ "id" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) @@ -1704,6 +1751,24 @@ instance HsJSONPB.ToJSONPB Change where options else encodeOptional_self_merged options ) + , ( let encodeOptional_merged_commit_sha = + ( case f32 of + Hs.Just (ChangeOptionalMergedCommitShaMergedCommitSha f32) -> + ( HsJSONPB.pair + "merged_commit_sha" + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f32)) + ) + Hs.Nothing -> Hs.mempty + ) + in \options -> + if HsJSONPB.optEmitNamedOneof options + then + ( "optional_merged_commit_sha" + .= (HsJSONPB.pairsOrNull [encodeOptional_merged_commit_sha] options) + ) + options + else encodeOptional_merged_commit_sha options + ) ] ) @@ -1851,6 +1916,22 @@ instance HsJSONPB.FromJSONPB Change where ) <|> (parseOptional_self_merged obj) ) + <*> ( let parseOptional_merged_commit_sha parseObj = + Hs.msum + [ Hs.Just + Hs.. ChangeOptionalMergedCommitShaMergedCommitSha + Hs.. Hs.coerce @(HsProtobuf.String Hs.Text) @(Hs.Text) + <$> (HsJSONPB.parseField parseObj "merged_commit_sha") + , Hs.pure Hs.Nothing + ] + in ( (obj .: "optional_merged_commit_sha") + Hs.>>= ( HsJSONPB.withObject + "optional_merged_commit_sha" + parseOptional_merged_commit_sha + ) + ) + <|> (parseOptional_merged_commit_sha obj) + ) ) ) @@ -1953,6 +2034,14 @@ instance Hs.NFData ChangeOptionalSelfMerged instance HsProtobuf.Named ChangeOptionalSelfMerged where nameOf _ = (Hs.fromString "ChangeOptionalSelfMerged") +data ChangeOptionalMergedCommitSha = ChangeOptionalMergedCommitShaMergedCommitSha Hs.Text + deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) + +instance Hs.NFData ChangeOptionalMergedCommitSha + +instance HsProtobuf.Named ChangeOptionalMergedCommitSha where + nameOf _ = (Hs.fromString "ChangeOptionalMergedCommitSha") + data ChangeCreatedEvent = ChangeCreatedEvent {} deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) @@ -2265,6 +2354,8 @@ data ChangeEvent = ChangeEvent , changeEventOptionalDuration :: Hs.Maybe ChangeEventOptionalDuration , changeEventDraft :: Hs.Bool + , changeEventOptionalMergedCommitSha :: + Hs.Maybe ChangeEventOptionalMergedCommitSha } deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) @@ -2297,6 +2388,8 @@ instance HsProtobuf.Message ChangeEvent where , changeEventLabels = changeEventLabels , changeEventOptionalDuration = changeEventOptionalDuration , changeEventDraft = changeEventDraft + , changeEventOptionalMergedCommitSha = + changeEventOptionalMergedCommitSha } = ( Hs.mconcat [ ( HsProtobuf.encodeMessageField @@ -2471,6 +2564,17 @@ instance HsProtobuf.Message ChangeEvent where (HsProtobuf.FieldNumber 24) changeEventDraft ) + , case changeEventOptionalMergedCommitSha of + Hs.Nothing -> Hs.mempty + Hs.Just x -> + case x of + ChangeEventOptionalMergedCommitShaMergedCommitSha y -> + ( HsProtobuf.encodeMessageField + (HsProtobuf.FieldNumber 25) + ( HsProtobuf.ForceEmit + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (y)) + ) + ) ] ) decodeMessage _ = @@ -2655,6 +2759,19 @@ instance HsProtobuf.Message ChangeEvent where HsProtobuf.decodeMessageField (HsProtobuf.FieldNumber 24) ) + <*> ( HsProtobuf.oneof + Hs.Nothing + [ + ( (HsProtobuf.FieldNumber 25) + , ( Hs.pure + (Hs.Just Hs.. ChangeEventOptionalMergedCommitShaMergedCommitSha) + ) + <*> ( HsProtobuf.coerceOver @(HsProtobuf.String Hs.Text) @(Hs.Text) + (HsProtobuf.decodeMessageField) + ) + ) + ] + ) dotProto _ = [ ( HsProtobufAST.DotProtoField (HsProtobuf.FieldNumber 1) @@ -2809,6 +2926,7 @@ instance HsJSONPB.ToJSONPB ChangeEvent where f22 f23 f24 + f25 ) = ( HsJSONPB.object [ "id" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) @@ -2895,6 +3013,24 @@ instance HsJSONPB.ToJSONPB ChangeEvent where else encodeOptional_duration options ) , "draft" .= f24 + , ( let encodeOptional_merged_commit_sha = + ( case f25 of + Hs.Just (ChangeEventOptionalMergedCommitShaMergedCommitSha f25) -> + ( HsJSONPB.pair + "merged_commit_sha" + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f25)) + ) + Hs.Nothing -> Hs.mempty + ) + in \options -> + if HsJSONPB.optEmitNamedOneof options + then + ( "optional_merged_commit_sha" + .= (HsJSONPB.objectOrNull [encodeOptional_merged_commit_sha] options) + ) + options + else encodeOptional_merged_commit_sha options + ) ] ) toEncodingPB @@ -2917,6 +3053,7 @@ instance HsJSONPB.ToJSONPB ChangeEvent where f22 f23 f24 + f25 ) = ( HsJSONPB.pairs [ "id" .= (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f1)) @@ -3003,6 +3140,24 @@ instance HsJSONPB.ToJSONPB ChangeEvent where else encodeOptional_duration options ) , "draft" .= f24 + , ( let encodeOptional_merged_commit_sha = + ( case f25 of + Hs.Just (ChangeEventOptionalMergedCommitShaMergedCommitSha f25) -> + ( HsJSONPB.pair + "merged_commit_sha" + (Hs.coerce @(Hs.Text) @(HsProtobuf.String Hs.Text) (f25)) + ) + Hs.Nothing -> Hs.mempty + ) + in \options -> + if HsJSONPB.optEmitNamedOneof options + then + ( "optional_merged_commit_sha" + .= (HsJSONPB.pairsOrNull [encodeOptional_merged_commit_sha] options) + ) + options + else encodeOptional_merged_commit_sha options + ) ] ) @@ -3100,6 +3255,22 @@ instance HsJSONPB.FromJSONPB ChangeEvent where <|> (parseOptional_duration obj) ) <*> obj .: "draft" + <*> ( let parseOptional_merged_commit_sha parseObj = + Hs.msum + [ Hs.Just + Hs.. ChangeEventOptionalMergedCommitShaMergedCommitSha + Hs.. Hs.coerce @(HsProtobuf.String Hs.Text) @(Hs.Text) + <$> (HsJSONPB.parseField parseObj "merged_commit_sha") + , Hs.pure Hs.Nothing + ] + in ( (obj .: "optional_merged_commit_sha") + Hs.>>= ( HsJSONPB.withObject + "optional_merged_commit_sha" + parseOptional_merged_commit_sha + ) + ) + <|> (parseOptional_merged_commit_sha obj) + ) ) ) @@ -3132,3 +3303,11 @@ instance Hs.NFData ChangeEventOptionalDuration instance HsProtobuf.Named ChangeEventOptionalDuration where nameOf _ = (Hs.fromString "ChangeEventOptionalDuration") + +data ChangeEventOptionalMergedCommitSha = ChangeEventOptionalMergedCommitShaMergedCommitSha Hs.Text + deriving (Hs.Show, Hs.Eq, Hs.Ord, Hs.Generic) + +instance Hs.NFData ChangeEventOptionalMergedCommitSha + +instance HsProtobuf.Named ChangeEventOptionalMergedCommitSha where + nameOf _ = (Hs.fromString "ChangeEventOptionalMergedCommitSha") diff --git a/schemas/monocle/protob/change.proto b/schemas/monocle/protob/change.proto index f5945a31d..c13d6f1e1 100644 --- a/schemas/monocle/protob/change.proto +++ b/schemas/monocle/protob/change.proto @@ -80,6 +80,9 @@ message Change { oneof optional_self_merged { bool self_merged = 31; } + oneof optional_merged_commit_sha { + string merged_commit_sha = 32; + } } message ChangeCreatedEvent {} @@ -127,4 +130,7 @@ message ChangeEvent { int32 duration = 23; } bool draft = 24; + oneof optional_merged_commit_sha { + string merged_commit_sha = 25; + } } diff --git a/src/Lentille/Gerrit.hs b/src/Lentille/Gerrit.hs index 09baca07d..c4f0e22b7 100644 --- a/src/Lentille/Gerrit.hs +++ b/src/Lentille/Gerrit.hs @@ -266,6 +266,7 @@ streamChange' env identCB serverUrl query prefixM = go 0 changeEventType = Just eType changeEventOptionalDuration = swapDuration <$> changeOptionalDuration changeEventDraft = changeDraft + changeEventOptionalMergedCommitSha = Nothing in ChangePB.ChangeEvent {..} toChangeCreatedEvent = baseEvent (ChangePB.ChangeEventTypeChangeCreated ChangePB.ChangeCreatedEvent) $ "CCE" <> changeId @@ -343,6 +344,7 @@ streamChange' env identCB serverUrl query prefixM = go 0 if status == MERGED then ChangePB.ChangeOptionalMergedByMergedBy <$> merger else Nothing + changeOptionalMergedCommitSha = Nothing changeBranch = from branch changeTargetBranch = from branch changeCreatedAt = Just $ toTimestamp created diff --git a/src/Lentille/GitHub/GraphQLFragments.hs b/src/Lentille/GitHub/GraphQLFragments.hs index f45a48267..e27fb73ea 100644 --- a/src/Lentille/GitHub/GraphQLFragments.hs +++ b/src/Lentille/GitHub/GraphQLFragments.hs @@ -40,6 +40,10 @@ fragmentPRData = bodyText state reviewDecision + mergeCommit { + __typename + mergedCommitOid: oid + } number mergeable isDraft diff --git a/src/Lentille/GitHub/PullRequests.hs b/src/Lentille/GitHub/PullRequests.hs index de9192a07..53aea7a4d 100644 --- a/src/Lentille/GitHub/PullRequests.hs +++ b/src/Lentille/GitHub/PullRequests.hs @@ -149,6 +149,9 @@ transformResponse host identCB result = do getPRMergedBy :: GetProjectPullRequestsRepositoryPullRequestsNodesMergedBy -> Text getPRMergedBy (GetProjectPullRequestsRepositoryPullRequestsNodesMergedBy _ login) = login + getPRMergedCommitSHA :: GetProjectPullRequestsRepositoryPullRequestsNodesMergeCommit -> LText + getPRMergedCommitSHA (GetProjectPullRequestsRepositoryPullRequestsNodesMergeCommit _ sha) = from $ getSHA sha + toLabels :: GetProjectPullRequestsRepositoryPullRequestsNodesLabels -> [Text] toLabels (GetProjectPullRequestsRepositoryPullRequestsNodesLabels nodes) = toLabel <$> catMaybes (fromMaybe [] nodes) @@ -250,6 +253,7 @@ transformResponse host identCB result = do , changeRepositoryShortname = from . snd $ repoOwnerName repository , changeAuthor = Just $ maybe getGhostIdent (getIdent . getPRAuthor) author , changeOptionalMergedBy = ChangeOptionalMergedByMergedBy . getIdent . getPRMergedBy <$> mergedBy + , changeOptionalMergedCommitSha = ChangeOptionalMergedCommitShaMergedCommitSha . getPRMergedCommitSHA <$> mergeCommit , changeBranch = from headRefName , changeTargetBranch = from baseRefName , changeCreatedAt = Just $ from createdAt diff --git a/src/Lentille/GitHub/UserPullRequests.hs b/src/Lentille/GitHub/UserPullRequests.hs index c2dbda821..f53659a63 100644 --- a/src/Lentille/GitHub/UserPullRequests.hs +++ b/src/Lentille/GitHub/UserPullRequests.hs @@ -146,6 +146,9 @@ transformResponse host identCB result = do getPRMergedBy :: GetUserPullRequestsUserPullRequestsNodesMergedBy -> Text getPRMergedBy (GetUserPullRequestsUserPullRequestsNodesMergedBy _ login) = login + getPRMergedCommitSHA :: GetUserPullRequestsUserPullRequestsNodesMergeCommit -> LText + getPRMergedCommitSHA (GetUserPullRequestsUserPullRequestsNodesMergeCommit _ sha) = from $ getSHA sha + toLabels :: GetUserPullRequestsUserPullRequestsNodesLabels -> [Text] toLabels (GetUserPullRequestsUserPullRequestsNodesLabels nodes) = toLabel <$> catMaybes (fromMaybe [] nodes) @@ -247,6 +250,7 @@ transformResponse host identCB result = do , changeRepositoryShortname = from . snd $ repoOwnerName repository , changeAuthor = Just $ maybe getGhostIdent (getIdent . getPRAuthor) author , changeOptionalMergedBy = ChangeOptionalMergedByMergedBy . getIdent . getPRMergedBy <$> mergedBy + , changeOptionalMergedCommitSha = ChangeOptionalMergedCommitShaMergedCommitSha . getPRMergedCommitSHA <$> mergeCommit , changeBranch = from headRefName , changeTargetBranch = from baseRefName , changeCreatedAt = Just $ from createdAt diff --git a/src/Lentille/GitHub/Utils.hs b/src/Lentille/GitHub/Utils.hs index ac63b6302..ae2874018 100644 --- a/src/Lentille/GitHub/Utils.hs +++ b/src/Lentille/GitHub/Utils.hs @@ -67,6 +67,9 @@ toDuration d1 d2 = ChangeOptionalDurationDuration . from $ diffTimeSec (from d1) toChangeFilePath :: ChangedFile -> ChangedFilePath toChangeFilePath (ChangedFile _ _ path) = ChangedFilePath path +toChangeEventMergedCommitSha :: ChangeOptionalMergedCommitSha -> ChangeEventOptionalMergedCommitSha +toChangeEventMergedCommitSha (ChangeOptionalMergedCommitShaMergedCommitSha sha) = ChangeEventOptionalMergedCommitShaMergedCommitSha sha + baseEvent :: ChangeEventType -> LText -> Change -> ChangeEvent baseEvent eType eId change = ChangeEvent @@ -89,6 +92,7 @@ baseEvent eType eId change = changeEventCreatedAt = Nothing , changeEventAuthor = Nothing , changeEventOptionalDuration = swapDuration <$> changeOptionalDuration change + , changeEventOptionalMergedCommitSha = toChangeEventMergedCommitSha <$> changeOptionalMergedCommitSha change } instance From DateTime ChangeOptionalClosedAt where diff --git a/src/Lentille/GitLab/MergeRequests.hs b/src/Lentille/GitLab/MergeRequests.hs index 21d5b5756..872f3854d 100644 --- a/src/Lentille/GitLab/MergeRequests.hs +++ b/src/Lentille/GitLab/MergeRequests.hs @@ -208,6 +208,7 @@ transformResponse host getIdentIdCB result = . ChangeOptionalMergedByMergedBy $ maybe (ghostIdent host) (toIdent' . getMergerUsername) mergeUser ) + changeOptionalMergedCommitSha = Nothing changeBranch = from sourceBranch changeTargetBranch = from targetBranch changeCreatedAt = (Just $ timeToTimestamp Nothing createdAt) @@ -287,6 +288,7 @@ transformResponse host getIdentIdCB result = changeEventLabels = changeLabels changeEventOptionalDuration = swapDuration <$> changeOptionalDuration changeEventId = "" + changeEventOptionalMergedCommitSha = Nothing in ChangeEvent {..} where toChangeFilePath ChangedFile {..} = ChangedFilePath changedFilePath diff --git a/src/Monocle/Backend/Documents.hs b/src/Monocle/Backend/Documents.hs index a9c8f2af3..c16b441c6 100644 --- a/src/Monocle/Backend/Documents.hs +++ b/src/Monocle/Backend/Documents.hs @@ -366,6 +366,7 @@ data EChange = EChange , echangeRepositoryFullname :: LText , echangeAuthor :: Author , echangeMergedBy :: Maybe Author + , echangeMergedCommitSha :: Maybe LText , echangeBranch :: LText , echangeTargetBranch :: LText , echangeCreatedAt :: UTCTime @@ -442,6 +443,7 @@ instance From ChangePB.Change EChange where , echangeRepositoryShortname = changeRepositoryShortname , echangeAuthor = from (ensureAuthor changeAuthor) , echangeMergedBy = toMergedByAuthor <$> changeOptionalMergedBy + , echangeMergedCommitSha = toMergedCommitSha <$> changeOptionalMergedCommitSha , echangeBranch = changeBranch , echangeTargetBranch = changeTargetBranch , echangeCreatedAt = from $ fromMaybe (error "CreatedAt field is mandatory") changeCreatedAt @@ -468,6 +470,7 @@ instance From ChangePB.Change EChange where ChangePB.Change_ChangeStateOpen -> EChangeOpen ChangePB.Change_ChangeStateMerged -> EChangeMerged ChangePB.Change_ChangeStateClosed -> EChangeClosed + toMergedCommitSha (ChangePB.ChangeOptionalMergedCommitShaMergedCommitSha sha) = sha instance From ChangePB.ChangedFile File where from ChangePB.ChangedFile {..} = @@ -576,6 +579,7 @@ data EChangeEvent = EChangeEvent , echangeeventTasksData :: Maybe [ETaskData] , echangeeventDuration :: Maybe Int , echangeeventDraft :: Maybe Bool + , echangeeventMergedCommitSha :: Maybe LText } deriving (Show, Eq, Generic) diff --git a/src/Monocle/Backend/Index.hs b/src/Monocle/Backend/Index.hs index 1814a3234..bcab3d2e0 100644 --- a/src/Monocle/Backend/Index.hs +++ b/src/Monocle/Backend/Index.hs @@ -61,12 +61,12 @@ instance ToJSON AuthorIndexMapping where toJSON AuthorIndexMapping = object ["properties" .= AuthorMapping] -cachedAuthorFields :: [Pair] -cachedAuthorFields = ["cached_author_muid" .= TextAndKWMapping] +cachedAuthorField :: [Pair] +cachedAuthorField = ["cached_author_muid" .= TextAndKWMapping] instance ToJSON CachedAuthorIndexMapping where toJSON CachedAuthorIndexMapping = - object ["properties" .= object cachedAuthorFields] + object ["properties" .= object cachedAuthorField] data DateIndexMapping = DateIndexMapping deriving (Eq, Show) @@ -77,6 +77,19 @@ instance ToJSON DateIndexMapping where , "format" .= ("date_time_no_millis" :: Text) ] +-- https://www.elastic.co/guide/en/elasticsearch/reference/7.17/enabled.html#enabled +data NonIndexedMapping = NonIndexedMapping deriving (Eq, Show) + +instance ToJSON NonIndexedMapping where + toJSON NonIndexedMapping = + object + [ "type" .= ("object" :: Text) + , "enabled" .= (False :: Bool) + ] + +mergedCommitField :: [Pair] +mergedCommitField = ["merged_commit_sha" .= NonIndexedMapping] + data TextAndKWMapping = TextAndKWMapping deriving (Eq, Show) instance ToJSON TextAndKWMapping where @@ -207,7 +220,8 @@ instance ToJSON ChangesIndexMapping where ] ] ] - <> cachedAuthorFields + <> cachedAuthorField + <> mergedCommitField ) ] @@ -226,7 +240,7 @@ createIndex indexName mapping = do retryPolicy = exponentialBackoff 500_000 <> limitRetries 7 configVersion :: ConfigVersion -configVersion = ConfigVersion 4 +configVersion = ConfigVersion 5 configIndex :: BH.IndexName configIndex = BH.IndexName "monocle.config" @@ -353,12 +367,19 @@ upgradeConfigV4 = do mkChangeBulkUpdate indexName change = BulkUpdate indexName (getChangeDocId change) $ toJSON change +upgradeConfigV5 :: forall es. MonoQuery :> es => IndexEffects es => Eff es () +upgradeConfigV5 = do + indexName <- getIndexName + logInfo "Applying migration to schema V5 on workspace" ["index" .= indexName] + void $ esPutMapping indexName mergedCommitField + upgrades :: forall es. (E.Fail :> es, MonoQuery :> es) => IndexEffects es => [(ConfigVersion, Eff es ())] upgrades = [ (ConfigVersion 1, upgradeConfigV1) , (ConfigVersion 2, upgradeConfigV2) , (ConfigVersion 3, void upgradeConfigV3) , (ConfigVersion 4, void upgradeConfigV4) + , (ConfigVersion 5, void upgradeConfigV5) ] newtype ConfigVersion = ConfigVersion Integer @@ -488,12 +509,14 @@ toEChangeEvent ChangePB.ChangeEvent {..} = , echangeeventTasksData = Nothing , echangeeventDuration = toDuration <$> changeEventOptionalDuration , echangeeventDraft = Just changeEventDraft + , echangeeventMergedCommitSha = toMergedCommitSha <$> changeEventOptionalMergedCommitSha } where author = toAuthor changeEventAuthor onAuthor = toAuthor changeEventOnAuthor eType = getEventType changeEventType toDuration (ChangePB.ChangeEventOptionalDurationDuration v) = fromInteger $ toInteger v + toMergedCommitSha (ChangePB.ChangeEventOptionalMergedCommitShaMergedCommitSha sha) = sha getEventType :: Maybe ChangePB.ChangeEventType -> EDocType getEventType eventTypeM = case eventTypeM of diff --git a/src/Monocle/Backend/Provisioner.hs b/src/Monocle/Backend/Provisioner.hs index f91d08c11..f1561725f 100644 --- a/src/Monocle/Backend/Provisioner.hs +++ b/src/Monocle/Backend/Provisioner.hs @@ -130,6 +130,7 @@ fakeChange from' to = do echangeCreatedAt <- dropTime <$> Faker.DateTime.utcBetween from' to echangeUpdatedAt <- dropTime <$> Faker.DateTime.utcBetween echangeCreatedAt to let echangeMergedBy = Nothing + let echangeMergedCommitSha = Nothing let echangeTargetBranch = "main" let echangeMergedAt = Nothing let echangeClosedAt = Nothing @@ -172,6 +173,7 @@ fakeChangeEvent from' to = do echangeeventLabels = Just [] echangeeventDuration = Nothing echangeeventDraft = Nothing + echangeeventMergedCommitSha = Nothing pure $ EChangeEvent {..} fakeTaskId :: Faker.Fake Text diff --git a/src/Monocle/Backend/Test.hs b/src/Monocle/Backend/Test.hs index 4910f7c07..d047030d5 100644 --- a/src/Monocle/Backend/Test.hs +++ b/src/Monocle/Backend/Test.hs @@ -67,6 +67,7 @@ fakeChangePB = , changeRepositoryShortname = mempty , changeAuthor = Nothing , changeOptionalMergedBy = Nothing + , changeOptionalMergedCommitSha = Nothing , changeBranch = mempty , changeTargetBranch = mempty , changeCreatedAt = Just (from fakeDate) @@ -107,6 +108,7 @@ fakeChange = , echangeCreatedAt = fakeDate , echangeUpdatedAt = fakeDate , echangeMergedBy = Nothing + , echangeMergedCommitSha = Nothing , echangeTargetBranch = "main" , echangeMergedAt = Nothing , echangeClosedAt = Nothing @@ -195,6 +197,10 @@ testIndexChanges = withTenant doTest (I.getChangeDocId fakeChange1Updated) echangeTitle (echangeTitle fakeChange1Updated) + checkEChangeField + (I.getChangeDocId fakeChange1Updated) + echangeMergedCommitSha + (echangeMergedCommitSha fakeChange1Updated) -- Check we can get the most recently updated change dateM <- I.getLastUpdatedDate $ from . echangeRepositoryFullname $ fakeChange1 assertEqual' "Got most recent change" (Just fakeDateAlt) dateM @@ -213,6 +219,7 @@ testIndexChanges = withTenant doTest fakeChange1 { echangeTitle = "My change 1 updated" , echangeUpdatedAt = fakeDateAlt + , echangeMergedCommitSha = Just "123" } fakeChange2 = mkFakeChange 2 "My change 2" mkFakeChange :: Int -> LText -> EChange @@ -1322,6 +1329,7 @@ emptyEvent = EChangeEvent {..} echangeeventLabels = mempty echangeeventDuration = Nothing echangeeventDraft = Nothing + echangeeventMergedCommitSha = Nothing showEvents :: [ScenarioEvent] -> Text showEvents xs = Text.intercalate ", " $ sort (map go xs) From 87b4080938eb87365d65ebaa316fbdef4d7c3ab5 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Mon, 30 Oct 2023 08:15:05 +0000 Subject: [PATCH 2/2] crawler - github - make the merged_commit_sha searcheable The previous commit sets the field as non indexed meaning that it was only stored an retreivable but not searchable. This commit changes this field setting as a standard keywork field to make it searcheable. --- src/Monocle/Backend/Index.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Monocle/Backend/Index.hs b/src/Monocle/Backend/Index.hs index bcab3d2e0..478538f9d 100644 --- a/src/Monocle/Backend/Index.hs +++ b/src/Monocle/Backend/Index.hs @@ -88,7 +88,7 @@ instance ToJSON NonIndexedMapping where ] mergedCommitField :: [Pair] -mergedCommitField = ["merged_commit_sha" .= NonIndexedMapping] +mergedCommitField = ["merged_commit_sha" .= KWMapping] data TextAndKWMapping = TextAndKWMapping deriving (Eq, Show)