fetchSymbolsAndAttributesGlean can query a different backend

Summary:
When computing snapshots, we need all dependent files to be indexed in order to have the xrefs. This is true for xlang navigation as well. In order to get the thrift xrefs from hack/c++, we would need to index thrift files as well.

At diff-time we index  different languages in different workflows, and the API doesn't allow to run both hack/thrift in the same workflow at the moment.

Another solution is for the snapshot generator to query an external db.  This is simpler to implement and also won't occur extra latency.

Downside is that if a diff adds thrift entities and hack/cpp client code, we may miss some xlang xrefs since the thrift symbol won't exist in the thrift db and we won't be able to resolve their location. I think it's a rare occurrence.

This diffs adds the ability for the document symbol handler to query a different backend for resolving xlang refs. This new path isn't used in Glass, but is exercised by the snapshot tool.

Reviewed By: simonmar

Differential Revision: D63459934

fbshipit-source-id: 4e588fde9a4d4f26c5efc35450c660c3e21c1e22

glean/glass/Glean/Glass/Handler.hs

@@ -15,6 +15,7 @@ module Glean.Glass.Handler
   (
   -- * listing symbols by file
     documentSymbolListX
+  , documentSymbolListXSnapshot
   , documentSymbolIndex
 
   -- ** find references
@@ -164,13 +165,38 @@ documentSymbolListX env r opts = do
       (if useSnapshots
         then fetchSymbolsAndAttributes env
         else (\dbInfo req opts be _ mlang -> do
-                ((res, log), err) <-
-                  fetchSymbolsAndAttributesGlean env dbInfo req opts be mlang
+                ((res, log), err) <- fetchSymbolsAndAttributesGlean
+                  env dbInfo req opts be Nothing mlang
                 return ((res, Snapshot.Unrequested, log), err)
               )
       )
       env r opts
 
+-- | Variation of documentSymbolListX
+-- Always use Glean (and not XDB snapshot backend).
+-- Possibly use a different backend o resolve xlang
+-- entities to their location.
+--
+-- Use case: constructing document symbols lists
+-- snapshots from local dbs, and resolving xlang
+-- entities location from global dbs not locally indexed.
+documentSymbolListXSnapshot
+  :: Glass.Env
+  -> DocumentSymbolsRequest
+  -> RequestOptions
+  -> Maybe (Some Glean.Backend, GleanDBInfo)
+  -> IO DocumentSymbolListXResult
+documentSymbolListXSnapshot env  r opts mGleanBe = do
+  fst3 <$>
+    runRepoFile
+      "documentSymbolListXSnapshot"
+      (\dbInfo req opts be _ mlang -> do
+          ((res, log), err) <- fetchSymbolsAndAttributesGlean
+            env dbInfo req opts be mGleanBe mlang
+          return ((res, Snapshot.Unrequested, log), err)
+          )
+      env r opts
+
 -- | Same as documentSymbolList() but construct a line-indexed map for easy
 -- cursor/position lookup, and add extra metadata
 documentSymbolIndex
@@ -695,20 +721,25 @@ fetchSymbolsAndAttributesGlean
   -> DocumentSymbolsRequest
   -> RequestOptions
   -> GleanBackend b
+  -> Maybe (Some Glean.Backend, GleanDBInfo)
   -> Maybe Language
   -> IO (
        (DocumentSymbolListXResult, QueryEachRepoLog),
        Maybe ErrorLogger
      )
-fetchSymbolsAndAttributesGlean env@Glass.Env{..} dbInfo req opts be mlang = do
+fetchSymbolsAndAttributesGlean
+  env@Glass.Env{tracer, gleanBackend}
+  dbInfo req opts be mOtherBackend mlang = do
   (res1, gLogs, elogs) <- traceSpan tracer "fetchDocumentSymbols" $
     fetchDocumentSymbols env file path mlimit
       (requestOptions_revision opts)
       (requestOptions_exact_revision opts)
       includeRefs includeXlangRefs
       (shouldFetchContentHash opts) be mlang dbInfo
 
-  res2 <- resolveIdlXrefs env res1 repo be dbInfo
+  let be = fromMaybe (gleanBackend, dbInfo) mOtherBackend
+
+  res2 <- resolveIdlXrefs env res1 repo be
 
   let res3 = toDocumentSymbolResult res2
   return ((res3, gLogs), elogs)
@@ -900,7 +931,7 @@ fetchSymbolsAndAttributes [email protected]{..} dbInfo req
 
     getFromGlean =
       Glass.withAllocationLimit env $
-        fetchSymbolsAndAttributesGlean env dbInfo req opts be mlang
+        fetchSymbolsAndAttributesGlean env dbInfo req opts be Nothing mlang
 
 xRefDataToRefEntitySymbol :: XRefData -> (Code.Entity, ReferenceRangeSymbolX)
 xRefDataToRefEntitySymbol XRefData{..} = (xrefEntity, xrefSymbol)
@@ -1028,33 +1059,36 @@ fetchDocumentSymbols [email protected]{..} (FileReference scsrepo path)
           in (locationRange_repository, locationRange_filepath)
       )) xrefs
 
--- Idl xrefs needs db determination and range resolution
+-- | Idl xrefs needs db determination and range resolution
+--   possibly using a separate backend than the one computed
+--   from the origin query.
+--   Always choose latest db, as exactRevision can't usually be
+--   enforced for xlang refs
 resolveIdlXrefs
   :: Glean.Backend b
   => Glass.Env
   -> DocumentSymbols
   -> RepoName
-  -> GleanBackend b
-  -> GleanDBInfo
+  -> (b, GleanDBInfo)
   -> IO DocumentSymbols
-resolveIdlXrefs env docSyms@DocumentSymbols{..} scsrepo b dbInfo = do
+resolveIdlXrefs
+  env@Glass.Env{tracer, sourceControl, repoMapping}
+  docSyms@DocumentSymbols{..}
+  scsrepo
+  (gleanBackend, dbInfo) = do
   case (unresolvedXrefsIdl, srcFile) of
     ((ent, _) : _, Just srcFile) -> do
-      backendRunHaxl b env $ do
-        -- we assume all idl xrefs belong to the same db
-        let lang = entityLanguage ent
-        xrefsIdl <- case getLatestRepo
-          (Glass.repoMapping env) dbInfo scsrepo lang of
-            --  we look only in the first repo available, this assumes
-            --  the Idl repo comes first in the repomapping
-            Just idlRepo -> do
-              xrefs <- withRepo idlRepo $
-                resolveEntitiesRange scsrepo fst unresolvedXrefsIdl
-              withRepo idlRepo $
-                mapM (toReferenceSymbolIdl scsrepo srcFile offsets lang) xrefs
-            _ -> return []
-        let idlRefs = xRefDataToRefEntitySymbol <$> xrefsIdl
-        return $ docSyms  { refs = refs ++ idlRefs, unresolvedXrefsIdl = [] }
+       -- we assume all idl xrefs belong to the same db
+      let lang = entityLanguage ent
+      gleanDBs <- getGleanRepos tracer sourceControl repoMapping dbInfo
+        scsrepo (Just lang) ChooseLatest Nothing
+      let gleanBe = GleanBackend {gleanDBs, tracer, gleanBackend}
+      idlRefs <- backendRunHaxl gleanBe env $ do
+        xrefsIdl <- withRepo (snd (NonEmpty.head gleanDBs)) $ do
+          xrefs <- resolveEntitiesRange scsrepo fst unresolvedXrefsIdl
+          mapM (toReferenceSymbolIdl scsrepo srcFile offsets lang) xrefs
+        return $ xRefDataToRefEntitySymbol <$> xrefsIdl
+      return $ docSyms  { refs = refs ++ idlRefs, unresolvedXrefsIdl = [] }
     _ -> return docSyms
 
 -- | Wrapper for tracking symbol/entity pairs through processing

glean/glass/Glean/Glass/Handler/Utils.hs

@@ -19,6 +19,7 @@ module Glean.Glass.Handler.Utils (
     -- * Utils for building handlers
     GleanBackend(..),
     backendRunHaxl,
+    getGleanRepos,
 
     revisionSpecifierError,
 

glean/glass/Glean/Glass/Repos.hs

@@ -36,7 +36,6 @@ module Glean.Glass.Repos
   , getRepoHash
   , getRepoHashForLocation
   , getDBRevision
-  , getLatestRepo
   ) where
 
 import Control.Concurrent.Stream
@@ -545,13 +544,3 @@ getDBRevision scmRevs gleanDB repo =
   fromMaybe (getRepoHash gleanDB) $ do
     scmRepoToHash <- HashMap.lookup gleanDB scmRevs
     scmRevision <$> HashMap.lookup repo scmRepoToHash
-
-getLatestRepo
-  :: RepoMapping -> GleanDBInfo -> RepoName -> Language -> Maybe Glean.Repo
-getLatestRepo repoMapping dbInfo repoName lang =
-  let GleanDBInfo{
-      latestRepos = Glean.LatestRepos { latestRepos = repos }
-    } = dbInfo in
-  case fromSCSRepo repoMapping repoName (Just lang) of
-    [] -> Nothing
-    GleanDBName repo : _ -> Map.lookup repo repos