Merge pull request #315 from cofacts/media-similarity

[3] Expose media similarity in ListArticles
cofacts · Sep 1, 2023 · 2d3dc48 · 2d3dc48
2 parents e8519e7 + a8cca5a
commit 2d3dc48
Show file tree

Hide file tree

Showing 11 changed files with 77 additions and 12 deletions.
diff --git a/src/__tests__/__snapshots__/auth.js.snap b/src/__tests__/__snapshots__/auth.js.snap
@@ -3,6 +3,7 @@
 exports[`verifyProfile authenticates user via profile ID 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": 0.2876821,
   "email": "[email protected]",
   "facebookId": "secret-fb-id",
@@ -16,6 +17,7 @@ Object {
 exports[`verifyProfile authenticates user via same email of existing user; also updates profile ID 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "email": "[email protected]",
   "facebookId": "secret-fb-id",
@@ -31,6 +33,7 @@ Object {
 exports[`verifyProfile creates new user if user does not exist 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "avatarUrl": "url-to-photo",
   "createdAt": "1989-06-04T00:00:00.000Z",

diff --git a/src/graphql/dataLoaders/__tests__/__snapshots__/userLoaderFactory.js.snap b/src/graphql/dataLoaders/__tests__/__snapshots__/userLoaderFactory.js.snap
@@ -3,6 +3,7 @@
 exports[`userLoaderFactory should return correct user with given slug 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": 0.6931472,
   "email": "[email protected]",
   "highlight": undefined,
@@ -16,6 +17,7 @@ Object {
 exports[`userLoaderFactory should return correct user with given slug 2`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": 0.6931472,
   "email": "[email protected]",
   "highlight": undefined,

diff --git a/src/graphql/models/Article.js b/src/graphql/models/Article.js
@@ -7,6 +7,7 @@ import {
   GraphQLInt,
   GraphQLBoolean,
   GraphQLEnumType,
+  GraphQLFloat,
 } from 'graphql';
 
 import {
@@ -548,7 +549,17 @@ const Article = new GraphQLObjectType({
 
 export const ArticleConnection = createConnectionType(
   'ArticleConnection',
-  Article
+  Article,
+  {
+    extraEdgeFields: {
+      mediaSimilarity: {
+        type: new GraphQLNonNull(GraphQLFloat),
+        description: `The search hit's similarity with provided mediaUrl.
+          Ranges from 0 to 1. 0 if mediaUrl is not provided, or the hit is not matched by mediaUrl.`,
+        resolve: ({ node }) => node._fields?.mediaSimilarity?.[0] ?? 0,
+      },
+    },
+  }
 );
 
 export default Article;
diff --git a/src/graphql/models/__tests__/__snapshots__/User.js.snap b/src/graphql/models/__tests__/__snapshots__/User.js.snap
@@ -3,6 +3,7 @@
 exports[`User model userFieldResolver returns the right backend user given appUserId 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "TEST_BACKEND",
   "appUserId": "userTest1",
@@ -16,6 +17,7 @@ Object {
 exports[`User model userFieldResolver returns the right backend user given db userId 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "TEST_BACKEND",
   "appUserId": "userTest1",
@@ -29,6 +31,7 @@ Object {
 exports[`User model userFieldResolver returns the right web user given userId 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "WEBSITE",
   "highlight": undefined,

diff --git a/src/graphql/queries/ListArticles.js b/src/graphql/queries/ListArticles.js
@@ -298,6 +298,14 @@ export default {
     }
 
     const body = {
+      // Source fields are disabled when `script_fields` presets.
+      // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/search-request-source-filtering.html
+      //      https://discuss.elastic.co/t/script-field-along-with-all-the-other-fields-using-painless/296384/2
+      // We need to turn _source back on.
+      //
+      _source: true,
+      script_fields: {},
+
       sort: getSortArgs(orderBy, {
         replyCount: o => ({ normalArticleReplyCount: { order: o } }),
         lastRepliedAt: o => ({
@@ -580,6 +588,15 @@ export default {
         return map;
       }, {});
 
+      body.script_fields.mediaSimilarity = {
+        script: {
+          lang: 'painless',
+          // Note: params.similarityMap.get(doc['attachmentHash'].value) may be null if no attahmentHash, or when attahmentHash not in similarityMap.
+          source: `doc.containsKey('attachmentHash') ? params.similarityMap.get(doc['attachmentHash'].value) : null`,
+          params: { similarityMap },
+        },
+      };
+
       // Make media search dominant text search
       const MULTIPLIER = 100;
 
@@ -596,8 +613,12 @@ export default {
           script_score: {
             script: {
               lang: 'painless',
-              params: { similarityMap },
+              // Every hit in this query is inside similarityMap, no need for null handling.
+              //
+              // `mediaSimilarity` cannot be used here because it only exists after search complete.
+              //
               source: `${MULTIPLIER} * params.similarityMap.get(doc['attachmentHash'].value)`,
+              params: { similarityMap },
             },
           },
         },

diff --git a/src/graphql/queries/__tests__/ListArticles.js b/src/graphql/queries/__tests__/ListArticles.js
@@ -990,6 +990,7 @@ describe('ListArticles', () => {
           ) {
             edges {
               score
+              mediaSimilarity
               node {
                 id
                 articleType
@@ -1006,6 +1007,7 @@ describe('ListArticles', () => {
           "ListArticles": Object {
             "edges": Array [
               Object {
+                "mediaSimilarity": 1,
                 "node": Object {
                   "articleType": "IMAGE",
                   "attachmentHash": "ffff8000",
@@ -1015,6 +1017,7 @@ describe('ListArticles', () => {
                 "score": 100,
               },
               Object {
+                "mediaSimilarity": 0.5,
                 "node": Object {
                   "articleType": "IMAGE",
                   "attachmentHash": "ffff8001",
@@ -1024,6 +1027,7 @@ describe('ListArticles', () => {
                 "score": 60.03248,
               },
               Object {
+                "mediaSimilarity": 0,
                 "node": Object {
                   "articleType": "TEXT",
                   "attachmentHash": "",
@@ -1139,6 +1143,7 @@ describe('ListArticles', () => {
           ) {
             edges {
               score
+              mediaSimilarity
               node {
                 id
                 articleType
@@ -1164,6 +1169,7 @@ describe('ListArticles', () => {
             <HIGHLIGHT>微之</HIGHLIGHT>，<HIGHLIGHT>微之</HIGHLIGHT>！<HIGHLIGHT>此夕此心</HIGHLIGHT>，<HIGHLIGHT>君知之乎</HIGHLIGHT>！
           ",
                 },
+                "mediaSimilarity": 0,
                 "node": Object {
                   "articleType": "TEXT",
                   "attachmentHash": "",

diff --git a/src/graphql/util.js b/src/graphql/util.js
@@ -222,16 +222,18 @@ async function defaultResolveTotalCount({
   after, // eslint-disable-line no-unused-vars
   ...searchContext
 }) {
-  return (await client.count({
-    ...searchContext,
-    body: {
-      ...searchContext.body,
-
-      // totalCount cannot support these
-      sort: undefined,
-      track_scores: undefined,
-    },
-  })).body.count;
+  try {
+    return (await client.count({
+      ...searchContext,
+      body: {
+        // count API only supports "query"
+        query: searchContext.body.query,
+      },
+    })).body.count;
+  } catch (e) /* istanbul ignore next */ {
+    console.error('[defaultResolveTotalCount]', JSON.stringify(e));
+    throw e;
+  }
 }
 
 export async function defaultResolveEdges(
@@ -368,6 +370,7 @@ export function createConnectionType(
     resolveLastCursor = defaultResolveLastCursor,
     resolveFirstCursor = defaultResolveFirstCursor,
     resolveHighlights = defaultResolveHighlights,
+    extraEdgeFields = {},
   } = {}
 ) {
   return new GraphQLObjectType({
@@ -395,6 +398,7 @@ export function createConnectionType(
                     type: Highlights,
                     resolve: resolveHighlights,
                   },
+                  ...extraEdgeFields,
                 },
               })
             )

diff --git a/src/scripts/__tests__/__snapshots__/cleanupUrls.js.snap b/src/scripts/__tests__/__snapshots__/cleanupUrls.js.snap
@@ -4,13 +4,15 @@ exports[`should clean up urls 1`] = `
 Array [
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "new-url",
     "inner_hits": undefined,
   },
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "url-processed",
@@ -19,6 +21,7 @@ Array [
   },
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "url-foo",
@@ -27,6 +30,7 @@ Array [
   },
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "curl-foo",
@@ -35,6 +39,7 @@ Array [
   },
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "url-bar",
@@ -43,6 +48,7 @@ Array [
   },
   Object {
     "_cursor": undefined,
+    "_fields": undefined,
     "_score": 1,
     "highlight": undefined,
     "id": "curl-bar",

diff --git a/src/util/__tests__/__snapshots__/scrapUrls.js.snap b/src/util/__tests__/__snapshots__/scrapUrls.js.snap
@@ -6,6 +6,7 @@ Array [
     "_cursor": Array [
       1485907200000,
     ],
+    "_fields": undefined,
     "_score": null,
     "canonical": "http://example.com/article/1111",
     "fetchedAt": "2017-02-01T00:00:00.000Z",
@@ -22,6 +23,7 @@ Array [
     "_cursor": Array [
       1485907200000,
     ],
+    "_fields": undefined,
     "_score": null,
     "canonical": "http://example.com/article/1111",
     "fetchedAt": "2017-02-01T00:00:00.000Z",

diff --git a/src/util/__tests__/__snapshots__/user.js.snap b/src/util/__tests__/__snapshots__/user.js.snap
@@ -3,6 +3,7 @@
 exports[`user utils CreateOrUpdateUser creates backend user if not existed 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "TEST_BACKEND",
   "appUserId": "testUser2",
@@ -34,6 +35,7 @@ Object {
 exports[`user utils CreateOrUpdateUser fills appId for website users 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "WEBSITE",
   "highlight": undefined,
@@ -47,6 +49,7 @@ Object {
 exports[`user utils CreateOrUpdateUser logs error if collision occurs 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "TEST_BACKEND",
   "appUserId": "testUser1",
@@ -78,6 +81,7 @@ Array [
 exports[`user utils CreateOrUpdateUser updates backend users' last active time if user already existed 1`] = `
 Object {
   "_cursor": undefined,
+  "_fields": undefined,
   "_score": undefined,
   "appId": "TEST_BACKEND",
   "appUserId": "testUser1",

diff --git a/src/util/client.js b/src/util/client.js
@@ -19,6 +19,8 @@ export function processMeta({
   inner_hits, // nested query search result highlighting.
 
   sort, // cursor when sorted
+
+  fields, // scripted fields (if any)
 }) {
   if (found || _score !== undefined) {
     return {
@@ -28,6 +30,7 @@ export function processMeta({
       _score,
       highlight,
       inner_hits,
+      _fields: fields,
     };
   }
   return null; // not found