Merge pull request #3802 from thematters/feat/remove-unused-fields-of…

…-article-table

feat: remove unused columns from article and draft table

db/migrations/20240130120100_alter_article_and_draft_drop_fields.js

@@ -0,0 +1,35 @@
+exports.up = async (knex) => {
+  await knex.schema.table('article', (t) => {
+    t.dropColumn('uuid')
+    t.dropColumn('title')
+    t.dropColumn('slug')
+    t.dropColumn('cover')
+    t.dropColumn('upstream_id')
+    t.dropColumn('live')
+    t.dropColumn('public')
+    t.dropColumn('content')
+    t.dropColumn('summary')
+    t.dropColumn('language')
+    t.dropColumn('data_hash')
+    t.dropColumn('media_hash')
+    t.dropColumn('iscn_id')
+    t.dropColumn('draft_id')
+  })
+
+  await knex.schema.table('draft', (t) => {
+    t.dropColumn('uuid')
+    t.dropColumn('summary_customized')
+    t.dropColumn('word_count')
+    t.dropColumn('language')
+    t.dropColumn('data_hash')
+    t.dropColumn('media_hash')
+    t.dropColumn('prev_draft_id')
+    t.dropColumn('iscn_id')
+    t.dropColumn('pin_state')
+    t.dropColumn('sensitive_by_admin')
+  })
+}
+
+exports.down = async () => {
+  // do nothing
+}

db/sql/author-tags-create-table-view.sql

@@ -24,7 +24,6 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
   WITH author_articles AS (
     SELECT author_id,
       count(*) ::int AS num_articles,
-      -- (array_agg(concat(id, '-', slug, '-', media_hash) ORDER BY updated_at DESC))[1:5] AS last_5,
       sum(word_count) ::int AS sum_word_count,
       max(updated_at) AS last_at
     FROM public.article
@@ -44,19 +43,19 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         ) ORDER BY month DESC)) -- [1:18]
       ) AS stats
     FROM (
-      SELECT author_id, date_trunc('month', created_at) ::date AS month,
+      SELECT author_id, date_trunc('month', a.created_at) ::date AS month,
         MAX(created_at) AS month_last,
         COUNT(a.id) ::int AS num_articles, -- COUNT(DISTINCT author_id) ::int AS num_authors
         sum(a.word_count) ::int AS sum_word_count,
         (ARRAY_AGG(jsonb_build_object(
-              'title', a.title,
-              'path', concat(a.id, '-', a.slug, '-', a.media_hash),
+              'title', avn.title,
+              'path', concat(a.id),
               'num_apprtors', num_apprtors,
               'sum_appreciations', sum_appreciations
           ) ORDER BY a.created_at DESC))[1:5] AS last_5
-      FROM articles_appr JOIN public.article a USING(id)
-      -- LEFT JOIN appreciation appr ON appr.reference_id = a.id
-      -- WHERE a.state IN ('active') -- NOT IN ('archived', 'banned')
+      FROM articles_appr
+      JOIN public.article a USING(id)
+      JOIN public.article_version_newest avn ON a.id=avn.article_id
       GROUP BY 1, 2
     ) st
     GROUP BY 1
@@ -73,12 +72,14 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         COUNT(*) ::int AS num_articles,
         sum(a.word_count) ::int AS sum_word_count,
         (ARRAY_AGG(jsonb_build_object(
-              'title', a.title, 'date', a.created_at ::date,
-              'path', concat(a.id, '-', a.slug, '-', a.media_hash),
+              'title', avn.title, 'date', a.created_at ::date,
+              'path', concat(a.id),
               'num_apprtors', num_apprtors,
               'sum_appreciations', sum_appreciations
           ) ORDER BY a.created_at DESC))[1:5] AS last_5
-      FROM public.article a JOIN articles_appr USING(id)
+      FROM public.article a
+      JOIN articles_appr USING(id)
+      JOIN public.article_version_newest avn ON a.id=avn.article_id
       LEFT JOIN article_tag at ON article_id=a.id -- AND a.state IN ('active') -- NOT IN ('archived', 'banned')
       GROUP BY 1, 2
     ) t LEFT JOIN tag ON tag_id=tag.id
@@ -97,8 +98,8 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
       SELECT a.author_id,
         -- (array_agg(a.id || '-' || slug || '-' || media_hash ORDER BY num_readers DESC, sum_read_time DESC))[1:5] AS top_5
         to_jsonb(array_agg(DISTINCT jsonb_build_object(
-              'title', a.title,
-              'path', concat(a.id, '-', slug, '-', media_hash),
+              'title', avn.title,
+              'path', concat(a.id),
               'num_readers', num_readers, 'num_readers_w3m', num_readers_w3m,
               'sum_read_time', sum_read_time, 'last_at', last_at
             ) -- ORDER BY num_readers DESC, sum_read_time DESC
@@ -115,7 +116,9 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         FROM article_read_count
         WHERE user_id IS NOT NULL
         GROUP BY 1
-      ) r JOIN public.article a ON r.article_id = a.id AND a.state IN ('active') -- NOT IN ('archived', 'banned')
+      ) r
+      JOIN public.article a ON r.article_id = a.id AND a.state IN ('active') -- NOT IN ('archived', 'banned')
+      JOIN public.article_version_newest avn ON a.id=avn.article_id
       GROUP BY 1
     ) aa
     JOIN (

db/sql/create-table-search-index-article.sql

@@ -4,14 +4,16 @@ DROP TABLE IF EXISTS search_index.article ;
 
 CREATE TABLE IF NOT EXISTS search_index.article AS
 SELECT * FROM (
-  SELECT DISTINCT ON (draft.article_id) article.id,
-    draft.article_id, draft_id, draft.author_id, draft.title AS title_orig, draft.title AS title, -- to be processed by opencc in JS
-    article.slug, draft.summary, draft.content AS content_orig,
-    draft.content AS content, '' AS text_content_orig, '' AS text_content, '' AS text_content_converted, -- to be processed by opencc in JS
-    draft.created_at, article.state, draft.publish_state
-  FROM draft JOIN article ON article_id=article.id AND article_id IS NOT NULL
-  WHERE state='active' AND publish_state='published'
-  ORDER BY draft.article_id DESC NULLS LAST
+  SELECT DISTINCT ON (article_version.article_id) article.id,
+    article_version.article_id, article.author_id, article_version.title AS title_orig, article_version.title AS title, -- to be processed by opencc in JS
+    '' as slug, article_version.summary, article_content.content AS content_orig,
+    article_content.content AS content, '' AS text_content_orig, '' AS text_content, '' AS text_content_converted, -- to be processed by opencc in JS
+    article_version.created_at, article.state
+  FROM public.article_version
+    JOIN public.article ON article_id=article.id AND article_id IS NOT NULL
+    JOIN public.article_content ON content_id=article_content.id
+  WHERE state='active'
+  ORDER BY article_version.article_id DESC NULLS LAST
 ) a
 LEFT JOIN (
   SELECT article_id, COUNT(*) ::int AS num_views, MAX(created_at) AS last_read_at

db/sql/stale-circles-create-table-view.sql

@@ -24,13 +24,13 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
   WITH uniq_articles AS (
     SELECT DISTINCT article_id FROM article_circle
   ), circle_articles AS (
-    SELECT circle_id, a.id, a.author_id, a.title, a.slug, a.word_count, a.data_hash, a.media_hash, ac.created_at, ac.updated_at
+    SELECT circle_id, a.id, a.author_id, avn.title, '' as slug, avn.word_count, avn.data_hash, avn.media_hash, ac.created_at, ac.updated_at
     FROM article_circle ac
     JOIN public.article a ON ac.article_id=a.id AND a.state IN ('active')
+    JOIN public.article_version_newest avn ON a.id=avn.article_id
   ), circle_articles_stats AS (
     SELECT circle_id,
       count(*) ::int AS num_articles,
-      -- (array_agg(concat(id, '-', slug, '-', media_hash) ORDER BY updated_at DESC))[1:5] AS last_5,
       sum(word_count) ::int AS sum_word_count,
       max(updated_at) AS last_at
     FROM circle_articles -- article_circle ac LEFT JOIN public.article a ON ac.article_id=a.id
@@ -75,13 +75,14 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         COUNT(a.id) ::int AS num_articles, -- COUNT(DISTINCT author_id) ::int AS num_authors
         SUM(a.word_count) ::int AS sum_word_count,
         (ARRAY_AGG(jsonb_build_object(
-              'title', a.title,
-              'path', concat(a.id, '-', a.slug, '-', a.media_hash),
+              'title', avn.title,
+              'path', concat(a.id),
               'num_apprtors', num_apprtors,
               'sum_appreciations', sum_appreciations
         ) ORDER BY a.created_at DESC))[1:5] AS last_5
       FROM articles_appr -- JOIN ( SELECT circle_id, a.* FROM circle_article JOIN public.article a ON article_id=a.id)
       JOIN circle_articles a USING(id)
+      JOIN article_version_newest avn ON a.id=avn.article_id
       -- LEFT JOIN appreciation appr ON appr.reference_id = a.id
       -- WHERE a.state IN ('active') -- NOT IN ('archived', 'banned')
       GROUP BY 1, 2
@@ -100,13 +101,14 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         COUNT(*) ::int AS num_articles,
         sum(a.word_count) ::int AS sum_word_count,
         (ARRAY_AGG(jsonb_build_object(
-              'title', a.title, 'date', a.created_at ::date,
-              'path', concat(a.id, '-', a.slug, '-', a.media_hash),
+              'title', avn.title, 'date', a.created_at ::date,
+              'path', concat(a.id),
               'num_apprtors', num_apprtors,
               'sum_appreciations', sum_appreciations
           ) ORDER BY a.created_at DESC))[1:5] AS last_5
       FROM circle_articles a -- article_circle ac LEFT JOIN public.article a ON ac.article_id=a.id
       JOIN articles_appr USING(id)
+      JOIN article_version_newest avn ON a.id=avn.article_id
       LEFT JOIN article_tag at ON article_id=a.id -- AND a.state IN ('active') -- NOT IN ('archived', 'banned')
       GROUP BY 1, 2
     ) t LEFT JOIN tag ON tag_id=tag.id
@@ -124,8 +126,8 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         ORDER BY num_readers_w3m DESC, num_readers DESC, sum_read_time DESC LIMIT 5 )) AS top_5
     FROM ( SELECT circle_id,
       to_jsonb(array_agg(DISTINCT jsonb_build_object(
-            'title', a.title,
-            'path', a.id || '-' || slug || '-' || media_hash,
+            'title', avn.title,
+            'path', a.id,
             'num_readers', num_readers, 'num_readers_w3m', num_readers_w3m,
             'sum_read_time', sum_read_time, 'last_at', last_at
           ) -- ORDER BY num_readers DESC, sum_read_time DESC
@@ -145,6 +147,7 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
           AND article_id IN (SELECT article_id FROM uniq_articles)
         GROUP BY 1
       ) r JOIN circle_articles a ON r.article_id = a.id -- AND a.state IN ('active') -- NOT IN ('archived', 'banned')
+      JOIN article_version_newest avn ON r.article_id=avn.article_id
       GROUP BY 1
     ) aa
     JOIN (

db/sql/stale-tags-create-table-view.sql

@@ -9,22 +9,22 @@ CREATE OR REPLACE FUNCTION pg_temp.slug(input text) RETURNS text AS $f$
 $f$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT;
 
 CREATE OR REPLACE FUNCTION pg_temp.array_distinct(
-  anyarray, -- input array 
+  anyarray, -- input array
   boolean DEFAULT false -- flag to ignore nulls
 ) RETURNS anyarray AS $f$
-  SELECT array_agg(DISTINCT x) 
-  FROM unnest($1) t(x) 
+  SELECT array_agg(DISTINCT x)
+  FROM unnest($1) t(x)
   WHERE CASE WHEN $2 THEN x IS NOT NULL ELSE true END;
 $f$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT;
 
 CREATE OR REPLACE FUNCTION pg_temp.array_uniq_stable(anyarray) RETURNS anyarray AS $f$
 SELECT
     array_agg(distinct_value ORDER BY first_index)
-FROM 
+FROM
     (SELECT
-        value AS distinct_value, 
-        min(index) AS first_index 
-    FROM 
+        value AS distinct_value,
+        min(index) AS first_index
+    FROM
         unnest($1) WITH ORDINALITY AS input(value, index)
     GROUP BY
         value
@@ -238,7 +238,7 @@ WITH article_tag_stats_by_slug AS (
     SELECT t.slug, author_id, -- date_trunc('month', at.created_at) ::date AS month,
       MAX(at.created_at) AS last_use,
       COUNT(DISTINCT article_id) ::int AS num_articles, -- , COUNT(*) ::int -- COUNT(DISTINCT author_id) ::int AS num_authors
-      (ARRAY_AGG(concat(a.id, '-', a.slug, '-', a.media_hash) ORDER BY a.created_at DESC))[1:5] AS last_5
+      (ARRAY_AGG(concat(a.id) ORDER BY a.created_at DESC))[1:5] AS last_5
     FROM public.article_tag at JOIN public.article a ON article_id=a.id AND a.state IN ('active')
     JOIN pg_temp.tag t ON tag_id=t.id
     -- WHERE at.created_at >= date_trunc('month', CURRENT_DATE - '18 months'::interval)
@@ -297,10 +297,11 @@ SELECT slug,
   -- SELECT t.id, asset.path AS cover
   -- FROM (
     SELECT id, COALESCE(t.cover,
-      (SELECT article.cover
+      (SELECT avn.cover
         FROM public.article_tag at
         LEFT JOIN public.article ON at.article_id=article.id AND article.state IN ('active') -- NOT IN ('archived', 'banned')
-        WHERE at.tag_id = t.id AND article.cover IS NOT NULL
+        LEFT JOIN public.article_version_newest avn ON at.article_id=avn.article_id
+        WHERE at.tag_id = t.id AND avn.cover IS NOT NULL
         ORDER BY at.id ASC
         LIMIT 1 -- find the earliest one article with cover, re-use as tagCover
       )

db/sql/stale-users-create-table-view.sql

@@ -48,7 +48,7 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         MAX(created_at) AS month_last,
         COUNT(id) ::int AS num_articles, -- COUNT(DISTINCT author_id) ::int AS num_authors
         sum(word_count) ::int AS sum_word_count,
-        (ARRAY_AGG(concat(id, '-', slug, '-', media_hash)))[1:5] AS last_5
+        (ARRAY_AGG(concat(id)))[1:5] AS last_5
       FROM article
       -- WHERE created_at >= date_trunc('month', CURRENT_DATE - '18 months'::interval)
       WHERE state IN ('active') -- NOT IN ('archived')
@@ -114,7 +114,7 @@ EXPLAIN (ANALYZE, BUFFERS, VERBOSE) CREATE TABLE :schema.:tablename AS
         COUNT(DISTINCT a.author_id) ::int AS num_commented_authors,
         -- SUM((SELECT COUNT(*) FROM regexp_matches(regexp_replace(c.content, '<[^\>]+>', '', 'g'), '\w', 'g'))) ::int AS sum_word_count,
         -- SUM((SELECT COUNT(*) FROM regexp_matches(regexp_replace(c.content, '<[^\>]+>', '', 'g'), '\w', 'g'))) ::int AS sum_word_count,
-        (ARRAY_AGG(concat(a.id, '-', a.slug, '-', a.media_hash, '#',
+        (ARRAY_AGG(concat(a.id, '#',
               CASE WHEN c.parent_comment_id IS NOT NULL THEN pg_temp.global_id('Comment', c.parent_comment_id ::int) || '-' ELSE '' END,
               pg_temp.global_id('Comment', c.id ::int) ))
         )[1:5] AS last_5

src/connectors/collectionService.ts

@@ -100,7 +100,6 @@ export class CollectionService extends BaseService<Collection> {
     const records = await this.knex('collection_article')
       .select(
         'article_id',
-        'draft_id',
         'order',
         this.knex.raw('count(1) OVER() AS total_count')
       )

src/definitions/collection.d.ts

@@ -12,6 +12,5 @@ export interface Collection {
 
 export interface CollectionArticle {
   articleId: string
-  draftId: string
   order: string
 }

src/mutations/draft/putDraft.ts

@@ -159,16 +159,14 @@ const resolver: GQLMutationResolvers['putDraft'] = async (
   }
 
   // assemble data
-  const resetSummary = summary === null || summary === ''
   const resetCover = cover === null
   const resetCircle = circleGlobalId === null
 
   const data: ItemData = omitBy(
     {
       authorId: id ? undefined : viewer.id,
       title: title?.trim(),
-      summary: summary?.trim(),
-      summaryCustomized: summary === undefined ? undefined : !resetSummary,
+      summary: summary === null ? null : summary?.trim(),
       content: content && normalizeArticleHTML(sanitizeHTML(content)),
       tags: tags?.length === 0 ? null : tags,
       cover: coverId,
@@ -282,7 +280,6 @@ const resolver: GQLMutationResolvers['putDraft'] = async (
     return draftService.baseUpdate(dbId, {
       ...data,
       // reset fields
-      summary: resetSummary ? null : data.summary,
       circleId: resetCircle ? null : data.circleId,
     })
   }

src/queries/draft/index.ts

@@ -31,6 +31,7 @@ const schema: GQLResolvers = {
     mediaHash: ({ mediaHash }) => mediaHash ?? '',
     wordCount: ({ content }) => (content ? countWords(content) : 0),
     summary: ({ summary, content }) => summary || makeSummary(content || ''),
+    summaryCustomized: ({ summary }) => !!summary,
     content: draftContent,
     cover: draftCover,
     collection,

src/queries/user/collection/articles.ts

@@ -18,7 +18,7 @@ const resolver: GQLCollectionResolvers['articles'] = async (
   const { skip, take } = fromConnectionArgs({ first, after })
 
   if (take === 0) {
-    const [_, count] = await collectionService.findAndCountArticlesInCollection(
+    const [, count] = await collectionService.findAndCountArticlesInCollection(
       collectionId,
       {
         skip,