Web production deployment

packages/api/src/jobs/export.ts

@@ -127,6 +127,10 @@ const uploadToBucket = async (
 
 export const exportJob = async (jobData: ExportJobData) => {
   const { userId, exportId } = jobData
+  logger.info('starting export job', {
+    userId,
+    exportId,
+  })
 
   try {
     const user = await findActiveUser(userId)

packages/api/src/jobs/process-youtube-video.ts

@@ -281,24 +281,24 @@ export const processYouTubeVideo = async (
     updatedLibraryItem.publishedAt = new Date(video.uploadDate)
   }
 
-  if ('getTranscript' in video && duration > 0 && duration < 1801) {
-    // If the video has a transcript available, put a placehold in and
-    // enqueue a job to process the full transcript
-    const updatedContent = await addTranscriptToReadableContent(
-      libraryItem.originalUrl,
-      libraryItem.readableContent,
-      TRANSCRIPT_PLACEHOLDER_TEXT
-    )
-
-    if (updatedContent) {
-      updatedLibraryItem.readableContent = updatedContent
-    }
-
-    await enqueueProcessYouTubeTranscript({
-      videoId,
-      ...jobData,
-    })
-  }
+  // if ('getTranscript' in video && duration > 0 && duration < 1801) {
+  //   // If the video has a transcript available, put a placehold in and
+  //   // enqueue a job to process the full transcript
+  //   const updatedContent = await addTranscriptToReadableContent(
+  //     libraryItem.originalUrl,
+  //     libraryItem.readableContent,
+  //     TRANSCRIPT_PLACEHOLDER_TEXT
+  //   )
+
+  //   if (updatedContent) {
+  //     updatedLibraryItem.readableContent = updatedContent
+  //   }
+
+  //   await enqueueProcessYouTubeTranscript({
+  //     videoId,
+  //     ...jobData,
+  //   })
+  // }
 
   if (updatedLibraryItem !== {}) {
     await updateLibraryItem(

packages/api/src/queue-processor.ts

@@ -159,25 +159,25 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          case 'refresh-all-feeds': {
-            const queue = await getQueue()
-            const counts = await queue?.getJobCounts('prioritized')
-            if (counts && counts.wait > 1000) {
-              return
-            }
-            return await refreshAllFeeds(appDataSource)
-          }
-          case 'refresh-feed': {
-            return await refreshFeed(job.data)
-          }
+          // case 'refresh-all-feeds': {
+          //   const queue = await getQueue()
+          //   const counts = await queue?.getJobCounts('prioritized')
+          //   if (counts && counts.wait > 1000) {
+          //     return
+          //   }
+          //   return await refreshAllFeeds(appDataSource)
+          // }
+          // case 'refresh-feed': {
+          //   return await refreshFeed(job.data)
+          // }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
-          case 'update-pdf-content': {
-            return updatePDFContentJob(job.data)
-          }
-          case THUMBNAIL_JOB:
-            return findThumbnail(job.data)
+          // case 'update-pdf-content': {
+          //   return updatePDFContentJob(job.data)
+          // }
+          // case THUMBNAIL_JOB:
+          //   return findThumbnail(job.data)
           case TRIGGER_RULE_JOB_NAME:
             return triggerRule(job.data)
           case UPDATE_LABELS_JOB:
@@ -192,12 +192,12 @@ export const createWorker = (connection: ConnectionOptions) =>
             return callWebhook(job.data)
           case EXPORT_ITEM_JOB_NAME:
             return exportItem(job.data)
-          case AI_SUMMARIZE_JOB_NAME:
-            return aiSummarize(job.data)
-          case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
-            return processYouTubeVideo(job.data)
-          case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
-            return processYouTubeTranscript(job.data)
+          // case AI_SUMMARIZE_JOB_NAME:
+          //   return aiSummarize(job.data)
+          // case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
+          //   return processYouTubeVideo(job.data)
+          // case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
+          //   return processYouTubeTranscript(job.data)
           case EXPORT_ALL_ITEMS_JOB_NAME:
             return exportAllItems(job.data)
           case SEND_EMAIL_JOB:
@@ -210,16 +210,16 @@ export const createWorker = (connection: ConnectionOptions) =>
             return saveNewsletterJob(job.data)
           case FORWARD_EMAIL_JOB:
             return forwardEmailJob(job.data)
-          case CREATE_DIGEST_JOB:
-            return createDigest(job.data)
+          // case CREATE_DIGEST_JOB:
+          //   return createDigest(job.data)
           case UPLOAD_CONTENT_JOB:
             return uploadContentJob(job.data)
-          case UPDATE_HOME_JOB:
-            return updateHome(job.data)
-          case SCORE_LIBRARY_ITEM_JOB:
-            return scoreLibraryItem(job.data)
-          case GENERATE_PREVIEW_CONTENT_JOB:
-            return generatePreviewContent(job.data)
+          // case UPDATE_HOME_JOB:
+          //   return updateHome(job.data)
+          // case SCORE_LIBRARY_ITEM_JOB:
+          //   return scoreLibraryItem(job.data)
+          // case GENERATE_PREVIEW_CONTENT_JOB:
+          //   return generatePreviewContent(job.data)
           case PRUNE_TRASH_JOB:
             return pruneTrashJob(job.data)
           case EXPIRE_FOLDERS_JOB_NAME:

packages/api/src/routers/export_router.ts

@@ -2,7 +2,11 @@ import cors from 'cors'
 import express, { Router } from 'express'
 import { TaskState } from '../generated/graphql'
 import { jobStateToTaskState } from '../queue-processor'
-import { countExportsWithin24Hours, saveExport } from '../services/export'
+import {
+  countExportsWithin24Hours,
+  countExportsWithinMinute,
+  saveExport,
+} from '../services/export'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { queueExportJob } from '../utils/createTask'
@@ -27,6 +31,17 @@ export function exportRouter() {
     const userId = claims.uid
 
     try {
+      const exportsWithinMinute = await countExportsWithinMinute(userId)
+      if (exportsWithinMinute >= 1) {
+        logger.error('User has reached the limit of exports within minute', {
+          userId,
+          exportsWithinMinute,
+        })
+        return res.status(400).send({
+          error: 'EXPORT_LIMIT_REACHED',
+        })
+      }
+
       const exportsWithin24Hours = await countExportsWithin24Hours(userId)
       if (exportsWithin24Hours >= 3) {
         logger.error('User has reached the limit of exports within 24 hours', {

packages/api/src/services/export.ts

@@ -13,6 +13,16 @@ export const saveExport = async (
   })
 }
 
+export const countExportsWithinMinute = async (
+  userId: string
+): Promise<number> => {
+  return getRepository(Export).countBy({
+    userId,
+    createdAt: MoreThan(new Date(Date.now() - 60 * 1000)),
+    state: In([TaskState.Pending, TaskState.Running, TaskState.Succeeded]),
+  })
+}
+
 export const countExportsWithin24Hours = async (
   userId: string
 ): Promise<number> => {

packages/api/src/utils/analytics.ts

@@ -21,18 +21,17 @@ class PostHogClient implements AnalyticClient {
   }
 
   capture({ distinctId, event, properties }: AnalyticEvent) {
-    // get client from request context
-    const client = httpContext.get<string>('client') || 'other'
-
-    this.client.capture({
-      distinctId,
-      event,
-      properties: {
-        ...properties,
-        client,
-        env: env.server.apiEnv,
-      },
-    })
+    // // get client from request context
+    // const client = httpContext.get<string>('client') || 'other'
+    // this.client.capture({
+    //   distinctId,
+    //   event,
+    //   properties: {
+    //     ...properties,
+    //     client,
+    //     env: env.server.apiEnv,
+    //   },
+    // })
   }
 
   async shutdownAsync() {

packages/web/lib/analytics.ts

@@ -1,6 +1,5 @@
 import { UserBasicData } from './networking/queries/useGetViewerQuery'
-import { intercomAppID, posthogApiKey, webBaseURL } from './appConfig'
-import posthog from 'posthog-js'
+import { intercomAppID } from './appConfig'
 
 const userInfo = (
   user: UserBasicData
@@ -19,17 +18,6 @@ const initAnalytics = (user?: UserBasicData): void => {
     vertical_padding: 120,
     custom_launcher_selector: '.custom-intercom-launcher',
   }
-  if (posthogApiKey) {
-    posthog.init(posthogApiKey, {
-      api_host: `${webBaseURL}/collect`,
-      autocapture: false,
-      person_profiles: 'identified_only',
-      disable_session_recording: false,
-      advanced_disable_decide: true,
-      advanced_disable_feature_flags: true,
-      advanced_disable_toolbar_metrics: true,
-    })
-  }
   if (user) {
     window.Intercom('boot', userInfo(user))
   }
@@ -46,15 +34,5 @@ export const setupAnalytics = (user?: UserBasicData): void => {
 
   if (user) {
     window.Intercom('update', userInfo(user))
-    posthog.identify(user.id, {
-      name: user.name,
-      username: user.profile.username,
-    })
-  }
-}
-
-export const deinitAnalytics = (): void => {
-  if (posthog && posthogApiKey) {
-    posthog.reset(true)
   }
 }

packages/web/lib/appConfig.ts

@@ -70,8 +70,6 @@ export const appleAuthRedirectURI =
 
 export const intercomAppID = process.env.NEXT_PUBLIC_INTERCOM_APP_ID
 
-export const posthogApiKey = process.env.NEXT_PUBLIC_POSTHOG_API_KEY
-
 export const googleID =
   appEnv == 'prod'
     ? process.env.NEXT_PUBLIC_GOOGLE_ID

packages/web/lib/logout.ts

@@ -1,5 +1,4 @@
 import { useQueryClient } from '@tanstack/react-query'
-import { deinitAnalytics } from './analytics'
 import { logoutMutation } from './networking/mutations/logoutMutation'
 import { useCallback } from 'react'
 import { useRouter } from 'next/router'
@@ -14,7 +13,6 @@ export const useLogout = () => {
       if (!result) {
         throw new Error('Logout failed')
       }
-      deinitAnalytics()
       queryClient.clear()
       console.log('cleared the query client')
       router.push('/login')

packages/web/next.config.js

@@ -45,10 +45,6 @@ const moduleExports = {
         }
       )
     }
-    rewrites.push({
-      source: '/collect/:match*',
-      destination: 'https://app.posthog.com/:match*',
-    })
     rewrites.push({
       source: '/home',
       destination: '/l/home',

packages/web/package.json

@@ -52,7 +52,6 @@
     "next": "^13.5.6",
     "node-html-markdown": "^1.3.0",
     "papaparse": "^5.4.1",
-    "posthog-js": "^1.158.3",
     "pspdfkit": "^2023.4.6",
     "re-resizable": "^6.9.11",
     "react": "^18.2.0",

packages/web/pages/[username]/[slug]/index.tsx

@@ -21,7 +21,6 @@ import { VerticalArticleActionsMenu } from '../../../components/templates/articl
 import { PdfHeaderSpacer } from '../../../components/templates/article/PdfHeaderSpacer'
 import { EpubContainerProps } from '../../../components/templates/article/EpubContainer'
 import { useSetPageLabels } from '../../../lib/hooks/useSetPageLabels'
-import { posthog } from 'posthog-js'
 import { PDFDisplaySettingsModal } from '../../../components/templates/article/PDFDisplaySettingsModal'
 import {
   ArticleReadingProgressMutationInput,
@@ -270,17 +269,6 @@ export default function Reader(): JSX.Element {
     }
   }, [actionHandler, goNextOrHome, goPreviousOrHome])
 
-  useEffect(() => {
-    if (libraryItem && viewerData) {
-      posthog.capture('link_read', {
-        link: libraryItem.id,
-        slug: libraryItem.slug,
-        reader: libraryItem.contentReader,
-        url: libraryItem.originalArticleUrl,
-      })
-    }
-  }, [libraryItem, viewerData])
-
   useRegisterActions(
     [
       {

packages/web/pages/_app.tsx

@@ -22,7 +22,6 @@ import {
 } from '../components/elements/KBar'
 import { updateTheme } from '../lib/themeUpdater'
 import { ThemeId } from '../components/tokens/stitches.config'
-import { posthog } from 'posthog-js'
 import { GoogleReCaptchaProvider } from '@google-recaptcha/react'
 import AsyncStorage from '@react-native-async-storage/async-storage'
 import { QueryClient } from '@tanstack/react-query'
@@ -96,16 +95,6 @@ const ConditionalCaptchaProvider = (props: {
 export function OmnivoreApp({ Component, pageProps }: AppProps): JSX.Element {
   const router = useRouter()
 
-  useEffect(() => {
-    const handleRouteChange = (url: string) => {
-      posthog.capture('$pageview')
-    }
-    router.events.on('routeChangeComplete', handleRouteChange)
-    return () => {
-      router.events.off('routeChangeComplete', handleRouteChange)
-    }
-  }, [router.events])
-
   return (
     <ConditionalCaptchaProvider>
       <Toaster />