Add SocialData as alternative datasource for tweets

.env.example

@@ -4,8 +4,11 @@ TOGETHER_API_KEY=your_together_api_key_here
 # Required if monitoring web pages (https://www.firecrawl.dev/)
 FIRECRAWL_API_KEY=your_firecrawl_api_key_here
 
-# Required if monitoring Twitter/X trends (https://developer.x.com/)
-X_API_BEARER_TOKEN=your_twitter_api_bearer_token_here
+# Required if monitoring Twitter/X trends (https://developer.x.com/) and SOCIALDATA_API_KEY not set
+X_API_BEARER_TOKEN=
+
+# Required if monitoring Twitter/X trends (https://socialdata.tools/) and X_API_BEARER_TOKEN not set
+SOCIALDATA_API_KEY=
 
 # Required: Incoming Webhook URL from Slack for notifications
 SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL

README.md

@@ -19,7 +19,7 @@ Learn how to set up Trend Finder and start monitoring trends in this video!
 ## How it Works
 
 1. **Data Collection** 📥
-   - Monitors selected influencers' posts on Twitter/X using the X API (Warning: the X API free plan is rate limited to only monitor 1 X account every 15 min)
+   - Monitors selected influencers' posts on Twitter/X using the X API (Warning: the X API free plan is rate limited to only monitor 1 X account every 15 min) or alternative API provided by [SocialData.tools](https://socialdata.tools)
    - Monitors websites for new releases and news with Firecrawl's /extract
    - Runs on a scheduled basis using cron jobs
 
@@ -48,6 +48,7 @@ Learn how to set up Trend Finder and start monitoring trends in this video!
 - **AI/ML**: Together AI
 - **Data Sources**:
   - Twitter/X API
+  - [SocialData.tools API](https://socialdata.tools)
   - Firecrawl
 - **Notifications**: Slack Webhooks
 - **Scheduling**: node-cron
@@ -76,8 +77,11 @@ TOGETHER_API_KEY=your_together_api_key_here
 # Required if monitoring web pages (https://www.firecrawl.dev/)
 FIRECRAWL_API_KEY=your_firecrawl_api_key_here
 
-# Required if monitoring Twitter/X trends (https://developer.x.com/)
-X_API_BEARER_TOKEN=your_twitter_api_bearer_token_here
+# Required if monitoring Twitter/X trends (https://developer.x.com/) and SOCIALDATA_API_KEY not set
+X_API_BEARER_TOKEN=
+
+# Required if monitoring Twitter/X trends (https://socialdata.tools/) and X_API_BEARER_TOKEN not set
+SOCIALDATA_API_KEY=
 
 # Required: Incoming Webhook URL from Slack for notifications
 SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL

src/services/getCronSources.ts

@@ -8,7 +8,7 @@ export async function getCronSources() {
     console.log("Fetching sources...");
 
     // Check for required API keys
-    const hasXApiKey = !!process.env.X_API_BEARER_TOKEN;
+    const hasXApiKey = !!process.env.X_API_BEARER_TOKEN || !!process.env.SOCIALDATA_API_KEY;
     const hasFirecrawlKey = !!process.env.FIRECRAWL_API_KEY;
 
     // Filter sources based on available API keys

src/services/scrapeSources.ts

@@ -29,7 +29,8 @@ export async function scrapeSources(sources: string[]) {
   let combinedText: { stories: any[] } = { stories: [] };
 
   // Configure these if you want to toggle behavior
-  const useTwitter = true;
+  const useTwitter = !!process.env.X_API_BEARER_TOKEN;
+  const useSocialData = !!process.env.SOCIALDATA_API_KEY;
   const useScrape = true;
 
   for (const source of sources) {
@@ -86,6 +87,55 @@ export async function scrapeSources(sources: string[]) {
           }
         }
       }
+      else if (useSocialData) {
+        const usernameMatch = source.match(/x\.com\/([^\/]+)/);
+        if (usernameMatch) {
+          const username = usernameMatch[1];
+
+          // Get tweets from the last 24 hours
+          const startTime = Math.floor((Date.now() - 24 * 60 * 60 * 1000) / 1000);
+
+          // Build the search query for tweets
+          const query = `from:${username} -filter:nativeretweets -filter:replies since_time:${startTime}`;
+          console.log(query)
+          const encodedQuery = encodeURIComponent(query);
+
+          // SocialData.tools API URL
+          const apiUrl = `https://api.socialdata.tools/twitter/search?query=${encodedQuery}`;
+
+          // Fetch recent tweets from the Twitter API
+          const response = await fetch(apiUrl, {
+            headers: {
+              Authorization: `Bearer ${process.env.SOCIALDATA_API_KEY}`,
+            },
+          });
+
+          if (!response.ok) {
+            throw new Error(`Failed to fetch tweets for ${username}: ${response.statusText}`);
+          }
+
+          const tweets = await response.json();
+
+          if (tweets.tweets?.length === 0) {
+            console.log(`No tweets found for username ${username}.`);
+          } else if (Array.isArray(tweets.tweets)) {
+            console.log(`Tweets found from username ${username}`);
+            const stories = tweets.tweets.map((tweet: any) => {
+              return {
+                headline: tweet.full_text,
+                link: `https://x.com/${tweet.user.screen_name}/status/${tweet.id_str}`,
+                date_posted: tweet.tweet_created_at,
+              };
+            });
+            combinedText.stories.push(...stories);
+          } else {
+            console.error(
+              "Expected tweets.data to be an array:",
+              tweets.data
+            );
+          }
+        }
+      }
     }
     // --- 2) Handle all other sources with Firecrawl extract ---
     else {