diff --git a/.env.example b/.env.example index ca9a774..0d5ec75 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,11 @@ TOGETHER_API_KEY=your_together_api_key_here # Required if monitoring web pages (https://www.firecrawl.dev/) FIRECRAWL_API_KEY=your_firecrawl_api_key_here -# Required if monitoring Twitter/X trends (https://developer.x.com/) -X_API_BEARER_TOKEN=your_twitter_api_bearer_token_here +# Required if monitoring Twitter/X trends (https://developer.x.com/) and SOCIALDATA_API_KEY not set +X_API_BEARER_TOKEN= + +# Required if monitoring Twitter/X trends (https://socialdata.tools/) and X_API_BEARER_TOKEN not set +SOCIALDATA_API_KEY= # Required: Incoming Webhook URL from Slack for notifications SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL \ No newline at end of file diff --git a/README.md b/README.md index ae6f686..1a4e325 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Learn how to set up Trend Finder and start monitoring trends in this video! ## How it Works 1. **Data Collection** 📥 - - Monitors selected influencers' posts on Twitter/X using the X API (Warning: the X API free plan is rate limited to only monitor 1 X account every 15 min) + - Monitors selected influencers' posts on Twitter/X using the X API (Warning: the X API free plan is rate limited to only monitor 1 X account every 15 min) or alternative API provided by [SocialData.tools](https://socialdata.tools) - Monitors websites for new releases and news with Firecrawl's /extract - Runs on a scheduled basis using cron jobs @@ -48,6 +48,7 @@ Learn how to set up Trend Finder and start monitoring trends in this video! - **AI/ML**: Together AI - **Data Sources**: - Twitter/X API + - [SocialData.tools API](https://socialdata.tools) - Firecrawl - **Notifications**: Slack Webhooks - **Scheduling**: node-cron @@ -76,8 +77,11 @@ TOGETHER_API_KEY=your_together_api_key_here # Required if monitoring web pages (https://www.firecrawl.dev/) FIRECRAWL_API_KEY=your_firecrawl_api_key_here -# Required if monitoring Twitter/X trends (https://developer.x.com/) -X_API_BEARER_TOKEN=your_twitter_api_bearer_token_here +# Required if monitoring Twitter/X trends (https://developer.x.com/) and SOCIALDATA_API_KEY not set +X_API_BEARER_TOKEN= + +# Required if monitoring Twitter/X trends (https://socialdata.tools/) and X_API_BEARER_TOKEN not set +SOCIALDATA_API_KEY= # Required: Incoming Webhook URL from Slack for notifications SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL diff --git a/src/services/getCronSources.ts b/src/services/getCronSources.ts index 613cf82..f340208 100644 --- a/src/services/getCronSources.ts +++ b/src/services/getCronSources.ts @@ -8,7 +8,7 @@ export async function getCronSources() { console.log("Fetching sources..."); // Check for required API keys - const hasXApiKey = !!process.env.X_API_BEARER_TOKEN; + const hasXApiKey = !!process.env.X_API_BEARER_TOKEN || !!process.env.SOCIALDATA_API_KEY; const hasFirecrawlKey = !!process.env.FIRECRAWL_API_KEY; // Filter sources based on available API keys diff --git a/src/services/scrapeSources.ts b/src/services/scrapeSources.ts index 00f5d42..6ed8e77 100644 --- a/src/services/scrapeSources.ts +++ b/src/services/scrapeSources.ts @@ -29,7 +29,8 @@ export async function scrapeSources(sources: string[]) { let combinedText: { stories: any[] } = { stories: [] }; // Configure these if you want to toggle behavior - const useTwitter = true; + const useTwitter = !!process.env.X_API_BEARER_TOKEN; + const useSocialData = !!process.env.SOCIALDATA_API_KEY; const useScrape = true; for (const source of sources) { @@ -86,6 +87,55 @@ export async function scrapeSources(sources: string[]) { } } } + else if (useSocialData) { + const usernameMatch = source.match(/x\.com\/([^\/]+)/); + if (usernameMatch) { + const username = usernameMatch[1]; + + // Get tweets from the last 24 hours + const startTime = Math.floor((Date.now() - 24 * 60 * 60 * 1000) / 1000); + + // Build the search query for tweets + const query = `from:${username} -filter:nativeretweets -filter:replies since_time:${startTime}`; + console.log(query) + const encodedQuery = encodeURIComponent(query); + + // SocialData.tools API URL + const apiUrl = `https://api.socialdata.tools/twitter/search?query=${encodedQuery}`; + + // Fetch recent tweets from the Twitter API + const response = await fetch(apiUrl, { + headers: { + Authorization: `Bearer ${process.env.SOCIALDATA_API_KEY}`, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch tweets for ${username}: ${response.statusText}`); + } + + const tweets = await response.json(); + + if (tweets.tweets?.length === 0) { + console.log(`No tweets found for username ${username}.`); + } else if (Array.isArray(tweets.tweets)) { + console.log(`Tweets found from username ${username}`); + const stories = tweets.tweets.map((tweet: any) => { + return { + headline: tweet.full_text, + link: `https://x.com/${tweet.user.screen_name}/status/${tweet.id_str}`, + date_posted: tweet.tweet_created_at, + }; + }); + combinedText.stories.push(...stories); + } else { + console.error( + "Expected tweets.data to be an array:", + tweets.data + ); + } + } + } } // --- 2) Handle all other sources with Firecrawl extract --- else {