From 658dc9fc888f5961d8a239cdd089825e24ad1994 Mon Sep 17 00:00:00 2001 From: mcamou Date: Thu, 30 Jan 2025 17:43:18 +0100 Subject: [PATCH] Refactoring and simplification --- internal/jobs/twitter.go | 212 +++++++++++------------------ internal/jobs/twitter/common.go | 2 +- internal/jobs/twitter/followers.go | 2 +- internal/jobs/twitter/profile.go | 2 +- internal/jobs/twitter/tweets.go | 55 -------- 5 files changed, 80 insertions(+), 193 deletions(-) delete mode 100644 internal/jobs/twitter/tweets.go diff --git a/internal/jobs/twitter.go b/internal/jobs/twitter.go index 41ab099..47ec6ad 100644 --- a/internal/jobs/twitter.go +++ b/internal/jobs/twitter.go @@ -35,27 +35,37 @@ func parseAccounts(accountPairs []string) []*twitter.TwitterAccount { } func (ts *TwitterScraper) getAuthenticatedScraper(baseDir string) (*twitter.Scraper, *twitter.TwitterAccount, error) { + // if baseDir is empty, use the default data directory + if baseDir == "" { + baseDir = ts.configuration.DataDir + } + account := ts.accountManager.GetNextAccount() if account == nil { ts.statsCollector.AddStat(stats.TwitterAuthErrors, 1) return nil, nil, fmt.Errorf("all accounts are rate-limited") } + scraper := twitter.NewScraper(account, baseDir) if scraper == nil { ts.statsCollector.AddStat(stats.TwitterAuthErrors, 1) logrus.Errorf("Authentication failed for %s", account.Username) return nil, account, fmt.Errorf("twitter authentication failed for %s", account.Username) } + return scraper, account, nil } -func (ts *TwitterScraper) handleRateLimit(err error, account *twitter.TwitterAccount) bool { +// handleError handles Twitter API errors, detecting rate limits and marking accounts as rate-limited if necessary +// It returns true if the account is rate-limited, false otherwise +func (ts *TwitterScraper) handleError(err error, account *twitter.TwitterAccount) bool { if strings.Contains(err.Error(), "Rate limit exceeded") { ts.statsCollector.AddStat(stats.TwitterRateErrors, 1) ts.accountManager.MarkAccountRateLimited(account) logrus.Warnf("rate limited: %s", account.Username) return true } + ts.statsCollector.AddStat(stats.TwitterErrors, 1) return false } @@ -80,7 +90,7 @@ func (ts *TwitterScraper) ScrapeFollowersForProfile(baseDir string, username str followingResponse, errString, _ := scraper.FetchFollowers(username, count, "") if errString != "" { err := fmt.Errorf("rate limited: %s", errString) - if ts.handleRateLimit(err, account) { + if ts.handleError(err, account) { return nil, err } @@ -88,6 +98,7 @@ func (ts *TwitterScraper) ScrapeFollowersForProfile(baseDir string, username str return nil, fmt.Errorf("error fetching followers: %s", errString) } + ts.statsCollector.AddStat(stats.TwitterProfiles, uint(len(followingResponse))) return followingResponse, nil } @@ -100,11 +111,11 @@ func (ts *TwitterScraper) ScrapeTweetsProfile(baseDir string, username string) ( ts.statsCollector.AddStat(stats.TwitterScrapes, 1) profile, err := scraper.GetProfile(username) if err != nil { - if ts.handleRateLimit(err, account) { - return twitterscraper.Profile{}, err - } + _ = ts.handleError(err, account) return twitterscraper.Profile{}, err } + + ts.statsCollector.AddStat(stats.TwitterProfiles, 1) return profile, nil } @@ -118,11 +129,10 @@ func (ts *TwitterScraper) ScrapeTweetsByQuery(baseDir string, query string, coun var tweets []*TweetResult ctx := context.Background() scraper.SetSearchMode(twitterscraper.SearchLatest) + for tweet := range scraper.SearchTweets(ctx, query, count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } + _ = ts.handleError(tweet.Error, account) return nil, tweet.Error } tweets = append(tweets, &TweetResult{Tweet: &tweet.Tweet}) @@ -132,16 +142,28 @@ func (ts *TwitterScraper) ScrapeTweetsByQuery(baseDir string, query string, coun return tweets, nil } +func (ts *TwitterScraper) ScrapeTweetByID(baseDir string, tweetID string) (*twitterscraper.Tweet, error) { + ts.statsCollector.AddStat(stats.TwitterScrapes, 1) + + scraper, account, err := ts.getAuthenticatedScraper(baseDir) + if err != nil { + return nil, err + } + + tweet, err := scraper.GetTweet(tweetID) + if err != nil { + _ = ts.handleError(err, account) + return nil, err + } + + ts.statsCollector.AddStat(stats.TwitterTweets, 1) + return tweet, nil +} + // End of adapted code from masa-oracle (commit: bf277c646d44c49cc387bc5219c900e96b06dc02) // GetTweet retrieves a tweet by ID func (ts *TwitterScraper) GetTweet(baseDir, tweetID string) (*TweetResult, error) { - - // if baseDir is empty, use the default data directory - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -150,9 +172,7 @@ func (ts *TwitterScraper) GetTweet(baseDir, tweetID string) (*TweetResult, error ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweet, err := scraper.GetTweet(tweetID) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } @@ -162,10 +182,6 @@ func (ts *TwitterScraper) GetTweet(baseDir, tweetID string) (*TweetResult, error // GetTweetReplies retrieves replies to a tweet func (ts *TwitterScraper) GetTweetReplies(baseDir, tweetID string, cursor string) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -174,25 +190,21 @@ func (ts *TwitterScraper) GetTweetReplies(baseDir, tweetID string, cursor string ts.statsCollector.AddStat(stats.TwitterScrapes, 1) var replies []*TweetResult tweets, threadCursor, err := scraper.GetTweetReplies(tweetID, cursor) + for i, tweet := range tweets { if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } replies = append(replies, &TweetResult{Tweet: tweet, ThreadCursor: threadCursor[i], Error: err}) } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(replies))) return replies, nil } // GetTweetRetweeters retrieves users who retweeted a tweet func (ts *TwitterScraper) GetTweetRetweeters(baseDir, tweetID string, count int, cursor string) ([]*twitterscraper.Profile, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(ts.configuration.DataDir) if err != nil { return nil, err @@ -201,21 +213,16 @@ func (ts *TwitterScraper) GetTweetRetweeters(baseDir, tweetID string, count int, ts.statsCollector.AddStat(stats.TwitterScrapes, 1) retweeters, _, err := scraper.GetTweetRetweeters(tweetID, count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } + ts.statsCollector.AddStat(stats.TwitterProfiles, uint(len(retweeters))) return retweeters, nil } // GetUserTweets retrieves tweets from a user func (ts *TwitterScraper) GetUserTweets(baseDir, username string, count int) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -225,23 +232,18 @@ func (ts *TwitterScraper) GetUserTweets(baseDir, username string, count int) ([] var tweets []*TweetResult for tweet := range scraper.GetTweets(context.Background(), username, count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } + _ = ts.handleError(tweet.Error, account) return nil, tweet.Error } tweets = append(tweets, &TweetResult{Tweet: &tweet.Tweet}) } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nil } // FetchUserTweets retrieves tweets from a user func (ts *TwitterScraper) FetchUserTweets(baseDir, username string, count int, cursor string) ([]*twitterscraper.Tweet, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -250,21 +252,16 @@ func (ts *TwitterScraper) FetchUserTweets(baseDir, username string, count int, c ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweets, nextCursor, err := scraper.FetchTweets(username, count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nextCursor, nil } // GetUserMedia retrieves media tweets from a user func (ts *TwitterScraper) GetUserMedia(baseDir, username string, count int) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -274,7 +271,7 @@ func (ts *TwitterScraper) GetUserMedia(baseDir, username string, count int) ([]* var media []*TweetResult for tweet := range scraper.GetTweetsAndReplies(context.Background(), username, count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { + if ts.handleError(tweet.Error, account) { return nil, tweet.Error } continue @@ -283,16 +280,13 @@ func (ts *TwitterScraper) GetUserMedia(baseDir, username string, count int) ([]* media = append(media, &TweetResult{Tweet: &tweet.Tweet}) } } + ts.statsCollector.AddStat(stats.TwitterOther, uint(len(media))) return media, nil } // FetchUserMedia retrieves media tweets from a user func (ts *TwitterScraper) FetchUserMedia(baseDir, username string, count int, cursor string) ([]*twitterscraper.Tweet, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -301,21 +295,16 @@ func (ts *TwitterScraper) FetchUserMedia(baseDir, username string, count int, cu ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweets, nextCursor, err := scraper.FetchTweetsAndReplies(username, count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nextCursor, nil } // GetBookmarks retrieves user's bookmarked tweets func (ts *TwitterScraper) GetBookmarks(baseDir string, count int) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -325,9 +314,7 @@ func (ts *TwitterScraper) GetBookmarks(baseDir string, count int) ([]*TweetResul var bookmarks []*TweetResult for tweet := range scraper.GetBookmarks(context.Background(), count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } + _ = ts.handleError(tweet.Error, account) return nil, tweet.Error } bookmarks = append(bookmarks, &TweetResult{Tweet: &tweet.Tweet}) @@ -335,16 +322,13 @@ func (ts *TwitterScraper) GetBookmarks(baseDir string, count int) ([]*TweetResul break } } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(bookmarks))) return bookmarks, nil } // FetchBookmarks retrieves user's bookmarked tweets func (ts *TwitterScraper) FetchBookmarks(baseDir string, count int, cursor string) ([]*twitterscraper.Tweet, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -353,21 +337,16 @@ func (ts *TwitterScraper) FetchBookmarks(baseDir string, count int, cursor strin ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweets, nextCursor, err := scraper.FetchBookmarks(count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nextCursor, nil } // GetHomeTweets retrieves tweets from user's home timeline func (ts *TwitterScraper) GetHomeTweets(baseDir string, count int) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -377,9 +356,7 @@ func (ts *TwitterScraper) GetHomeTweets(baseDir string, count int) ([]*TweetResu var tweets []*TweetResult for tweet := range scraper.GetHomeTweets(context.Background(), count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } + _ = ts.handleError(tweet.Error, account) return nil, tweet.Error } tweets = append(tweets, &TweetResult{Tweet: &tweet.Tweet}) @@ -387,16 +364,13 @@ func (ts *TwitterScraper) GetHomeTweets(baseDir string, count int) ([]*TweetResu break } } + ts.statsCollector.AddStat(stats.TwitterOther, uint(len(tweets))) return tweets, nil } // FetchHomeTweets retrieves tweets from user's home timeline func (ts *TwitterScraper) FetchHomeTweets(baseDir string, count int, cursor string) ([]*twitterscraper.Tweet, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -405,21 +379,16 @@ func (ts *TwitterScraper) FetchHomeTweets(baseDir string, count int, cursor stri ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweets, nextCursor, err := scraper.FetchHomeTweets(count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nextCursor, nil } // GetForYouTweets retrieves tweets from For You timeline func (ts *TwitterScraper) GetForYouTweets(baseDir string, count int) ([]*TweetResult, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -429,26 +398,22 @@ func (ts *TwitterScraper) GetForYouTweets(baseDir string, count int) ([]*TweetRe var tweets []*TweetResult for tweet := range scraper.GetForYouTweets(context.Background(), count) { if tweet.Error != nil { - if ts.handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } + _ = ts.handleError(tweet.Error, account) return nil, tweet.Error } + tweets = append(tweets, &TweetResult{Tweet: &tweet.Tweet}) if len(tweets) >= count { break } } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nil } // FetchForYouTweets retrieves tweets from For You timeline func (ts *TwitterScraper) FetchForYouTweets(baseDir string, count int, cursor string) ([]*twitterscraper.Tweet, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -457,21 +422,16 @@ func (ts *TwitterScraper) FetchForYouTweets(baseDir string, count int, cursor st ts.statsCollector.AddStat(stats.TwitterScrapes, 1) tweets, nextCursor, err := scraper.FetchForYouTweets(count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterTweets, uint(len(tweets))) return tweets, nextCursor, nil } // GetProfileByID retrieves a user profile by ID func (ts *TwitterScraper) GetProfileByID(baseDir, userID string) (*twitterscraper.Profile, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -480,11 +440,10 @@ func (ts *TwitterScraper) GetProfileByID(baseDir, userID string) (*twitterscrape ts.statsCollector.AddStat(stats.TwitterScrapes, 1) profile, err := scraper.GetProfileByID(userID) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } + ts.statsCollector.AddStat(stats.TwitterProfiles, 1) return &profile, nil } @@ -502,18 +461,16 @@ func (ts *TwitterScraper) SearchProfile(query string, count int) ([]*twitterscra if len(profiles) >= count { break } + profiles = append(profiles, profile) } + ts.statsCollector.AddStat(stats.TwitterProfiles, uint(len(profiles))) return profiles, nil } // GetTrends retrieves current trending topics func (ts *TwitterScraper) GetTrends(baseDir string) ([]string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, err @@ -522,21 +479,17 @@ func (ts *TwitterScraper) GetTrends(baseDir string) ([]string, error) { ts.statsCollector.AddStat(stats.TwitterScrapes, 1) trends, err := scraper.GetTrends() if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } + + // TODO Should this be the number of topics, or 1 ? ts.statsCollector.AddStat(stats.TwitterOther, uint(len(trends))) return trends, nil } // GetFollowers retrieves users that follow a user func (ts *TwitterScraper) GetFollowers(baseDir, user string, count int, cursor string) ([]*twitterscraper.Profile, string, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } - // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { return nil, "", err @@ -545,20 +498,16 @@ func (ts *TwitterScraper) GetFollowers(baseDir, user string, count int, cursor s ts.statsCollector.AddStat(stats.TwitterScrapes, 1) followers, nextCursor, err := scraper.FetchFollowers(user, count, cursor) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, "", err - } + _ = ts.handleError(err, account) return nil, "", err } + ts.statsCollector.AddStat(stats.TwitterProfiles, uint(len(followers))) return followers, nextCursor, nil } // GetFollowing retrieves users that a user follows func (ts *TwitterScraper) GetFollowing(baseDir, username string, count int) ([]*twitterscraper.Profile, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { @@ -569,20 +518,16 @@ func (ts *TwitterScraper) GetFollowing(baseDir, username string, count int) ([]* following, errString, _ := scraper.FetchFollowing(username, count, "") if errString != "" { err := fmt.Errorf("error fetching following: %s", errString) - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } + ts.statsCollector.AddStat(stats.TwitterProfiles, uint(len(following))) return following, nil } // GetSpace retrieves space information by ID func (ts *TwitterScraper) GetSpace(baseDir, spaceID string) (*twitterscraper.Space, error) { - if baseDir == "" { - baseDir = ts.configuration.DataDir - } // get the authenticated scraper scraper, account, err := ts.getAuthenticatedScraper(baseDir) if err != nil { @@ -592,11 +537,10 @@ func (ts *TwitterScraper) GetSpace(baseDir, spaceID string) (*twitterscraper.Spa ts.statsCollector.AddStat(stats.TwitterScrapes, 1) space, err := scraper.GetSpace(spaceID) if err != nil { - if ts.handleRateLimit(err, account) { - return nil, err - } + _ = ts.handleError(err, account) return nil, err } + ts.statsCollector.AddStat(stats.TwitterOther, 1) return space, nil } @@ -671,7 +615,7 @@ func (ws *TwitterScraper) ExecuteJob(j types.Job) (types.JobResult, error) { }, err case "getbyid": - tweet, err := twitter.ScrapeTweetByID(ws.configuration.DataDir, args.Query, ws.statsCollector) + tweet, err := ws.ScrapeTweetByID(ws.configuration.DataDir, args.Query) if err != nil { return types.JobResult{Error: err.Error()}, err } @@ -860,8 +804,6 @@ func (ws *TwitterScraper) ExecuteJob(j types.Job) (types.JobResult, error) { }, err } - // Do the web scraping here - // For now, just return the URL return types.JobResult{ Error: "invalid search type", }, fmt.Errorf("invalid search type") diff --git a/internal/jobs/twitter/common.go b/internal/jobs/twitter/common.go index 8a3ee55..d9e8cb0 100644 --- a/internal/jobs/twitter/common.go +++ b/internal/jobs/twitter/common.go @@ -63,7 +63,7 @@ func getAuthenticatedScraper(baseDir string) (*Scraper, *TwitterAccount, error) return scraper, account, nil } -func handleRateLimit(err error, account *TwitterAccount) bool { +func handleError(err error, account *TwitterAccount) bool { if strings.Contains(err.Error(), "Rate limit exceeded") { accountManager.MarkAccountRateLimited(account) logrus.Warnf("rate limited: %s", account.Username) diff --git a/internal/jobs/twitter/followers.go b/internal/jobs/twitter/followers.go index b0e1e2b..0423164 100644 --- a/internal/jobs/twitter/followers.go +++ b/internal/jobs/twitter/followers.go @@ -17,7 +17,7 @@ func ScrapeFollowersForProfile(baseDir string, username string, count int) ([]*t followingResponse, errString, _ := scraper.FetchFollowers(username, count, "") if errString != "" { err := fmt.Errorf("rate limited: %s", errString) - if handleRateLimit(err, account) { + if handleError(err, account) { return nil, err } diff --git a/internal/jobs/twitter/profile.go b/internal/jobs/twitter/profile.go index ba8328c..ea75e6a 100644 --- a/internal/jobs/twitter/profile.go +++ b/internal/jobs/twitter/profile.go @@ -12,7 +12,7 @@ func ScrapeTweetsProfile(baseDir string, username string) (twitterscraper.Profil profile, err := scraper.GetProfile(username) if err != nil { - if handleRateLimit(err, account) { + if handleError(err, account) { return twitterscraper.Profile{}, err } return twitterscraper.Profile{}, err diff --git a/internal/jobs/twitter/tweets.go b/internal/jobs/twitter/tweets.go deleted file mode 100644 index 58f5e3b..0000000 --- a/internal/jobs/twitter/tweets.go +++ /dev/null @@ -1,55 +0,0 @@ -package twitter - -import ( - "context" - - twitterscraper "github.com/imperatrona/twitter-scraper" - "github.com/masa-finance/tee-worker/internal/jobs/stats" -) - -type TweetResult struct { - Tweet *twitterscraper.Tweet - Error error -} - -func ScrapeTweetsByQuery(baseDir string, query string, count int) ([]*TweetResult, error) { - scraper, account, err := getAuthenticatedScraper(baseDir) - if err != nil { - return nil, err - } - - var tweets []*TweetResult - ctx := context.Background() - scraper.SetSearchMode(twitterscraper.SearchLatest) - for tweet := range scraper.SearchTweets(ctx, query, count) { - if tweet.Error != nil { - if handleRateLimit(tweet.Error, account) { - return nil, tweet.Error - } - return nil, tweet.Error - } - tweets = append(tweets, &TweetResult{Tweet: &tweet.Tweet}) - } - return tweets, nil -} - -func ScrapeTweetByID(baseDir string, tweetID string, c *stats.StatsCollector) (*twitterscraper.Tweet, error) { - c.AddStat(stats.TwitterScrapes, 1) - - scraper, account, err := getAuthenticatedScraper(baseDir) - if err != nil { - return nil, err - } - - tweet, err := scraper.GetTweet(tweetID) - if err != nil { - c.AddStat(stats.TwitterErrors, 1) - if handleRateLimit(err, account) { - return nil, err - } - return nil, err - } - - c.AddStat(stats.TwitterTweets, 1) - return tweet, nil -}