Skip to content

Commit

Permalink
Merge pull request #321 from lockdown-systems/305-faster-deletes
Browse files Browse the repository at this point in the history 
Faster deletes
  • Loading branch information
@micahflee
micahflee authored Dec 8, 2024
2 parents 9516bc9 + 0bd3466 commit 9722314
Show file tree
Hide file tree
Showing 21 changed files with 832 additions and 321 deletions.
3 changes: 2 additions & 1 deletion src/account_x.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ vi.mock('electron', () => ({
session: {
fromPartition: vi.fn().mockReturnValue({
webRequest: {
onCompleted: vi.fn()
onCompleted: vi.fn(),
onSendHeaders: vi.fn(),
}
})
},
Expand Down
139 changes: 127 additions & 12 deletions src/account_x.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
XJob,
XProgress, emptyXProgress,
XTweetItem,
XTweetItemArchive,
XArchiveStartResponse, emptyXArchiveStartResponse,
XRateLimitInfo, emptyXRateLimitInfo,
XIndexMessagesStartResponse,
Expand Down Expand Up @@ -154,6 +155,16 @@ function convertXJobRowToXJob(row: XJobRow): XJob {
}

function convertTweetRowToXTweetItem(row: XTweetRow): XTweetItem {
return {
id: row.tweetID,
t: row.text,
l: row.likeCount,
r: row.retweetCount,
d: row.createdAt,
};
}

function convertTweetRowToXTweetItemArchive(row: XTweetRow): XTweetItemArchive {
return {
url: `https://x.com/${row.path}`,
tweetID: row.tweetID,
Expand All @@ -180,6 +191,8 @@ export class XAccountController {
public mitmController: IMITMController;
private progress: XProgress = emptyXProgress();

private cookies: Record<string, string> = {};

constructor(accountID: number, mitmController: IMITMController) {
this.mitmController = mitmController;

Expand Down Expand Up @@ -212,6 +225,23 @@ export class XAccountController {
this.deleteDMsMarkDeleted(conversationID);
}
});

ses.webRequest.onSendHeaders((details) => {
// Keep track of cookies
if (details.url.startsWith("https://x.com/") && details.requestHeaders) {
const cookieHeader = details.requestHeaders['Cookie'];
if (cookieHeader) {
const cookies = cookieHeader.split(';');
cookies.forEach((cookie) => {
const parts = cookie.split('=');
if (parts.length == 2) {
this.cookies[parts[0].trim()] = parts[1].trim();
}
});
log.info("XAccountController: cookies", this.cookies);
}
}
});
}

cleanup() {
Expand Down Expand Up @@ -1078,14 +1108,14 @@ export class XAccountController {
if (this.account) {
const tweetsResp: XTweetRow[] = exec(
this.db,
'SELECT tweetID, createdAt, path, username FROM tweet WHERE username = ? AND text NOT LIKE ? ORDER BY createdAt',
'SELECT tweetID, text, likeCount, retweetCount, createdAt, path FROM tweet WHERE username = ? AND text NOT LIKE ? ORDER BY createdAt',
[this.account.username, "RT @%"],
"all"
) as XTweetRow[];

const items: XTweetItem[] = [];
const items: XTweetItemArchive[] = [];
for (let i = 0; i < tweetsResp.length; i++) {
items.push(convertTweetRowToXTweetItem(tweetsResp[i]))
items.push(convertTweetRowToXTweetItemArchive(tweetsResp[i]))
}

return {
Expand Down Expand Up @@ -1325,42 +1355,105 @@ export class XAccountController {
// Both likes and retweets thresholds
tweets = exec(
this.db,
'SELECT tweetID, createdAt, path, username FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? AND retweetCount <= ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? AND retweetCount <= ? ORDER BY createdAt ASC',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsLikesThreshold, this.account.deleteTweetsRetweetsThreshold],
"all"
) as XTweetRow[];
} else if (this.account.deleteTweetsLikesThresholdEnabled && !this.account.deleteTweetsRetweetsThresholdEnabled) {
// Just likes threshold
tweets = exec(
this.db,
'SELECT tweetID, createdAt, path, username FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? ORDER BY createdAt ASC',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsLikesThreshold],
"all"
) as XTweetRow[];
} else if (!this.account.deleteTweetsLikesThresholdEnabled && this.account.deleteTweetsRetweetsThresholdEnabled) {
// Just retweets threshold
tweets = exec(
this.db,
'SELECT tweetID, createdAt, path, username FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND retweetCount <= ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND retweetCount <= ? ORDER BY createdAt ASC',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsRetweetsThreshold],
"all"
) as XTweetRow[];
} else {
// Neither likes nor retweets threshold
tweets = exec(
this.db,
'SELECT tweetID, createdAt, path, username FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? ORDER BY createdAt ASC',
["RT @%", this.account.username, daysOldTimestamp],
"all"
) as XTweetRow[];
}

log.debug("XAccountController.deleteTweetsStart", tweets);
// log.debug("XAccountController.deleteTweetsStart", tweets);
return {
tweets: tweets.map((row) => (convertTweetRowToXTweetItem(row))),
};
}

// Returns the count of tweets that are not archived
// If total is true, return the total count of tweets not archived
// Otherwise, return the count of tweets not archived that will be deleted
async deleteTweetsCountNotArchived(total: boolean): Promise<number> {
if (!this.db) {
this.initDB();
}

if (!this.account) {
throw new Error("Account not found");
}

// Select just the tweets that need to be deleted based on the settings
let count: Sqlite3Count;

if (total) {
// Count all non-deleted, non-archived tweets, with no filters
count = exec(
this.db,
'SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedAt IS NULL AND text NOT LIKE ? AND username = ?',
["RT @%", this.account.username],
"get"
) as Sqlite3Count;
} else {
const daysOldTimestamp = this.account.deleteTweetsDaysOldEnabled ? getTimestampDaysAgo(this.account.deleteTweetsDaysOld) : getTimestampDaysAgo(0);
if (this.account.deleteTweetsLikesThresholdEnabled && this.account.deleteTweetsRetweetsThresholdEnabled) {
// Both likes and retweets thresholds
count = exec(
this.db,
'SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? AND retweetCount <= ?',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsLikesThreshold, this.account.deleteTweetsRetweetsThreshold],
"get"
) as Sqlite3Count;
} else if (this.account.deleteTweetsLikesThresholdEnabled && !this.account.deleteTweetsRetweetsThresholdEnabled) {
// Just likes threshold
count = exec(
this.db,
'SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ?',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsLikesThreshold],
"get"
) as Sqlite3Count;
} else if (!this.account.deleteTweetsLikesThresholdEnabled && this.account.deleteTweetsRetweetsThresholdEnabled) {
// Just retweets threshold
count = exec(
this.db,
'SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND retweetCount <= ?',
["RT @%", this.account.username, daysOldTimestamp, this.account.deleteTweetsRetweetsThreshold],
"get"
) as Sqlite3Count;
} else {
// Neither likes nor retweets threshold
count = exec(
this.db,
'SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ?',
["RT @%", this.account.username, daysOldTimestamp],
"get"
) as Sqlite3Count;
}
}

return count.count;
}

// When you start deleting retweets, return a list of tweets to delete
async deleteRetweetsStart(): Promise<XDeleteTweetsStartResponse> {
if (!this.db) {
Expand All @@ -1375,12 +1468,12 @@ export class XAccountController {
const daysOldTimestamp = this.account.deleteRetweetsDaysOldEnabled ? getTimestampDaysAgo(this.account.deleteRetweetsDaysOld) : getTimestampDaysAgo(0);
const tweets: XTweetRow[] = exec(
this.db,
'SELECT id, tweetID, username FROM tweet WHERE deletedAt IS NULL AND text LIKE ? AND createdAt <= ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND text LIKE ? AND createdAt <= ? ORDER BY createdAt ASC',
["RT @%", daysOldTimestamp],
"all"
) as XTweetRow[];

log.debug("XAccountController.deleteRetweetsStart", tweets);
// log.debug("XAccountController.deleteRetweetsStart", tweets);
return {
tweets: tweets.map((row) => (convertTweetRowToXTweetItem(row))),
};
Expand All @@ -1399,12 +1492,12 @@ export class XAccountController {
// Select just the tweets that need to be unliked based on the settings
const tweets: XTweetRow[] = exec(
this.db,
'SELECT id, tweetID, username FROM tweet WHERE deletedAt IS NULL AND isLiked = ? ORDER BY createdAt DESC',
'SELECT tweetID, text, likeCount, retweetCount, createdAt FROM tweet WHERE deletedAt IS NULL AND isLiked = ? ORDER BY createdAt ASC',
[1],
"all"
) as XTweetRow[];

log.debug("XAccountController.deleteLikesStart", tweets);
// log.debug("XAccountController.deleteLikesStart", tweets);
return {
tweets: tweets.map((row) => (convertTweetRowToXTweetItem(row))),
};
Expand Down Expand Up @@ -2051,6 +2144,10 @@ export class XAccountController {
};
}

async getCookie(name: string): Promise<string | null> {
return this.cookies[name] || null;
}

async getConfig(key: string): Promise<string | null> {
return getConfig(key, this.db);
}
Expand Down Expand Up @@ -2396,6 +2493,15 @@ export const defineIPCX = () => {
}
});

ipcMain.handle('X:deleteTweetsCountNotArchived', async (_, accountID: number, total: boolean): Promise<number> => {
try {
const controller = getXAccountController(accountID);
return await controller.deleteTweetsCountNotArchived(total);
} catch (error) {
throw new Error(packageExceptionForReport(error as Error));
}
});

ipcMain.handle('X:deleteRetweetsStart', async (_, accountID: number): Promise<XDeleteTweetsStartResponse> => {
try {
const controller = getXAccountController(accountID);
Expand Down Expand Up @@ -2450,6 +2556,15 @@ export const defineIPCX = () => {
}
});

ipcMain.handle('X:getCookie', async (_, accountID: number, name: string): Promise<string | null> => {
try {
const controller = getXAccountController(accountID);
return await controller.getCookie(name);
} catch (error) {
throw new Error(packageExceptionForReport(error as Error));
}
});

ipcMain.handle('X:getConfig', async (_, accountID: number, key: string): Promise<string | null> => {
try {
const controller = getXAccountController(accountID);
Expand Down
22 changes: 14 additions & 8 deletions src/database.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,14 @@ export const runMainMigrations = () => {
deleteMyData BOOLEAN DEFAULT 0,
archiveTweets BOOLEAN DEFAULT 1,
archiveTweetsHTML BOOLEAN DEFAULT 0,
archiveLikes BOOLEAN DEFAULT 0,
archiveLikes BOOLEAN DEFAULT 1,
archiveDMs BOOLEAN DEFAULT 1,
deleteTweets BOOLEAN DEFAULT 1,
deleteTweetsDaysOld INTEGER DEFAULT 0,
deleteTweetsLikesThresholdEnabled BOOLEAN DEFAULT 0,
deleteTweetsLikesThreshold INTEGER DEFAULT 20,
deleteTweetsRetweetsThresholdEnabled BOOLEAN DEFAULT 0,
deleteTweetsRetweetsThreshold INTEGER DEFAULT 20,
deleteTweetsArchiveEnabled BOOLEAN DEFAULT 0,
deleteRetweets BOOLEAN DEFAULT 1,
deleteRetweetsDaysOld INTEGER DEFAULT 0,
deleteLikes BOOLEAN DEFAULT 0,
Expand Down Expand Up @@ -150,7 +149,14 @@ export const runMainMigrations = () => {
status TEXT DEFAULT 'new'
);`,
]
}
},
// Add archiveMyData to xAccount
{
name: "add archiveMyData to xAccount",
sql: [
`ALTER TABLE xAccount ADD COLUMN archiveMyData BOOLEAN DEFAULT 0;`,
]
},
]);
}

Expand Down Expand Up @@ -186,6 +192,7 @@ interface XAccountRow {
importFromArchive: boolean;
saveMyData: boolean;
deleteMyData: boolean;
archiveMyData: boolean;
archiveTweets: boolean;
archiveTweetsHTML: boolean;
archiveLikes: boolean;
Expand All @@ -197,7 +204,6 @@ interface XAccountRow {
deleteTweetsLikesThreshold: number;
deleteTweetsRetweetsThresholdEnabled: boolean;
deleteTweetsRetweetsThreshold: number;
deleteTweetsArchiveEnabled: boolean;
deleteRetweets: boolean;
deleteRetweetsDaysOldEnabled: boolean;
deleteRetweetsDaysOld: number;
Expand Down Expand Up @@ -380,6 +386,7 @@ export const getXAccount = (id: number): XAccount | null => {
importFromArchive: !!row.importFromArchive,
saveMyData: !!row.saveMyData,
deleteMyData: !!row.deleteMyData,
archiveMyData: !!row.archiveMyData,
archiveTweets: !!row.archiveTweets,
archiveTweetsHTML: !!row.archiveTweetsHTML,
archiveLikes: !!row.archiveLikes,
Expand All @@ -391,7 +398,6 @@ export const getXAccount = (id: number): XAccount | null => {
deleteTweetsLikesThreshold: row.deleteTweetsLikesThreshold,
deleteTweetsRetweetsThresholdEnabled: !!row.deleteTweetsRetweetsThresholdEnabled,
deleteTweetsRetweetsThreshold: row.deleteTweetsRetweetsThreshold,
deleteTweetsArchiveEnabled: !!row.deleteTweetsArchiveEnabled,
deleteRetweets: !!row.deleteRetweets,
deleteRetweetsDaysOldEnabled: !!row.deleteRetweetsDaysOldEnabled,
deleteRetweetsDaysOld: row.deleteRetweetsDaysOld,
Expand Down Expand Up @@ -420,6 +426,7 @@ export const getXAccounts = (): XAccount[] => {
importFromArchive: !!row.importFromArchive,
saveMyData: !!row.saveMyData,
deleteMyData: !!row.deleteMyData,
archiveMyData: !!row.archiveMyData,
archiveTweets: !!row.archiveTweets,
archiveTweetsHTML: !!row.archiveTweetsHTML,
archiveLikes: !!row.archiveLikes,
Expand All @@ -431,7 +438,6 @@ export const getXAccounts = (): XAccount[] => {
deleteTweetsLikesThreshold: row.deleteTweetsLikesThreshold,
deleteTweetsRetweetsThresholdEnabled: !!row.deleteTweetsRetweetsThresholdEnabled,
deleteTweetsRetweetsThreshold: row.deleteTweetsRetweetsThreshold,
deleteTweetsArchiveEnabled: !!row.deleteTweetsArchiveEnabled,
deleteRetweets: !!row.deleteRetweets,
deleteRetweetsDaysOldEnabled: !!row.deleteRetweetsDaysOldEnabled,
deleteRetweetsDaysOld: row.deleteRetweetsDaysOld,
Expand Down Expand Up @@ -468,6 +474,7 @@ export const saveXAccount = (account: XAccount) => {
importFromArchive = ?,
saveMyData = ?,
deleteMyData = ?,
archiveMyData = ?,
archiveTweets = ?,
archiveTweetsHTML = ?,
archiveLikes = ?,
Expand All @@ -479,7 +486,6 @@ export const saveXAccount = (account: XAccount) => {
deleteTweetsLikesThreshold = ?,
deleteTweetsRetweetsThresholdEnabled = ?,
deleteTweetsRetweetsThreshold = ?,
deleteTweetsArchiveEnabled = ?,
deleteRetweets = ?,
deleteRetweetsDaysOldEnabled = ?,
deleteRetweetsDaysOld = ?,
Expand All @@ -497,6 +503,7 @@ export const saveXAccount = (account: XAccount) => {
account.importFromArchive ? 1 : 0,
account.saveMyData ? 1 : 0,
account.deleteMyData ? 1 : 0,
account.archiveMyData ? 1 : 0,
account.archiveTweets ? 1 : 0,
account.archiveTweetsHTML ? 1 : 0,
account.archiveLikes ? 1 : 0,
Expand All @@ -508,7 +515,6 @@ export const saveXAccount = (account: XAccount) => {
account.deleteTweetsLikesThreshold,
account.deleteTweetsRetweetsThresholdEnabled ? 1 : 0,
account.deleteTweetsRetweetsThreshold,
account.deleteTweetsArchiveEnabled ? 1 : 0,
account.deleteRetweets ? 1 : 0,
account.deleteRetweetsDaysOldEnabled ? 1 : 0,
account.deleteRetweetsDaysOld,
Expand Down
Loading

0 comments on commit 9722314

Please sign in to comment.