Skip to content

Commit

Permalink
Merge pull request #21 from shashank40/bugfix/mendableai/data-connect…
Browse files Browse the repository at this point in the history
…ors-issue-#20-notion-rate-limit-catch

Fix: Fix Notion connector 429 rate-limit error issue #20 /claim #20
  • Loading branch information
nickscamara authored Mar 6, 2024
2 parents 8879316 + f8b1d3c commit 0e20cdd
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 14 deletions.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"build-tsup": "tsup",
"pre-publish": "npm run test",
"publish": "npm run build-tsup && npm publish --access public",
"beta-publish": "npm run build-tsup && npm publish --tag beta --access public",
"format": "prettier --write \"src/**/*.(js|ts)\"",
"lint": "eslint src --ext .js,.ts",
"lint:fix": "eslint src --fix --ext .js,.ts",
Expand Down
47 changes: 33 additions & 14 deletions src/providers/Notion/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Nango } from "@nangohq/node";
import { Client } from "@notionhq/client";
import { APIErrorCode, Client } from "@notionhq/client";
import { DataProvider } from "../DataProvider";
import { Document } from "../../entities/Document";
import { NangoAuthorizationOptions } from "../GoogleDrive";
Expand All @@ -10,6 +10,7 @@ import {
RichTextItemResponse,
SearchResponse,
} from "@notionhq/client/build/src/api-endpoints";
import rateLimitDelay from "../../utils/RateLimitDelay";

export type NotionInputOptions = object;

Expand Down Expand Up @@ -50,11 +51,19 @@ async function recursiveBlockChildren(
): Promise<NotionBlockWithChildren[]> {
const blocks: NotionBlockWithChildren[] = [];
let req: ListBlockChildrenResponse;
const i = 0;

do {
req = await notion.blocks.children.list({ block_id });

try {
req = await notion.blocks.children.list({ block_id });
} catch (error) {
if (error.code === APIErrorCode.RateLimited) {
await rateLimitDelay(error.headers.get("retry-after"));
continue;
}
// Handle other errors
console.error("Error occurred:", error);
break; // Exit the loop if an error occurs
}
const results = req.results as BlockObjectResponse[];

for (const block of results) {
Expand All @@ -68,7 +77,7 @@ async function recursiveBlockChildren(
: [],
});
}
} while (req.has_more);
} while (req && req.has_more);

return blocks;
}
Expand Down Expand Up @@ -312,14 +321,24 @@ export class NotionDataProvider implements DataProvider<NotionOptions> {
let req: SearchResponse = undefined;

do {
req = await this.notion.search({
start_cursor: req?.next_cursor,
filter: {
property: "object",
value: "page",
},
page_size: 100,
});
try {
req = await this.notion.search({
start_cursor: req?.next_cursor,
filter: {
property: "object",
value: "page",
},
page_size: 100,
});
} catch (error) {
if (error.code === APIErrorCode.RateLimited) {
await rateLimitDelay(error.headers.get("retry-after"));
continue;
}

console.error("Error occurred:", error);
break;
}

const pages = req.results.filter(
(x) => x.object === "page"
Expand All @@ -335,7 +354,7 @@ export class NotionDataProvider implements DataProvider<NotionOptions> {
);

all.push(...pagesWithBlocks);
} while (req.has_more);
} while (req && req.has_more);

const pages = all.map(({ page, blocks }) => {
return {
Expand Down
8 changes: 8 additions & 0 deletions src/utils/RateLimitDelay.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
export default async function rateLimitDelay(
exponentialBackoff: number
): Promise<void> {
console.log(`Rate limited, retrying in ${exponentialBackoff} seconds...`);
await new Promise((resolve) =>
setTimeout(resolve, exponentialBackoff * 1000)
);
}

0 comments on commit 0e20cdd

Please sign in to comment.