diff --git a/services/embedding-indexer/package-lock.json b/services/embedding-indexer/package-lock.json index 781a349..3baf27c 100644 --- a/services/embedding-indexer/package-lock.json +++ b/services/embedding-indexer/package-lock.json @@ -14,6 +14,7 @@ "pg": "^8.11.3" }, "devDependencies": { + "@types/axios": "^0.14.0", "@types/node": "^20.11.16", "@types/node-cron": "^3.0.11", "@types/pg": "^8.11.0", @@ -83,6 +84,16 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, + "node_modules/@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha512-KqQnQbdYE54D7oa/UmYVMZKq7CO4l8DEENzOKc4aBRwxCXSlJXGz83flFx5L7AWrOQnmuN3kVsRdt+GZPPjiVQ==", + "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!", + "dev": true, + "dependencies": { + "axios": "*" + } + }, "node_modules/@types/node": { "version": "20.11.16", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.16.tgz", diff --git a/services/embedding-indexer/package.json b/services/embedding-indexer/package.json index 5867c64..1c91c7a 100644 --- a/services/embedding-indexer/package.json +++ b/services/embedding-indexer/package.json @@ -11,6 +11,7 @@ "author": "Spenser Bushey", "license": "ISC", "devDependencies": { + "@types/axios": "^0.14.0", "@types/node": "^20.11.16", "@types/node-cron": "^3.0.11", "@types/pg": "^8.11.0", diff --git a/services/embedding-indexer/src/reindex.ts b/services/embedding-indexer/src/reindex.ts index f9e43f6..30380cb 100644 --- a/services/embedding-indexer/src/reindex.ts +++ b/services/embedding-indexer/src/reindex.ts @@ -1,11 +1,16 @@ import { query } from "./db"; +import Axios from "axios"; export const reindex = async () => { const results = await query( + // TODO: Remove limit ` - SELECT id - FROM image - WHERE embedding IS NULL + SELECT id + FROM image + WHERE + embedding IS NULL + AND NOT index_error + LIMIT 1 `, [] ); @@ -13,5 +18,26 @@ export const reindex = async () => { }; const indexItem = async (id: string) => { - console.log(`Indexing item id ${id}`) -} + console.log(`Indexing item id ${id}`); + const image_url = `https://download.audiobookcovers.com/png/1000/${id}.png`; + const clip_base_url = process.env.CLIP_API_URL || "http://localhost:8080"; + const clip_url = `${clip_base_url}/embedding/image/url?url=${encodeURIComponent( + image_url + )}`; + const embeddingInsertQuery = ` + UPDATE image + SET test_embedding = $1::vector + WHERE id = $2 + `; + const indexErrorQuery = ` + UPDATE image + SET index_error = true + WHERE id = $1 + `; + Axios.get(clip_url) + .then((result) => JSON.stringify(result.data)) + .then((embedding) => { + query(embeddingInsertQuery, [embedding, id]); + }) + .catch(() => query(indexErrorQuery, [id])); +}; diff --git a/services/embedding-indexer/tsconfig.json b/services/embedding-indexer/tsconfig.json index f6bec8e..ec0e897 100644 --- a/services/embedding-indexer/tsconfig.json +++ b/services/embedding-indexer/tsconfig.json @@ -27,7 +27,7 @@ /* Modules */ "module": "commonjs", /* Specify what module code is generated. */ "rootDir": "./src", /* Specify the root folder within your source files. */ - // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */