From f27e840d58c54289802e6856357f66c520fd1a38 Mon Sep 17 00:00:00 2001 From: Justin Parker Date: Mon, 11 Nov 2024 21:33:24 -0800 Subject: [PATCH] transformers v3 upgrade --- CHANGELOG.md | 6 +++ README.md | 26 +++++------ chunkit.js | 26 +++++++---- config.js | 3 +- embeddingUtils.js | 7 ++- example/example-chunkit.js | 2 +- webui/package-lock.json | 94 ++++++-------------------------------- webui/package.json | 2 +- webui/public/index.html | 11 ++--- webui/public/main.js | 40 ++++++++++++++-- webui/public/models.json | 4 ++ webui/public/styles.css | 24 ++++++---- webui/server.js | 24 +++++----- 13 files changed, 123 insertions(+), 146 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bf998..b4d1cbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [2.3.0] - 2024-11-11 +### Updated +- Updated `transformers.js` from v2 to v3 +- Migrated quantization option from `onnxEmbeddingModelQuantized` (boolean) to `dtype` ('p32', 'p16', 'q8', 'q4') +- Updated Web UI to use new `dtype` option + ## [2.2.5] - 2024-11-08 ### Updated - Updated Web UI styles for smaller screens diff --git a/README.md b/README.md index 6865882..8da374b 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ const myChunks = await chunkit(documents, chunkitOptions); - `combineChunks`: Boolean (optional, default `true`) - Determines whether to reblance and combine chunks into larger ones up to the max token limit. - `combineChunksSimilarityThreshold`: Float (optional, default `0.5`) - Threshold for combining chunks based on similarity during the rebalance and combining phase. - `onnxEmbeddingModel`: String (optional, default `Xenova/all-MiniLM-L6-v2`) - ONNX model used for creating embeddings. - - `onnxEmbeddingModelQuantized`: Boolean (optional, default `true`) - Indicates whether to use a quantized version of the embedding model. + - `dtype`: String (optional, default `fp32`) - Precision of the embedding model (options: `fp32`, `fp16`, `q8`, `q4`). - `localModelPath`: String (optional, default `null`) - Local path to save and load models (example: `./models`). - `modelCacheDir`: String (optional, default `null`) - Directory to cache downloaded models (example: `./models`). - `returnEmbedding`: Boolean (optional, default `false`) - If set to `true`, each chunk will include an embedding vector. This is useful for applications that require semantic understanding of the chunks. The embedding model will be the same as the one specified in `onnxEmbeddingModel`. @@ -88,7 +88,7 @@ The output is an array of chunks, each containing the following properties: - `number_of_chunks`: Integer - The total number of final chunks returned from the input text. - `chunk_number`: Integer - The number of the current chunk. - `model_name`: String - The name of the embedding model used. -- `is_model_quantized`: Boolean - Indicates whether the embedding model is quantized. +- `dtype`: String - The precision of the embedding model used (options: `fp32`, `fp16`, `q8`, `q4`). - `text`: String - The chunked text. - `embedding`: Array - The embedding vector (if `returnEmbedding` is `true`). - `token_length`: Integer - The token length (if `returnTokenLength` is `true`). @@ -213,18 +213,16 @@ The behavior of the `chunkit` function can be finely tuned using several optiona #### Curated ONNX Embedding Models -| Model | Quantized | Link | Size | -| -------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------- | -| nomic-ai/nomic-embed-text-v1.5 | true | [https://huggingface.co/nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | 138 MB | -| nomic-ai/nomic-embed-text-v1.5 | false | [https://huggingface.co/nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | 548 MB | -| Xenova/all-MiniLM-L6-v2 | true | [https://huggingface.co/Xenova/all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) | 23 MB | -| Xenova/all-MiniLM-L6-v2 | false | [https://huggingface.co/Xenova/all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) | 90.4 MB | -| Xenova/paraphrase-multilingual-MiniLM-L12-v2 | true | [https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2) | 118 MB | -| thenlper/gte-base | false | [https://huggingface.co/thenlper/gte-base](https://huggingface.co/thenlper/gte-base) | 436 MB | -| Xenova/all-distilroberta-v1 | true | [https://huggingface.co/Xenova/all-distilroberta-v1](https://huggingface.co/Xenova/all-distilroberta-v1) | 82.1 MB | -| Xenova/all-distilroberta-v1 | false | [https://huggingface.co/Xenova/all-distilroberta-v1](https://huggingface.co/Xenova/all-distilroberta-v1) | 326 MB | -| BAAI/bge-base-en-v1.5 | false | [https://huggingface.co/BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) | 436 MB | -| BAAI/bge-small-en-v1.5 | false | [https://huggingface.co/BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) | 133 MB | +| Model | Precision | Link | Size | +| -------------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------- | +| nomic-ai/nomic-embed-text-v1.5 | fp32, q8 | [https://huggingface.co/nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | 548 MB, 138 MB | +| thenlper/gte-base | fp32 | [https://huggingface.co/thenlper/gte-base](https://huggingface.co/thenlper/gte-base) | 436 MB | +| Xenova/all-MiniLM-L6-v2 | fp32, fp16, q8 | [https://huggingface.co/Xenova/all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) | 23 MB, 45 MB, 90 MB | +| Xenova/paraphrase-multilingual-MiniLM-L12-v2 | fp32, fp16, q8 | [https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2) | 470 MB, 235 MB, 118 MB | +| Xenova/all-distilroberta-v1 | fp32, fp16, q8 | [https://huggingface.co/Xenova/all-distilroberta-v1](https://huggingface.co/Xenova/all-distilroberta-v1) | 326 MB, 163 MB, 82 MB | +| BAAI/bge-base-en-v1.5 | fp32 | [https://huggingface.co/BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) | 436 MB | +| BAAI/bge-small-en-v1.5 | fp32 | [https://huggingface.co/BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) | 133 MB | +| yashvardhan7/snowflake-arctic-embed-m-onnx | fp32 | [https://huggingface.co/yashvardhan7/snowflake-arctic-embed-m-onnx](https://huggingface.co/yashvardhan7/snowflake-arctic-embed-m-onnx) | 436 MB | Each of these parameters allows you to customize the `chunkit` function to better fit the text size, content complexity, and performance requirements of your application. diff --git a/chunkit.js b/chunkit.js index 1fa0a85..ba04623 100644 --- a/chunkit.js +++ b/chunkit.js @@ -29,7 +29,8 @@ export async function chunkit( combineChunks = DEFAULT_CONFIG.COMBINE_CHUNKS, combineChunksSimilarityThreshold = DEFAULT_CONFIG.COMBINE_CHUNKS_SIMILARITY_THRESHOLD, onnxEmbeddingModel = DEFAULT_CONFIG.ONNX_EMBEDDING_MODEL, - onnxEmbeddingModelQuantized = DEFAULT_CONFIG.ONNX_EMBEDDING_MODEL_QUANTIZED, + dtype = DEFAULT_CONFIG.DTYPE, + onnxEmbeddingModelQuantized, localModelPath = DEFAULT_CONFIG.LOCAL_MODEL_PATH, modelCacheDir = DEFAULT_CONFIG.MODEL_CACHE_DIR, returnEmbedding = DEFAULT_CONFIG.RETURN_EMBEDDING, @@ -43,10 +44,13 @@ export async function chunkit( throw new Error('Input must be an array of document objects'); } - // Initialize embedding utilities with paths - const { modelName, isQuantized } = await initializeEmbeddingUtils( + // if legacy boolean is used (onnxEmbeddingModelQuantized), set dtype (model precision) to 'q8' + if (onnxEmbeddingModelQuantized === true) { dtype = 'q8'; } + + // Initialize embedding utilities and set optional paths + const { modelName, dtype: usedDtype } = await initializeEmbeddingUtils( onnxEmbeddingModel, - onnxEmbeddingModelQuantized, + dtype, localModelPath, modelCacheDir ); @@ -96,7 +100,7 @@ export async function chunkit( console.log(`--------------`); console.log(`-- Chunk ${(index + 1)} --`); console.log(`--------------`); - console.log(chunk); + console.log(chunk.substring(0, 50) + '...'); }); } @@ -112,7 +116,7 @@ export async function chunkit( console.log("--------------------"); console.log("Chunk " + (index + 1)); console.log("--------------------"); - console.log(chunk); + console.log(chunk.substring(0, 50) + '...'); }); } } else { @@ -131,7 +135,7 @@ export async function chunkit( number_of_chunks: numberOfChunks, chunk_number: index + 1, model_name: modelName, - is_model_quantized: isQuantized, + dtype: usedDtype, text: prefixedChunk }; @@ -177,7 +181,8 @@ export async function cramit( logging = DEFAULT_CONFIG.LOGGING, maxTokenSize = DEFAULT_CONFIG.MAX_TOKEN_SIZE, onnxEmbeddingModel = DEFAULT_CONFIG.ONNX_EMBEDDING_MODEL, - onnxEmbeddingModelQuantized = DEFAULT_CONFIG.ONNX_EMBEDDING_MODEL_QUANTIZED, + onnxEmbeddingModelQuantized, + dtype = DEFAULT_CONFIG.DTYPE, localModelPath = DEFAULT_CONFIG.LOCAL_MODEL_PATH, modelCacheDir = DEFAULT_CONFIG.MODEL_CACHE_DIR, returnEmbedding = DEFAULT_CONFIG.RETURN_EMBEDDING, @@ -191,6 +196,9 @@ export async function cramit( throw new Error('Input must be an array of document objects'); } + // if legacy boolean is used (onnxEmbeddingModelQuantized), set dtype (model precision) to 'q8' + if (onnxEmbeddingModelQuantized === true) { dtype = 'q8'; } + // Initialize embedding utilities with paths const { modelName, isQuantized } = await initializeEmbeddingUtils( onnxEmbeddingModel, @@ -222,7 +230,7 @@ export async function cramit( console.log(`--------------`); console.log(`-- Chunk ${(index + 1)} --`); console.log(`--------------`); - console.log(chunk); + console.log(chunk.substring(0, 50) + '...'); }); } diff --git a/config.js b/config.js index eafde86..b3a8c84 100644 --- a/config.js +++ b/config.js @@ -8,8 +8,7 @@ export const DEFAULT_CONFIG = { COMBINE_CHUNKS: true, COMBINE_CHUNKS_SIMILARITY_THRESHOLD: 0.6, ONNX_EMBEDDING_MODEL: "Xenova/all-MiniLM-L6-v2", - ONNX_EMBEDDING_MODEL_QUANTIZED: true, - DTYPE: 'q4', + DTYPE: 'fp32', LOCAL_MODEL_PATH: null, MODEL_CACHE_DIR: null, RETURN_EMBEDDING: false, diff --git a/embeddingUtils.js b/embeddingUtils.js index 3174a25..8322d20 100644 --- a/embeddingUtils.js +++ b/embeddingUtils.js @@ -9,7 +9,7 @@ const embeddingCache = new Map(); // -------------------------------------------- export async function initializeEmbeddingUtils( onnxEmbeddingModel, - onnxEmbeddingModelQuantized, + dtype = 'fp32', localModelPath = null, modelCacheDir = null ) { @@ -20,15 +20,14 @@ export async function initializeEmbeddingUtils( tokenizer = await AutoTokenizer.from_pretrained(onnxEmbeddingModel); generateEmbedding = await pipeline('feature-extraction', onnxEmbeddingModel, { - dtype: onnxEmbeddingModelQuantized ? 'q8' : 'fp32', + dtype: dtype, }); - // Clear the embedding cache when initializing with a new model embeddingCache.clear(); return { modelName: onnxEmbeddingModel, - isQuantized: onnxEmbeddingModelQuantized + dtype: dtype }; } diff --git a/example/example-chunkit.js b/example/example-chunkit.js index 28b3b7b..bbe81a8 100644 --- a/example/example-chunkit.js +++ b/example/example-chunkit.js @@ -38,7 +38,7 @@ let myTestChunks = await chunkit( combineChunks: true, // enable rebalancing combineChunksSimilarityThreshold: 0.6, onnxEmbeddingModel: "nomic-ai/nomic-embed-text-v1.5", - onnxEmbeddingModelQuantized: true, + dtype: "q8", localModelPath: "../models", modelCacheDir: "../models", returnEmbedding: false, diff --git a/webui/package-lock.json b/webui/package-lock.json index 6c04e4e..f2913f7 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -1,12 +1,12 @@ { "name": "semantic-chunking-webui", - "version": "1.1.6", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "semantic-chunking-webui", - "version": "1.1.6", + "version": "1.3.0", "license": "ISC", "dependencies": { "cors": "^2.8.5", @@ -18,7 +18,6 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", - "license": "MIT", "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" @@ -30,14 +29,12 @@ "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", - "license": "MIT", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -61,7 +58,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -70,7 +66,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", - "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -89,7 +84,6 @@ "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "license": "MIT", "dependencies": { "safe-buffer": "5.2.1" }, @@ -101,7 +95,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -110,7 +103,6 @@ "version": "0.7.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -118,14 +110,12 @@ "node_modules/cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "license": "MIT" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "node_modules/cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", - "license": "MIT", "dependencies": { "object-assign": "^4", "vary": "^1" @@ -138,7 +128,6 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -147,7 +136,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -164,7 +152,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -173,7 +160,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -183,7 +169,6 @@ "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "license": "BSD-2-Clause", "engines": { "node": ">=12" }, @@ -194,14 +179,12 @@ "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -210,7 +193,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "license": "MIT", "dependencies": { "get-intrinsic": "^1.2.4" }, @@ -222,7 +204,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", "engines": { "node": ">= 0.4" } @@ -230,14 +211,12 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -246,7 +225,6 @@ "version": "4.21.1", "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", - "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -288,7 +266,6 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", - "license": "MIT", "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", @@ -306,7 +283,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -315,7 +291,6 @@ "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -324,7 +299,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -333,7 +307,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", - "license": "MIT", "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -352,7 +325,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "license": "MIT", "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -364,7 +336,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" }, @@ -376,7 +347,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -388,7 +358,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -400,7 +369,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, @@ -412,7 +380,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", - "license": "MIT", "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", @@ -428,7 +395,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -439,14 +405,12 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "license": "MIT", "engines": { "node": ">= 0.10" } @@ -455,7 +419,6 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -464,7 +427,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -473,7 +435,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -482,7 +443,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", "bin": { "mime": "cli.js" }, @@ -494,7 +454,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -503,7 +462,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", "dependencies": { "mime-db": "1.52.0" }, @@ -514,14 +472,12 @@ "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -530,7 +486,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -539,7 +494,6 @@ "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -551,7 +505,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", "dependencies": { "ee-first": "1.1.1" }, @@ -563,7 +516,6 @@ "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -571,14 +523,12 @@ "node_modules/path-to-regexp": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "license": "MIT" + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" @@ -591,7 +541,6 @@ "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.0.6" }, @@ -606,7 +555,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "license": "MIT", "engines": { "node": ">= 0.6" } @@ -615,7 +563,6 @@ "version": "2.5.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "license": "MIT", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -643,20 +590,17 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "MIT" + ] }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", - "license": "MIT", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -680,7 +624,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -688,14 +631,12 @@ "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serve-static": { "version": "1.16.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", - "license": "MIT", "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", @@ -710,7 +651,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -726,14 +666,12 @@ "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "es-errors": "^1.3.0", @@ -751,7 +689,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -760,7 +697,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", "engines": { "node": ">=0.6" } @@ -769,7 +705,6 @@ "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "license": "MIT", "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" @@ -782,7 +717,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -791,7 +725,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", "engines": { "node": ">= 0.4.0" } @@ -800,7 +733,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", "engines": { "node": ">= 0.8" } diff --git a/webui/package.json b/webui/package.json index f3dd0ff..643594f 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "semantic-chunking-webui", - "version": "1.1.6", + "version": "1.3.0", "type": "module", "description": "Web UI for semantic-chunking library", "main": "server.js", diff --git a/webui/public/index.html b/webui/public/index.html index 8a0b81b..587ce36 100644 --- a/webui/public/index.html +++ b/webui/public/index.html @@ -123,13 +123,12 @@

Model Settings

- + +
- - + + +
diff --git a/webui/public/main.js b/webui/public/main.js index 78ac0d2..da8c2a3 100644 --- a/webui/public/main.js +++ b/webui/public/main.js @@ -276,8 +276,8 @@ form.addEventListener('submit', async (e) => { let errorMessage = error.message; if (errorMessage.includes('Could not locate file:')) { - errorMessage += '

Some models may not have both a quantized & non-quantized version,'; - errorMessage += '
please toggle this option and try again, or choose a different model'; + errorMessage += '

Not all models have all precision options available.'; + errorMessage += '
Please try a different precision level and/or model and try again.'; } showToast(errorMessage); @@ -349,7 +349,9 @@ getCodeBtn.onclick = () => { // Generate Code function function generateCode(formData) { - // No need to convert checkbox values since they're already booleans + const dtypeValues = ['fp32', 'fp16', 'q8', 'q4']; + const dtype = dtypeValues[parseInt(formData.dtype)]; + return `// import the semantic-chunking library import { chunkit } from 'semantic-chunking'; @@ -374,7 +376,7 @@ const myChunks = await chunkit( combineChunks: ${formData.combineChunks}, combineChunksSimilarityThreshold: ${formData.combineChunksSimilarityThreshold}, onnxEmbeddingModel: "${formData.onnxEmbeddingModel}", - onnxEmbeddingModelQuantized: ${formData.onnxEmbeddingModelQuantized}, + dtype: "${dtype}", localModelPath: "./models", modelCacheDir: "./models", returnEmbedding: ${formData.returnEmbedding}, @@ -493,4 +495,32 @@ processingTimeSpan.parentNode.insertBefore(resizeToggle, processingTimeSpan.next resizeToggle.addEventListener('click', () => { resultsJson.classList.toggle('wrapped'); resizeToggle.classList.toggle('wrapped'); -}); \ No newline at end of file +}); + +// Add this to your existing range input handlers +const dtypeInput = document.getElementById('dtype'); +const dtypeDisplay = dtypeInput.nextElementSibling; + +function updateDtypeDisplay(value) { + const dtypeValues = { + 0: { text: 'fp32 - Full Precision', class: 'precision-full' }, + 1: { text: 'fp16 - Half Precision', class: 'precision-half' }, + 2: { text: 'q8 - 8-bit Quantized', class: 'precision-q8' }, + 3: { text: 'q4 - 4-bit Quantized', class: 'precision-q4' } + }; + + const dtype = dtypeValues[value]; + const number = dtypeDisplay.querySelector('.number'); + const description = dtypeDisplay.querySelector('.description'); + + number.className = `number ${dtype.class}`; + number.textContent = value; + description.className = `description ${dtype.class}`; + description.textContent = dtype.text; +} + +// Initial update +updateDtypeDisplay(dtypeInput.value); + +// Update on change +dtypeInput.addEventListener('input', (e) => updateDtypeDisplay(e.target.value)); \ No newline at end of file diff --git a/webui/public/models.json b/webui/public/models.json index f93ecf3..20ec464 100644 --- a/webui/public/models.json +++ b/webui/public/models.json @@ -27,6 +27,10 @@ { "value": "BAAI/bge-small-en-v1.5", "label": "BAAI/bge-small-en-v1.5" + }, + { + "value": "yashvardhan7/snowflake-arctic-embed-m-onnx", + "label": "yashvardhan7/snowflake-arctic-embed-m-onnx" } ] } diff --git a/webui/public/styles.css b/webui/public/styles.css index 0527670..94b255c 100644 --- a/webui/public/styles.css +++ b/webui/public/styles.css @@ -30,6 +30,10 @@ body { display: grid; } +.margin-top-5 { margin-top: 5px; } +.margin-top-10 { margin-top: 10px; } +.margin-top-15 { margin-top: 15px; } +.margin-top-20 { margin-top: 20px; } .container { display: flex; @@ -325,6 +329,7 @@ input[type="range"]::-webkit-slider-thumb { .value-display .number { min-width: 40px; + text-align: center; } .value-display .description { @@ -333,17 +338,15 @@ input[type="range"]::-webkit-slider-thumb { } /* Similarity level colors */ -.similarity-low { - color: #ff6b6b; -} - -.similarity-moderate { - color: #ffd93d; -} +.similarity-low { color: #ff6b6b; } +.similarity-moderate { color: #ffd93d; } +.similarity-high { color: #6bff84; } -.similarity-high { - color: #6bff84; -} +/* precision level colors */ +.precision-q4 { color: #FF5722; } +.precision-q8 { color: #FFC107; } +.precision-half { color: #8BC34A; } +.precision-full { color: #4CAF50; } /* Switch styling */ .switch { @@ -929,3 +932,4 @@ pre { #resultsJson.wrapped { white-space: pre-wrap; } + diff --git a/webui/server.js b/webui/server.js index d80caef..a46ed68 100644 --- a/webui/server.js +++ b/webui/server.js @@ -33,16 +33,21 @@ app.get('/', (req, res) => { // Chunking API endpoint app.post('/api/chunk', async (req, res) => { try { - const { documentText, documentName, ...options } = req.body; + const { documentText, documentName, dtype, onnxEmbeddingModelQuantized, ...options } = req.body; + + // Convert dtype value to string mapping + const dtypeValues = ['fp32', 'fp16', 'q8', 'q4']; + const dtypeString = dtypeValues[parseInt(dtype)] || 'fp32'; - // Input validation - if (!documentText) { - return res.status(400).json({ error: 'Document text is required' }); - } + // Process the text with new dtype option + const documents = [{ + document_name: documentName || 'sample text', + document_text: documentText + }]; - // Convert string values to appropriate types const processedOptions = { ...options, + dtype: dtypeString, maxTokenSize: parseInt(options.maxTokenSize), similarityThreshold: parseFloat(options.similarityThreshold), dynamicThresholdLowerBound: parseFloat(options.dynamicThresholdLowerBound), @@ -50,7 +55,6 @@ app.post('/api/chunk', async (req, res) => { numSimilaritySentencesLookahead: parseInt(options.numSimilaritySentencesLookahead), combineChunks: options.combineChunks === true, combineChunksSimilarityThreshold: parseFloat(options.combineChunksSimilarityThreshold), - onnxEmbeddingModelQuantized: options.onnxEmbeddingModelQuantized === true, returnEmbedding: options.returnEmbedding === true, returnTokenLength: options.returnTokenLength === true, logging: options.logging === true, @@ -58,13 +62,7 @@ app.post('/api/chunk', async (req, res) => { modelCacheDir: path.join(__dirname, 'models') }; - // Process the text - const documents = [{ - document_name: documentName || 'sample text', - document_text: documentText - }]; const result = await chunkit(documents, processedOptions); - res.json(result); } catch (error) { console.error('Error processing chunk:', error);