diff --git a/config/config.js b/config/config.js index ae64754..65c8175 100644 --- a/config/config.js +++ b/config/config.js @@ -11,7 +11,7 @@ console.log('Loaded environment variables:', { }); module.exports = { - PAPERLESS_AI_VERSION: '2.2.1', + PAPERLESS_AI_VERSION: '2.3.0', CONFIGURED: false, predefinedMode: process.env.PROCESS_PREDEFINED_DOCUMENTS, paperless: { @@ -25,6 +25,11 @@ module.exports = { apiUrl: process.env.OLLAMA_API_URL || 'http://localhost:11434', model: process.env.OLLAMA_MODEL || 'llama2' }, + custom: { + apiUrl: process.env.CUSTOM_BASE_URL || '', + apiKey: process.env.CUSTOM_API_KEY || '', + model: process.env.CUSTOM_MODEL || '' + }, aiProvider: process.env.AI_PROVIDER || 'openai', scanInterval: process.env.SCAN_INTERVAL || '*/30 * * * *', specialPromptPreDefinedTags: `You are a document analysis AI. You will analyze the document. diff --git a/public/js/settings.js b/public/js/settings.js index 909609c..98b3dfc 100644 --- a/public/js/settings.js +++ b/public/js/settings.js @@ -28,7 +28,6 @@ class ThemeManager { } } -// Form Management class FormManager { constructor() { this.form = document.getElementById('setupForm'); @@ -64,25 +63,50 @@ class FormManager { const provider = this.aiProvider.value; const openaiSettings = document.getElementById('openaiSettings'); const ollamaSettings = document.getElementById('ollamaSettings'); + const customSettings = document.getElementById('customSettings'); + + // Get all provider-specific fields const openaiKey = document.getElementById('openaiKey'); const ollamaUrl = document.getElementById('ollamaUrl'); const ollamaModel = document.getElementById('ollamaModel'); + const customBaseUrl = document.getElementById('customBaseUrl'); + const customApiKey = document.getElementById('customApiKey'); + const customModel = document.getElementById('customModel'); - if (provider === 'openai') { - openaiSettings.classList.remove('hidden'); - ollamaSettings.classList.add('hidden'); - openaiKey.required = true; - ollamaUrl.required = false; - ollamaModel.required = false; - } else { - openaiSettings.classList.add('hidden'); - ollamaSettings.classList.remove('hidden'); - openaiKey.required = false; - ollamaUrl.required = true; - ollamaModel.required = true; + // Hide all settings sections first + openaiSettings.classList.add('hidden'); + ollamaSettings.classList.add('hidden'); + customSettings.classList.add('hidden'); + + // Reset all required fields + openaiKey.required = false; + ollamaUrl.required = false; + ollamaModel.required = false; + customBaseUrl.required = false; + customApiKey.required = false; + customModel.required = false; + + // Show and set required fields based on selected provider + switch (provider) { + case 'openai': + openaiSettings.classList.remove('hidden'); + openaiKey.required = true; + break; + case 'ollama': + ollamaSettings.classList.remove('hidden'); + ollamaUrl.required = true; + ollamaModel.required = true; + break; + case 'custom': + customSettings.classList.remove('hidden'); + customBaseUrl.required = true; + customApiKey.required = true; + customModel.required = true; + break; } } + // Rest of the class methods remain the same toggleTagsInput() { const showTags = this.showTags.value; const tagsInputSection = document.getElementById('tagsInputSection'); diff --git a/public/js/setup.js b/public/js/setup.js index d40440e..c8bf09d 100644 --- a/public/js/setup.js +++ b/public/js/setup.js @@ -30,7 +30,6 @@ class ThemeManager { } } -// Form Management class FormManager { constructor() { this.form = document.getElementById('setupForm'); @@ -73,22 +72,46 @@ class FormManager { const provider = this.aiProvider.value; const openaiSettings = document.getElementById('openaiSettings'); const ollamaSettings = document.getElementById('ollamaSettings'); + const customSettings = document.getElementById('customSettings'); + + // Get all required fields const openaiKey = document.getElementById('openaiKey'); const ollamaUrl = document.getElementById('ollamaUrl'); const ollamaModel = document.getElementById('ollamaModel'); + const customBaseUrl = document.getElementById('customBaseUrl'); + const customApiKey = document.getElementById('customApiKey'); + const customModel = document.getElementById('customModel'); - if (provider === 'openai') { - openaiSettings.style.display = 'block'; - ollamaSettings.style.display = 'none'; - openaiKey.required = true; - ollamaUrl.required = false; - ollamaModel.required = false; - } else { - openaiSettings.style.display = 'none'; - ollamaSettings.style.display = 'block'; - openaiKey.required = false; - ollamaUrl.required = true; - ollamaModel.required = true; + // Hide all settings first + openaiSettings.style.display = 'none'; + ollamaSettings.style.display = 'none'; + customSettings.style.display = 'none'; + + // Reset all required attributes + openaiKey.required = false; + ollamaUrl.required = false; + ollamaModel.required = false; + customBaseUrl.required = false; + customApiKey.required = false; + customModel.required = false; + + // Show and set required fields based on selected provider + switch (provider) { + case 'openai': + openaiSettings.style.display = 'block'; + openaiKey.required = true; + break; + case 'ollama': + ollamaSettings.style.display = 'block'; + ollamaUrl.required = true; + ollamaModel.required = true; + break; + case 'custom': + customSettings.style.display = 'block'; + customBaseUrl.required = true; + customApiKey.required = true; + customModel.required = true; + break; } } diff --git a/routes/setup.js b/routes/setup.js index a912eaf..cbebeb5 100644 --- a/routes/setup.js +++ b/routes/setup.js @@ -808,9 +808,14 @@ router.post('/setup', express.json(), async (req, res) => { username, password, paperlessUsername, - useExistingData + useExistingData, + customApiKey, + customBaseUrl, + customModel } = req.body; + console.log('Setup request received:', req.body); + const normalizeArray = (value) => { if (!value) return []; if (Array.isArray(value)) return value; @@ -862,7 +867,10 @@ router.post('/setup', express.json(), async (req, res) => { PROMPT_TAGS: normalizeArray(promptTags), USE_EXISTING_DATA: useExistingData || 'no', API_KEY: apiToken, - JWT_SECRET: jwtToken + JWT_SECRET: jwtToken, + CUSTOM_API_KEY: customApiKey || '', + CUSTOM_BASE_URL: customBaseUrl || '', + CUSTOM_MODEL: customModel || '' }; // Validate AI provider config @@ -884,8 +892,19 @@ router.post('/setup', express.json(), async (req, res) => { } config.OLLAMA_API_URL = ollamaUrl || 'http://localhost:11434'; config.OLLAMA_MODEL = ollamaModel || 'llama3.2'; + }else if (aiProvider === 'custom') { + console.log('Custom AI provider selected'); + const isCustomValid = await setupService.validateCustomConfig(customBaseUrl, customApiKey, customModel); + if (!isCustomValid) { + return res.status(400).json({ + error: 'Custom connection failed. Please check URL, API Key and Model.' + }); + } + config.CUSTOM_BASE_URL = customBaseUrl; + config.CUSTOM_API_KEY = customApiKey; + config.CUSTOM_MODEL = customModel; } - + // Save configuration await setupService.saveConfig(config); const hashedPassword = await bcrypt.hash(password, 15); @@ -929,7 +948,10 @@ router.post('/settings', express.json(), async (req, res) => { usePromptTags, promptTags, paperlessUsername, - useExistingData + useExistingData, + customApiKey, + customBaseUrl, + customModel } = req.body; const currentConfig = { @@ -950,7 +972,10 @@ router.post('/settings', express.json(), async (req, res) => { USE_PROMPT_TAGS: process.env.USE_PROMPT_TAGS || 'no', PROMPT_TAGS: process.env.PROMPT_TAGS || '', USE_EXISTING_DATA: process.env.USE_EXISTING_DATA || 'no', - API_KEY: process.env.API_KEY || '' + API_KEY: process.env.API_KEY || '', + CUSTOM_API_KEY: process.env.CUSTOM_API_KEY || '', + CUSTOM_BASE_URL: process.env.CUSTOM_BASE_URL || '', + CUSTOM_MODEL: process.env.CUSTOM_MODEL || '' }; const normalizeArray = (value) => { @@ -1017,6 +1042,9 @@ router.post('/settings', express.json(), async (req, res) => { if (usePromptTags) updatedConfig.USE_PROMPT_TAGS = usePromptTags; if (promptTags) updatedConfig.PROMPT_TAGS = normalizeArray(promptTags); if (useExistingData) updatedConfig.USE_EXISTING_DATA = useExistingData; + if (customApiKey) updatedConfig.CUSTOM_API_KEY = customApiKey; + if (customBaseUrl) updatedConfig.CUSTOM_BASE_URL = customBaseUrl; + if (customModel) updatedConfig.CUSTOM_MODEL = customModel; let apiToken = ''; //generate a random secure api token diff --git a/services/aiServiceFactory.js b/services/aiServiceFactory.js index 53e76af..b8c20a0 100644 --- a/services/aiServiceFactory.js +++ b/services/aiServiceFactory.js @@ -1,6 +1,7 @@ const config = require('../config/config'); const openaiService = require('./openaiService'); const ollamaService = require('./ollamaService'); +const customService = require('./customService'); class AIServiceFactory { static getService() { @@ -10,6 +11,8 @@ class AIServiceFactory { case 'openai': default: return openaiService; + case 'custom': + return customService; } } } diff --git a/services/customService.js b/services/customService.js new file mode 100644 index 0000000..283c951 --- /dev/null +++ b/services/customService.js @@ -0,0 +1,314 @@ +const OpenAI = require('openai'); +const config = require('../config/config'); +const tiktoken = require('tiktoken'); +const paperlessService = require('./paperlessService'); +const fs = require('fs').promises; +const path = require('path'); + +class CustomOpenAIService { + constructor() { + this.client = null; + this.tokenizer = null; + } + + initialize() { + if (!this.client && config.aiProvider === 'custom') { + this.client = new OpenAI({ + baseURL: config.custom.apiUrl, + apiKey: config.custom.apiKey + }); + } + } + + // Calculate tokens for a given text + async calculateTokens(text) { + if (!this.tokenizer) { + // Use the appropriate model encoding + this.tokenizer = await tiktoken.encoding_for_model(process.env.OPENAI_MODEL || "gpt-4o-mini"); + } + return this.tokenizer.encode(text).length; + } + + // Calculate tokens for a given text + async calculateTotalPromptTokens(systemPrompt, additionalPrompts = []) { + let totalTokens = 0; + + // Count tokens for system prompt + totalTokens += await this.calculateTokens(systemPrompt); + + // Count tokens for additional prompts + for (const prompt of additionalPrompts) { + if (prompt) { // Only count if prompt exists + totalTokens += await this.calculateTokens(prompt); + } + } + + // Add tokens for message formatting (approximately 4 tokens per message) + const messageCount = 1 + additionalPrompts.filter(p => p).length; // Count system + valid additional prompts + totalTokens += messageCount * 4; + + return totalTokens; + } + + // Truncate text to fit within token limit + async truncateToTokenLimit(text, maxTokens) { + const tokens = await this.calculateTokens(text); + if (tokens <= maxTokens) return text; + + // Simple truncation strategy - could be made more sophisticated + const ratio = maxTokens / tokens; + return text.slice(0, Math.floor(text.length * ratio)); + } + + async analyzeDocument(content, existingTags = [], existingCorrespondentList = [], id) { + const cachePath = path.join('./public/images', `${id}.png`); + try { + this.initialize(); + const now = new Date(); + const timestamp = now.toLocaleString('de-DE', { dateStyle: 'short', timeStyle: 'short' }); + + if (!this.client) { + throw new Error('Custom OpenAI client not initialized'); + } + + // Handle thumbnail caching + try { + await fs.access(cachePath); + console.log('[DEBUG] Thumbnail already cached'); + } catch (err) { + console.log('Thumbnail not cached, fetching from Paperless'); + + const thumbnailData = await paperlessService.getThumbnailImage(id); + + if (!thumbnailData) { + console.warn('Thumbnail nicht gefunden'); + } + + await fs.mkdir(path.dirname(cachePath), { recursive: true }); + await fs.writeFile(cachePath, thumbnailData); + } + + // Format existing tags + const existingTagsList = existingTags + .map(tag => tag.name) + .join(', '); + + + let systemPrompt = ''; + let promptTags = ''; + const model = config.custom.model; + // Get system prompt and model + if(process.env.USE_EXISTING_DATA === 'yes') { + systemPrompt = ` + Prexisting tags: ${existingTagsList}\n\n + Prexisiting correspondent: ${existingCorrespondentList}\n\n + ` + process.env.SYSTEM_PROMPT + '\n\n' + config.mustHavePrompt; + promptTags = ''; + } else { + systemPrompt = process.env.SYSTEM_PROMPT + '\n\n' + config.mustHavePrompt; + promptTags = ''; + } + if (process.env.USE_PROMPT_TAGS === 'yes') { + promptTags = process.env.PROMPT_TAGS; + systemPrompt = ` + Take these tags and try to match one or more to the document content.\n\n + ` + config.specialPromptPreDefinedTags; + } + + // Calculate total prompt tokens including all components + const totalPromptTokens = await this.calculateTotalPromptTokens( + systemPrompt, + process.env.USE_PROMPT_TAGS === 'yes' ? [promptTags] : [] + ); + + // Calculate available tokens + const maxTokens = 128000; // Model's maximum context length + const reservedTokens = totalPromptTokens + 1000; // Reserve for response + const availableTokens = maxTokens - reservedTokens; + + // Truncate content if necessary + const truncatedContent = await this.truncateToTokenLimit(content, availableTokens); + + // Make API request + const response = await this.client.chat.completions.create({ + model: model, + messages: [ + { + role: "system", + content: systemPrompt + }, + { + role: "user", + content: truncatedContent + } + ], + temperature: 0.3, + }); + + // Handle response + if (!response?.choices?.[0]?.message?.content) { + throw new Error('Invalid API response structure'); + } + + // Log token usage + console.log(`[DEBUG] [${timestamp}] OpenAI request sent`); + console.log(`[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`); + + const usage = response.usage; + const mappedUsage = { + promptTokens: usage.prompt_tokens, + completionTokens: usage.completion_tokens, + totalTokens: usage.total_tokens + }; + + let jsonContent = response.choices[0].message.content; + jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); + + let parsedResponse; + try { + parsedResponse = JSON.parse(jsonContent); + } catch (error) { + console.error('Failed to parse JSON response:', error); + throw new Error('Invalid JSON response from API'); + } + + // Validate response structure + if (!parsedResponse || !Array.isArray(parsedResponse.tags) || typeof parsedResponse.correspondent !== 'string') { + throw new Error('Invalid response structure: missing tags array or correspondent string'); + } + + return { + document: parsedResponse, + metrics: mappedUsage, + truncated: truncatedContent.length < content.length + }; + } catch (error) { + console.error('Failed to analyze document:', error); + return { + document: { tags: [], correspondent: null }, + metrics: null, + error: error.message + }; + } + } + + async writePromptToFile(systemPrompt, truncatedContent) { + const filePath = './logs/prompt.txt'; + const maxSize = 10 * 1024 * 1024; + + try { + const stats = await fs.stat(filePath); + if (stats.size > maxSize) { + await fs.unlink(filePath); // Delete the file if is biger 10MB + } + } catch (error) { + if (error.code !== 'ENOENT') { + console.warn('[WARNING] Error checking file size:', error); + } + } + + try { + await fs.appendFile(filePath, systemPrompt + truncatedContent + '\n\n'); + } catch (error) { + console.error('[ERROR] Error writing to file:', error); + } + } + + async analyzePlayground(content, prompt) { + const musthavePrompt = ` + Return the result EXCLUSIVELY as a JSON object. The Tags and Title MUST be in the language that is used in the document.: + { + "title": "xxxxx", + "correspondent": "xxxxxxxx", + "tags": ["Tag1", "Tag2", "Tag3", "Tag4"], + "document_date": "YYYY-MM-DD", + "language": "en/de/es/..." + }`; + + try { + this.initialize(); + const now = new Date(); + const timestamp = now.toLocaleString('de-DE', { dateStyle: 'short', timeStyle: 'short' }); + + if (!this.client) { + throw new Error('OpenAI client not initialized - missing API key'); + } + + // Calculate total prompt tokens including musthavePrompt + const totalPromptTokens = await this.calculateTotalPromptTokens( + prompt + musthavePrompt // Combined system prompt + ); + + // Calculate available tokens + const maxTokens = 128000; + const reservedTokens = totalPromptTokens + 1000; // Reserve for response + const availableTokens = maxTokens - reservedTokens; + + // Truncate content if necessary + const truncatedContent = await this.truncateToTokenLimit(content, availableTokens); + + // Make API request + const response = await this.client.chat.completions.create({ + model: process.env.OPENAI_MODEL, + messages: [ + { + role: "system", + content: prompt + musthavePrompt + }, + { + role: "user", + content: truncatedContent + } + ], + temperature: 0.3, + }); + + // Handle response + if (!response?.choices?.[0]?.message?.content) { + throw new Error('Invalid API response structure'); + } + + // Log token usage + console.log(`[DEBUG] [${timestamp}] OpenAI request sent`); + console.log(`[DEBUG] [${timestamp}] Total tokens: ${response.usage.total_tokens}`); + + const usage = response.usage; + const mappedUsage = { + promptTokens: usage.prompt_tokens, + completionTokens: usage.completion_tokens, + totalTokens: usage.total_tokens + }; + + let jsonContent = response.choices[0].message.content; + jsonContent = jsonContent.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); + + let parsedResponse; + try { + parsedResponse = JSON.parse(jsonContent); + } catch (error) { + console.error('Failed to parse JSON response:', error); + throw new Error('Invalid JSON response from API'); + } + + // Validate response structure + if (!parsedResponse || !Array.isArray(parsedResponse.tags) || typeof parsedResponse.correspondent !== 'string') { + throw new Error('Invalid response structure: missing tags array or correspondent string'); + } + + return { + document: parsedResponse, + metrics: mappedUsage, + truncated: truncatedContent.length < content.length + }; + } catch (error) { + console.error('Failed to analyze document:', error); + return { + document: { tags: [], correspondent: null }, + metrics: null, + error: error.message + }; + } + } +} + +module.exports = new CustomOpenAIService(); \ No newline at end of file diff --git a/services/setupService.js b/services/setupService.js index 492422f..8b0501a 100644 --- a/services/setupService.js +++ b/services/setupService.js @@ -62,6 +62,30 @@ class SetupService { } } + async validateCustomConfig(url, apiKey, model) { + const config = { + baseURL: url, + apiKey: apiKey, + model: model + }; + try { + const openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: config.baseURL, + }); + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: "Test" }], + model: config.model, + }); + return completion.choices && completion.choices.length > 0; + } catch (error) { + console.error('Custom AI validation error:', error.message); + return false; + } + } + + + async validateOllamaConfig(url, model) { try { const response = await axios.post(`${url}/api/generate`, { diff --git a/views/settings.ejs b/views/settings.ejs index 8211122..c074ce9 100644 --- a/views/settings.ejs +++ b/views/settings.ejs @@ -173,19 +173,20 @@ class="w-full px-3 py-2 border border-gray-300 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500"> + - +
+ id="openaiKey" + name="openaiKey" + value="<%= config.OPENAI_API_KEY %>" + class="w-full px-3 py-2 border border-gray-300 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500"> @@ -203,25 +204,62 @@
- +
+ id="ollamaUrl" + name="ollamaUrl" + value="<%= config.OLLAMA_API_URL %>" + class="w-full px-3 py-2 border border-gray-300 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500">
+ id="ollamaModel" + name="ollamaModel" + value="<%= config.OLLAMA_MODEL %>" + class="w-full px-3 py-2 border border-gray-300 rounded-lg shadow-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500"> +
+
+ + +
+
+ + +
+ +
+ +
+ + +
+
+ +
+ +
diff --git a/views/setup.ejs b/views/setup.ejs index 0df0a28..1368d54 100644 --- a/views/setup.ejs +++ b/views/setup.ejs @@ -159,9 +159,12 @@ + - +
@@ -189,7 +192,7 @@
- +
@@ -210,6 +213,43 @@ class="modern-input">
+ + +
+
+ + +
+ +
+ +
+ + +
+
+ +
+ + +
+