Added tesseract to OCR notes selected as text. Translating OCRed text…

… is also possible.
frianasoa · Dec 16, 2024 · d1619a8 · d1619a8
1 parent f3d9634
commit d1619a8
Show file tree

Hide file tree

Showing 17 changed files with 646 additions and 15 deletions.
diff --git a/bootstrap.js b/bootstrap.js
@@ -271,6 +271,8 @@ async function startup({ id, version, resourceURI, rootURI = resourceURI.spec })
 	Services.scriptloader.loadSubScript(rootURI + 'core/format.js');
 	Services.scriptloader.loadSubScript(rootURI + 'core/ai.js');
 	Services.scriptloader.loadSubScript(rootURI + 'core/io.js');
+	Services.scriptloader.loadSubScript(rootURI + 'core/os.js');
+	Services.scriptloader.loadSubScript(rootURI + 'core/ocr.js');
 	Services.scriptloader.loadSubScript(rootURI + 'content/notes/actions.js');
 	Services.scriptloader.loadSubScript(rootURI + 'core/annotations.js');
 	Services.scriptloader.loadSubScript(rootURI + 'core/languages.js');
@@ -311,6 +313,8 @@ async function startup({ id, version, resourceURI, rootURI = resourceURI.spec })
 
 
 	ZeNotes.Format = Format;
+	ZeNotes.Ocr = Ocr;
+	ZeNotes.Os = Os;
 	Annotations.initmenu();
 	Settings.inject();
 	await ZeNotes.main();

diff --git a/content/notes/actions.js b/content/notes/actions.js
@@ -54,21 +54,43 @@ Actions = {
 		Zotero.OpenPDF.openToPage(attachment, annotationpage, annotationkey);
 	},
 
+	tesseractocr(data, annotationid)
+	{
+		var annotation = Zotero.Items.get(annotationid);
+		var currentcomment = annotation.annotationComment;
+		Zotero.ZeNotes.Ocr.tesseract(annotationid).then(txt=>{
+			if(currentcomment==null)
+			{
+				currentcomment = "";
+			}
+			annotation.annotationComment = currentcomment+"\n\n<b>[OCR]</b>\n"+txt+"\n";
+			annotation.saveTx({skipSelect:true}).then(e=>{
+				Zotero.ZeNotes.Ui.reload();
+			});
+		});
+	},
+
 	googletranslate(annotation, direct=false){
 		var tl = Zotero.ZeNotes.Prefs.get("target-language");
 		var currentcomment = annotation.annotationComment;
 		if(currentcomment==null)
 		{
 			currentcomment = "";
 		}
+
+		var maintext = annotation["annotationText"];
+		if(annotation.annotationType=="image")
+		{
+			maintext = Zotero.ZeNotes.Format.annotationpart(currentcomment, "OCR");
+		}
 
 		var mode = "api-key";
 		if(Zotero.ZeNotes.Prefs.getb("google-translate-key")=="")
 		{
 			mode="free-0";
 		}
 
-		Zotero.ZeNotes.Ai.Google.translate(annotation["annotationText"], tl, mode).then(r=>{
+		Zotero.ZeNotes.Ai.Google.translate(maintext, tl, mode).then(r=>{
 			if(direct)
 			{
 				annotation.annotationComment = currentcomment+"\n\n<b>[Translation]</b>\n"+r[0]+"\n";
@@ -102,8 +124,14 @@ Actions = {
 		{
 			currentcomment = "";
 		}
+
+		var maintext = annotation["annotationText"];
+		if(annotation.annotationType=="image")
+		{
+			maintext = Zotero.ZeNotes.Format.annotationpart(currentcomment, "OCR");
+		}
 
-		Zotero.ZeNotes.Ai.DeepL.translate(annotation["annotationText"], tl).then(r=>{
+		Zotero.ZeNotes.Ai.DeepL.translate(maintext, tl).then(r=>{
 			if(direct)
 			{
 				annotation.annotationComment = currentcomment+"\n\n<b>[Translation]</b>\n"+r[0]+"\n";
@@ -137,8 +165,14 @@ Actions = {
 		{
 			currentcomment = "";
 		}
+
+		var maintext = annotation["annotationText"];
+		if(annotation.annotationType=="image")
+		{
+			maintext = Zotero.ZeNotes.Format.annotationpart(currentcomment, "OCR");
+		}
 
-		Zotero.ZeNotes.Ai.Custom.translate(annotation["annotationText"], tl).then(r=>{
+		Zotero.ZeNotes.Ai.Custom.translate(maintext, tl).then(r=>{
 			if(direct)
 			{
 				annotation.annotationComment = currentcomment+"\n\n<b>[Translation]</b>\n"+r[0]+"\n";
@@ -173,8 +207,14 @@ Actions = {
 		{
 			currentcomment = "";
 		}
+
+		var maintext = annotation["annotationText"];
+		if(annotation.annotationType=="image")
+		{
+			maintext = Zotero.ZeNotes.Format.annotationpart(currentcomment, "OCR");
+		}
 
-		Zotero.ZeNotes.Ai.OpenAi.translate(annotation["annotationText"], tl).then(r=>{
+		Zotero.ZeNotes.Ai.OpenAi.translate(maintext, tl).then(r=>{
 			if(direct)
 			{
 				annotation.annotationComment = currentcomment+"\n\n<b>[Translation]</b>\n"+r[0]+"\n";
@@ -297,6 +337,11 @@ Actions = {
 		});
 	},
 
+	localapi()
+	{
+
+	},
+
 	customapicustomprompt(data, target, annotation) {
 		if(Zotero.ZeNotes.Prefs.getb("custom-api-key")=="")
 		{

diff --git a/content/notes/menus.js b/content/notes/menus.js
@@ -132,6 +132,7 @@ Menus = {
 			}
 
 
+
 			if(Zotero.ZeNotes.Prefs.getb("bard-api-key")!="")
 			{
 				items_ai["paraphrase-annotation"]["items"]["paraphrase-bard"] = {name: "Using Bard", icon: "fa-b"};
@@ -157,6 +158,20 @@ Menus = {
 				items_ai["custom-prompt-on-table"]["items"]["custom-prompt-table-custom-api"] = {name: "Using "+name, icon: "fa-c"};
 			}
 
+			if(Zotero.ZeNotes.Os.tesseractpath())
+			{
+				var language = Zotero.ZeNotes.Ocr.languagename();
+
+				items_ai["local-api-ocr"] = 
+				{
+					name: "OCR",
+					icon: "fa-o",
+					items: 
+					{
+						"local-api-ocr-tesseract": {name: "Using Tesseract ["+language+"]", icon: "fa-t"}
+					}
+				}
+			}
 			items_ai["sep-ai-01"] = "---------";
 		}
 
@@ -413,6 +428,14 @@ Menus = {
 				Actions.customapicustomprompt(data, target, annotation);
 			}
 		}
+		else if(key.startsWith("local-api"))
+		{
+			data = Table.celldata(td);
+			if(key.endsWith("-tesseract"))
+			{
+				Actions.tesseractocr(data, annotationid);
+			}
+		}
 
 		else if(key.startsWith("paraphrase"))
 		{

diff --git a/content/notes/notes.js b/content/notes/notes.js
@@ -11,7 +11,6 @@ Notes = {
 			this.body.style.fontSize = fontsize+"px";
 		}
 		this.fontsize = 1;
-
 	},
 
 	isvertical()

diff --git a/content/settings/preferences.js b/content/settings/preferences.js
@@ -3,6 +3,54 @@ if(typeof Zotero_Preferences == 'undefined') {
 	};	
 }
 
+Zotero_Preferences.SettingsUi = {
+	select(sel, options, prefid)
+	{
+		var templateopt = document.createElementNS("http://www.w3.org/1999/xhtml", "option");
+		if (Zotero.platformMajorVersion>=102)
+		{
+			var menulist = document.createXULElement("menulist");
+			var menupopup = document.createXULElement("menupopup");
+			templateopt = document.createXULElement("menuitem");
+			menulist.appendChild(menupopup);
+
+			menulist.onchange = sel.onchange;
+			menulist.setAttribute("id", sel.id);
+
+			menulist.addEventListener("command", function(e){
+				e.currentTarget.onchange(e);
+			});
+
+			sel.parentNode.replaceChild(menulist, sel);
+			sel = menupopup;
+		}
+		for(o of options)
+		{
+			let opt = templateopt.cloneNode(true);
+			opt.innerText = o.label;
+			opt.setAttribute("label", o.label);
+			opt.setAttribute("value", o.value);
+			sel.appendChild(opt);
+		}
+
+		var value = Zotero.ZeNotes.Prefs.get(prefid);
+		Array.from(sel.querySelectorAll("option")).forEach(opt=>{
+			if(value==opt.value)
+			{
+				opt.setAttribute("selected", "true");
+			}
+		})
+
+		Array.from(sel.querySelectorAll("menuitem")).forEach(opt=>{
+			if(value==opt.value)
+			{
+				sel.parentNode.selectedItem = opt;
+			}
+		})
+	}
+}
+
+
 Zotero_Preferences.ZeNotes = {
 	async init(){
 		if(!this.parser)
@@ -103,8 +151,8 @@ Zotero_Preferences.ZeNotes = {
 					templateopt = document.createXULElement("menuitem");
 
 					menulist.setAttribute("id", el.id);
-					menulist.setAttribute("style", el.getAttribute("style"));
-					menulist.style.display = "inline-block";
+					// menulist.setAttribute("style", el.getAttribute("style"));
+					// menulist.style.display = "inline-block";
 					menulist.appendChild(menupopup);
 					p.replaceChild(menulist, el);
 
@@ -326,6 +374,14 @@ Zotero_Preferences.ZeNotes = {
 					["dropbox-ms-per-file", "zn-dropbox-ms-per-file"]
 				];
 				break;
+
+			case 'tesseract':
+				args = [
+					["tesseract-path", "zn-tesseract-path"],
+					["tesseract-language", "zn-tesseract-language"]
+				];
+				Zotero_Preferences.ZeNotes.fillocrlanguages();
+				break;
 
 			case 'prompts':
 				args = [
@@ -595,6 +651,30 @@ Zotero_Preferences.ZeNotes = {
 			);
 			document.insertBefore(pi, document.documentElement);
 		}
+	},
+
+	async fillocrlanguages()
+	{
+		var languages = await Zotero.ZeNotes.Ocr.languages();
+		var sel = document.getElementById("zn-tesseract-language");
+		var options = Object.entries(languages).map(([key, value]) => ({
+		  label: value,
+		  value: key,
+		}));
+		Zotero_Preferences.SettingsUi.select(sel, options, "tesseract-language");
+	},
+
+	async fillocrlanguages2()
+	{
+		var languages = await Zotero.ZeNotes.Ocr.languages();
+		var sel = document.getElementById("zn-tesseract-language");
+		for(let code in languages)
+		{
+			let opt = document.createElementNS("http://www.w3.org/1999/xhtml", "option");
+			opt.innerHTML = languages[code];
+			opt.value = code;
+			sel.appendChild(opt);
+		}
 	}
 }
 

diff --git a/content/settings/preferences.xhtml b/content/settings/preferences.xhtml
@@ -38,6 +38,11 @@
 			<div class="zn-include" data-src="dropbox.xhtml" onload="Zotero_Preferences.ZeNotes.initpanel('dropbox')"> </div>
 		</fieldset>
 
+		<fieldset>
+			<legend class='zn-legend'>Tesseract</legend>
+			<div class="zn-include" data-src="tesseract.xhtml" onload="Zotero_Preferences.ZeNotes.initpanel('tesseract')"> </div>
+		</fieldset>
+
 		<fieldset>
 			<legend class='zn-legend'>Exclude from prompt</legend>
 			<div class="zn-include" data-src="exclude-from-prompt.xhtml" onload="Zotero_Preferences.ZeNotes.initpanel('exclude-from-prompt')"> </div>

diff --git a/content/settings/preferences.xul b/content/settings/preferences.xul
@@ -19,6 +19,7 @@
 	<script src="chrome://zotero/content/include.js"></script>
 	<script src="chrome://zotero/content/charsetMenu.js"></script>
 	<script src="chrome://zenotes/content/settings/preferences.js" type="application/javascript"/>
+	<script src="chrome://zenotes/content/settings/settingsui.js" type="application/javascript"/>
 
 	<script type="application/javascript">
 	<![CDATA[		

diff --git a/content/settings/preferences6.xul b/content/settings/preferences6.xul
@@ -22,6 +22,7 @@
 				<tab id="zn-prompt-settings" label="Custom prompts"/>
 				<tab id="zn-prompt-data-settings" label="Prompts data"/>
 				<tab id="zn-dropbox-panel" label="Dropbox settings"/>
+				<tab id="zn-tesseract-panel" label="Tesseract settings"/>
 				<tab id="zn-global-settings" label="Global setting"/>
 				<tab id="zn-advanced-settings" label="Advanced"/>
 			</tabs>
@@ -75,6 +76,14 @@
 						<box class="zn-include" src="dropbox.xhtml" onload="Zotero_Preferences.ZeNotes.initpanel('dropbox')"> </box>
 					</groupbox>
 				</tabpanel>
+
+				<tabpanel orient="vertical" id="zn-tesseract-panel">
+					<groupbox>
+						<caption label="Tesseract settings"/>
+						<box class="zn-include" src="tesseract.xhtml" onload="Zotero_Preferences.ZeNotes.initpanel('tesseract')"> </box>
+					</groupbox>
+				</tabpanel>
+
 				<tabpanel orient="vertical" id="zn-global-panel">
 					<groupbox>
 						<caption label="Global display"/>
@@ -114,6 +123,7 @@
 	<script src="chrome://zenotes/content/lib/jquery-contextmenu/2.9.2/jquery.contextMenu.min.js" type="application/javascript"></script>
 	<script src="chrome://zenotes/content/settings/zotero.js" type="application/javascript"></script>
 	<script src="chrome://zenotes/content/settings/zntable.js" type="application/javascript"></script>
+	<script src="chrome://zenotes/content/settings/settings-ui.js" type="application/javascript"></script>
 	<script src="chrome://zenotes/content/settings/languages.js" type="application/javascript"></script>
 	<script src="chrome://zenotes/content/settings/preferences.js" type="application/javascript"></script>
 	<script src="chrome://zenotes/content/settings/preferences6.js" type="application/javascript"/>

diff --git a/content/settings/tesseract.xhtml b/content/settings/tesseract.xhtml
@@ -0,0 +1,18 @@
+<div xmlns="http://www.w3.org/1999/xhtml">
+	<table width="100%">
+	<tr>
+		<td>Tesseract exec path</td>
+		<td>
+			<input id="zn-tesseract-path" type="text" onchange="Zotero_Preferences.ZeNotes.setpreference(event, 'tesseract-path');" style="width:100%;" placeholder="Leave empty to use the default path" />
+		</td>
+	</tr>
+	<tr>
+		<td>Tesseract language</td>
+		<td>
+			<select id="zn-tesseract-language" onchange="Zotero_Preferences.ZeNotes.setpreference(event, 'tesseract-language');" style="width:100%;">
+				<option value="">Choose language</option>
+			</select>
+		</td>
+	</tr>	
+	</table>
+</div>
diff --git a/core/format.js b/core/format.js
@@ -59,6 +59,27 @@ Format = {
 			tagged_items: Object.values(tagged_items),
 		}
 	},
+
+	annotationpart(annotationtext, marker="OCR") {
+		const a = `<b>[${marker}]</b>`;
+		const b = `<b>[`;
+
+		// Find the index of 'a'
+		const startIndex = annotationtext.indexOf(a);
+		if (startIndex === -1) return ""; // If 'a' is not found, return empty string
+
+		// Start after 'a'
+		const start = startIndex + a.length;
+
+		// Find the index of 'b' after 'a'
+		const endIndex = annotationtext.indexOf(b, start);
+
+		// If 'b' is not found, return text till the end
+		if (endIndex === -1) return annotationtext.substring(start).trim();;
+
+		// Return text between 'a' and 'b'
+		return annotationtext.substring(start, endIndex).trim();
+	},
 
 	async itemnotes(item)
 	{
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,7 +11,6 @@ Notes = { @@
     			this.body.style.fontSize = fontsize+"px";
     		}
     		this.fontsize = 1;
     	},
     	isvertical()
@@ Expand Down @@